From caea72438e2b4459071311f244b29375fadc1228 Mon Sep 17 00:00:00 2001 From: Bertrand Darbon Date: Wed, 22 Feb 2023 18:21:18 +0100 Subject: [PATCH 01/10] feat(BTC): Use cached balance on bitcoin accounts On update on bitcoin_outputs, the sql trigger updates the account balance; --- core/src/database/DatabaseSessionPool.hpp | 2 +- core/src/database/migrations.cpp | 75 +++++++++++++++++++ core/src/database/migrations.hpp | 6 ++ .../src/wallet/bitcoin/BitcoinLikeAccount.cpp | 6 +- .../BitcoinLikeUTXODatabaseHelper.cpp | 11 +-- .../database/BitcoinLikeUTXODatabaseHelper.h | 5 +- 6 files changed, 90 insertions(+), 15 deletions(-) diff --git a/core/src/database/DatabaseSessionPool.hpp b/core/src/database/DatabaseSessionPool.hpp index 542996aae1..f27dad5f69 100644 --- a/core/src/database/DatabaseSessionPool.hpp +++ b/core/src/database/DatabaseSessionPool.hpp @@ -63,7 +63,7 @@ namespace ledger { const std::string &dbName, const std::string &password = ""); - static const int CURRENT_DATABASE_SCHEME_VERSION = 31; + static const int CURRENT_DATABASE_SCHEME_VERSION = 32; void performDatabaseMigration(); void performDatabaseRollback(); diff --git a/core/src/database/migrations.cpp b/core/src/database/migrations.cpp index a7272ef27a..39075a4c22 100644 --- a/core/src/database/migrations.cpp +++ b/core/src/database/migrations.cpp @@ -1265,5 +1265,80 @@ namespace ledger { void rollback<31>(soci::session &sql, api::DatabaseBackendType /*type*/) { sql << "ALTER TABLE tezos_transactions DROP explorer_id"; } + + template <> + void migrate<32>(soci::session &sql, api::DatabaseBackendType /*type*/) { + sql << R"( +ALTER TABLE bitcoin_accounts + ADD COLUMN balance BIGINT default 0; + +CREATE OR REPLACE function sum_utxos(p_account_uid varchar(255), p_dust_amount integer) + RETURNS BIGINT + LANGUAGE plpgsql +AS +$$ +DECLARE + balance BIGINT; +BEGIN + SELECT sum(o.amount)::bigint + INTO balance + FROM bitcoin_outputs AS o + LEFT OUTER JOIN bitcoin_inputs AS i ON i.previous_tx_uid = o.transaction_uid + AND i.previous_output_idx = o.idx + WHERE i.previous_tx_uid IS NULL + AND o.account_uid = p_account_uid + AND o.amount > p_dust_amount; + + return balance; +end; +$$; + +CREATE OR REPLACE FUNCTION update_account_balance() + RETURNS TRIGGER + LANGUAGE PLPGSQL +AS +$$ +DECLARE + new_balance BIGINT; +BEGIN + SELECT sum_utxos INTO new_balance FROM sum_utxos(NEW.account_uid, 0); + + UPDATE bitcoin_accounts SET balance = new_balance where uid = NEW.account_uid; + + RETURN NEW; +end; +$$; + +CREATE OR REPLACE TRIGGER bitcoin_outputs_changed + AFTER INSERT OR UPDATE + ON bitcoin_outputs + FOR EACH ROW +EXECUTE PROCEDURE update_account_balance(); + + +-- Populate account balance for first time +DO +$$ + DECLARE + account bitcoin_accounts%rowtype; + BEGIN + FOR account in SELECT * FROM bitcoin_accounts + LOOP + update bitcoin_accounts + set balance = sum_utxos(account.uid, 0) + where uid = account.uid; + END LOOP; + end; +$$; +)"; + } + + template <> + void rollback<32>(soci::session &sql, api::DatabaseBackendType /*type*/) { + sql << "DROP TRIGGER bitcoin_outputs_changed on bitcoin_outputs;"; + sql << "DROP FUNCTION update_account_balance;"; + sql << "DROP FUNCTION sum_utxos;"; + sql << "ALTER TABLE bitcoin_accounts DROP COLUMN balance;"; + } } // namespace core } // namespace ledger diff --git a/core/src/database/migrations.hpp b/core/src/database/migrations.hpp index bdfae156e8..07dd47e8c5 100644 --- a/core/src/database/migrations.hpp +++ b/core/src/database/migrations.hpp @@ -293,6 +293,12 @@ namespace ledger { void migrate<31>(soci::session &sql, api::DatabaseBackendType type); template <> void rollback<31>(soci::session &sql, api::DatabaseBackendType type); + + // add explorer uid for tezos transactions + template <> + void migrate<32>(soci::session &sql, api::DatabaseBackendType type); + template <> + void rollback<32>(soci::session &sql, api::DatabaseBackendType type); } // namespace core } // namespace ledger diff --git a/core/src/wallet/bitcoin/BitcoinLikeAccount.cpp b/core/src/wallet/bitcoin/BitcoinLikeAccount.cpp index e63eac73f0..026fa8b059 100644 --- a/core/src/wallet/bitcoin/BitcoinLikeAccount.cpp +++ b/core/src/wallet/bitcoin/BitcoinLikeAccount.cpp @@ -577,10 +577,8 @@ namespace ledger { return FuturePtr::async(getWallet()->getPool()->getThreadPoolExecutionContext(), [=]() -> std::shared_ptr { const auto &uid = self->getAccountUid(); soci::session sql(self->getWallet()->getDatabase()->getReadonlyPool()); - auto keychain = self->getKeychain(); - // make sure ALL UTXOs are contained within the balance - although no transaction will be crafted with those - constexpr auto worthlessUtxoAmount = 0; - const auto sum = BitcoinLikeUTXODatabaseHelper::sumUTXO(sql, uid, worthlessUtxoAmount); + auto keychain = self->getKeychain(); + const auto sum = BitcoinLikeUTXODatabaseHelper::getBalance(sql, uid); const Amount balance(self->getWallet()->getCurrency(), 0, sum); self->getWallet()->updateBalanceCache(self->getIndex(), balance); return std::make_shared(balance); diff --git a/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.cpp b/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.cpp index 3bb89e491a..829d8301f4 100644 --- a/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.cpp +++ b/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.cpp @@ -86,13 +86,10 @@ namespace ledger { return out.size(); } - BigInt BitcoinLikeUTXODatabaseHelper::sumUTXO(soci::session &sql, const std::string &accountUid, int64_t dustAmount) { - const rowset rows = (sql.prepare << "SELECT sum(o.amount)::bigint" - " FROM bitcoin_outputs AS o " - " LEFT OUTER JOIN bitcoin_inputs AS i ON i.previous_tx_uid = o.transaction_uid " - " AND i.previous_output_idx = o.idx" - " WHERE i.previous_tx_uid IS NULL AND o.account_uid = :uid AND o.amount > :dustAmount", - use(accountUid), use(dustAmount)); + BigInt BitcoinLikeUTXODatabaseHelper::getBalance(soci::session &sql, const std::string &accountUid) { + // balance is computed by a sql trigger in migrations.cpp step 31 + const rowset rows = (sql.prepare << "SELECT balance from bitcoin_accounts WHERE uid=:uid", + use(accountUid)); for (auto &row : rows) { if (row.get_indicator(0) != i_null) { diff --git a/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h b/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h index a03db8fa7a..4bc9c9ddeb 100644 --- a/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h +++ b/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h @@ -51,9 +51,8 @@ namespace ledger { int64_t dustAmount, std::vector &out); - static BigInt sumUTXO(soci::session &sql, - const std::string &accountUid, - int64_t dustAmount); + static BigInt getBalance(soci::session &sql, + const std::string &accountUid); static std::size_t UTXOcount(soci::session &sql, const std::string &accountUid, From e61be058bcf9ca88db17754a1a32cabe917a85b6 Mon Sep 17 00:00:00 2001 From: Bertrand Darbon Date: Mon, 13 Mar 2023 15:38:33 +0100 Subject: [PATCH 02/10] Have separated trigger in bitcoin_inputs --- core/src/database/migrations.cpp | 66 +++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/core/src/database/migrations.cpp b/core/src/database/migrations.cpp index 39075a4c22..c1b36d4130 100644 --- a/core/src/database/migrations.cpp +++ b/core/src/database/migrations.cpp @@ -1269,51 +1269,81 @@ namespace ledger { template <> void migrate<32>(soci::session &sql, api::DatabaseBackendType /*type*/) { sql << R"( -ALTER TABLE bitcoin_accounts - ADD COLUMN balance BIGINT default 0; +alter table bitcoin_accounts + ADD COLUMN balance BIGINT default 0; -CREATE OR REPLACE function sum_utxos(p_account_uid varchar(255), p_dust_amount integer) +CREATE OR REPLACE function update_balance(p_account_uid varchar(255)) RETURNS BIGINT LANGUAGE plpgsql AS $$ DECLARE - balance BIGINT; + v_balance BIGINT; BEGIN + RAISE WARNING 'Update balance of account %', p_account_uid; SELECT sum(o.amount)::bigint - INTO balance + INTO v_balance FROM bitcoin_outputs AS o LEFT OUTER JOIN bitcoin_inputs AS i ON i.previous_tx_uid = o.transaction_uid AND i.previous_output_idx = o.idx WHERE i.previous_tx_uid IS NULL - AND o.account_uid = p_account_uid - AND o.amount > p_dust_amount; + AND o.account_uid = p_account_uid; - return balance; + UPDATE bitcoin_accounts SET balance = v_balance where uid = p_account_uid; + + RAISE WARNING 'New balance = %', v_balance; + return v_balance; end; $$; -CREATE OR REPLACE FUNCTION update_account_balance() +CREATE OR REPLACE FUNCTION bitcoin_outputs_changed() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ -DECLARE - new_balance BIGINT; BEGIN - SELECT sum_utxos INTO new_balance FROM sum_utxos(NEW.account_uid, 0); - - UPDATE bitcoin_accounts SET balance = new_balance where uid = NEW.account_uid; - + RAISE WARNING 'Triggered bitcoin_outputs_changed with account uid %', NEW.account_uid; + PERFORM update_balance(NEW.account_uid); + RAISE WARNING 'End trigger bitcoin_outputs_changed'; RETURN NEW; end; $$; +CREATE OR REPLACE FUNCTION bitcoin_inputs_changed() + RETURNS TRIGGER + LANGUAGE PLPGSQL +AS +$$ +DECLARE + uid varchar(255); +BEGIN + RAISE WARNING 'Triggered bitcoin_inputs_changed with uid %', NEW.uid; + select o.account_uid into uid from bitcoin_inputs as i + right outer join bitcoin_outputs as o on o.transaction_uid = i.previous_tx_uid + and o.idx = i.previous_output_idx + WHERE i.previous_tx_uid IS NOT NULL and i.uid=NEW.uid; + + IF (uid IS NOT NULL) THEN + RAISE WARNING 'uid IS NOT NULL: %', uid; + PERFORM update_balance(uid); + END IF; + RAISE WARNING 'End trigger bitcoin_inputs_changed'; + + return NEW; +end; +$$; + CREATE OR REPLACE TRIGGER bitcoin_outputs_changed AFTER INSERT OR UPDATE ON bitcoin_outputs FOR EACH ROW -EXECUTE PROCEDURE update_account_balance(); +EXECUTE PROCEDURE bitcoin_outputs_changed(); + +CREATE OR REPLACE TRIGGER bitcoin_inputs_changed + AFTER INSERT OR UPDATE + ON bitcoin_inputs + FOR EACH ROW +EXECUTE PROCEDURE bitcoin_inputs_changed(); -- Populate account balance for first time @@ -1324,9 +1354,7 @@ DO BEGIN FOR account in SELECT * FROM bitcoin_accounts LOOP - update bitcoin_accounts - set balance = sum_utxos(account.uid, 0) - where uid = account.uid; + PERFORM update_balance(account.uid); END LOOP; end; $$; From 90431c85211e89f4f4836f19046017d3db80f7e8 Mon Sep 17 00:00:00 2001 From: Bertrand Darbon Date: Mon, 13 Mar 2023 15:39:36 +0100 Subject: [PATCH 03/10] comment debug raises --- core/src/database/migrations.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/database/migrations.cpp b/core/src/database/migrations.cpp index c1b36d4130..7d800f64b7 100644 --- a/core/src/database/migrations.cpp +++ b/core/src/database/migrations.cpp @@ -1280,7 +1280,7 @@ AS DECLARE v_balance BIGINT; BEGIN - RAISE WARNING 'Update balance of account %', p_account_uid; + -- RAISE NOTICE 'Update balance of account %', p_account_uid; SELECT sum(o.amount)::bigint INTO v_balance FROM bitcoin_outputs AS o @@ -1291,7 +1291,7 @@ BEGIN UPDATE bitcoin_accounts SET balance = v_balance where uid = p_account_uid; - RAISE WARNING 'New balance = %', v_balance; + -- RAISE NOTICE 'New balance = %', v_balance; return v_balance; end; $$; @@ -1302,9 +1302,9 @@ CREATE OR REPLACE FUNCTION bitcoin_outputs_changed() AS $$ BEGIN - RAISE WARNING 'Triggered bitcoin_outputs_changed with account uid %', NEW.account_uid; + -- RAISE NOTICE 'Triggered bitcoin_outputs_changed with account uid %', NEW.account_uid; PERFORM update_balance(NEW.account_uid); - RAISE WARNING 'End trigger bitcoin_outputs_changed'; + -- RAISE NOTICE 'End trigger bitcoin_outputs_changed'; RETURN NEW; end; $$; @@ -1317,17 +1317,17 @@ AS DECLARE uid varchar(255); BEGIN - RAISE WARNING 'Triggered bitcoin_inputs_changed with uid %', NEW.uid; + -- RAISE NOTICE 'Triggered bitcoin_inputs_changed with uid %', NEW.uid; select o.account_uid into uid from bitcoin_inputs as i right outer join bitcoin_outputs as o on o.transaction_uid = i.previous_tx_uid and o.idx = i.previous_output_idx WHERE i.previous_tx_uid IS NOT NULL and i.uid=NEW.uid; IF (uid IS NOT NULL) THEN - RAISE WARNING 'uid IS NOT NULL: %', uid; + -- RAISE NOTICE 'uid IS NOT NULL: %', uid; PERFORM update_balance(uid); END IF; - RAISE WARNING 'End trigger bitcoin_inputs_changed'; + -- RAISE NOTICE 'End trigger bitcoin_inputs_changed'; return NEW; end; From 12e3bf18786fd99ee4cc99e0d80069b7dd730303 Mon Sep 17 00:00:00 2001 From: Bertrand Darbon Date: Tue, 14 Mar 2023 15:29:34 +0100 Subject: [PATCH 04/10] Fix rollback --- core/src/database/migrations.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/database/migrations.cpp b/core/src/database/migrations.cpp index 7d800f64b7..81e20f92a8 100644 --- a/core/src/database/migrations.cpp +++ b/core/src/database/migrations.cpp @@ -1364,8 +1364,10 @@ DO template <> void rollback<32>(soci::session &sql, api::DatabaseBackendType /*type*/) { sql << "DROP TRIGGER bitcoin_outputs_changed on bitcoin_outputs;"; - sql << "DROP FUNCTION update_account_balance;"; - sql << "DROP FUNCTION sum_utxos;"; + sql << "DROP TRIGGER bitcoin_inputs_changed on bitcoin_inputs;"; + sql << "DROP FUNCTION update_balance;"; + sql << "DROP FUNCTION bitcoin_outputs_changed;"; + sql << "DROP FUNCTION bitcoin_inputs_changed;"; sql << "ALTER TABLE bitcoin_accounts DROP COLUMN balance;"; } } // namespace core From c0a0e468fcc13428a345d1ca7fd8a85890017469 Mon Sep 17 00:00:00 2001 From: Bertrand Darbon Date: Tue, 14 Mar 2023 15:29:43 +0100 Subject: [PATCH 05/10] Use postgres 14 for tests --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f2ddd949a0..9083759b95 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,7 +21,7 @@ jobs: password: ${{ secrets.CI_BOT_TOKEN }} services: postgres: - image: postgres:12 + image: postgres:14 env: POSTGRES_USER: libcore POSTGRES_PASSWORD: libcore_pwd From b1d763e01a345a71e0b232652f455effcd744f0d Mon Sep 17 00:00:00 2001 From: Bertrand Darbon Date: Tue, 14 Mar 2023 15:51:32 +0100 Subject: [PATCH 06/10] Use postgres 14 for tests `OR REPLACE` on triggers is only supported from postgres 14 https://www.postgresql.org/docs/14/sql-createtrigger.html --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9083759b95..c8a798837d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,8 +86,8 @@ jobs: - name: Setup Macos run: | brew install openssl - brew install postgresql@13 - brew link --overwrite postgresql@13 + brew install postgresql@14 + brew link --overwrite postgresql@14 - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: From 4ed907fcca0291190a9f8b84c10de41de6be920f Mon Sep 17 00:00:00 2001 From: Bertrand Darbon Date: Thu, 6 Apr 2023 11:16:39 +0200 Subject: [PATCH 07/10] Remove triggers & update balance after sync --- .github/workflows/tests.yml | 2 +- core/src/database/migrations.cpp | 97 +------------------ .../BitcoinLikeUTXODatabaseHelper.cpp | 32 +++++- .../database/BitcoinLikeUTXODatabaseHelper.h | 4 + .../BlockchainExplorerAccountSynchronizer.cpp | 6 ++ 5 files changed, 42 insertions(+), 99 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c8a798837d..935c02d005 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,7 +21,7 @@ jobs: password: ${{ secrets.CI_BOT_TOKEN }} services: postgres: - image: postgres:14 + image: postgres:12 env: POSTGRES_USER: libcore POSTGRES_PASSWORD: libcore_pwd diff --git a/core/src/database/migrations.cpp b/core/src/database/migrations.cpp index 81e20f92a8..af9eb0bfec 100644 --- a/core/src/database/migrations.cpp +++ b/core/src/database/migrations.cpp @@ -1268,106 +1268,11 @@ namespace ledger { template <> void migrate<32>(soci::session &sql, api::DatabaseBackendType /*type*/) { - sql << R"( -alter table bitcoin_accounts - ADD COLUMN balance BIGINT default 0; - -CREATE OR REPLACE function update_balance(p_account_uid varchar(255)) - RETURNS BIGINT - LANGUAGE plpgsql -AS -$$ -DECLARE - v_balance BIGINT; -BEGIN - -- RAISE NOTICE 'Update balance of account %', p_account_uid; - SELECT sum(o.amount)::bigint - INTO v_balance - FROM bitcoin_outputs AS o - LEFT OUTER JOIN bitcoin_inputs AS i ON i.previous_tx_uid = o.transaction_uid - AND i.previous_output_idx = o.idx - WHERE i.previous_tx_uid IS NULL - AND o.account_uid = p_account_uid; - - UPDATE bitcoin_accounts SET balance = v_balance where uid = p_account_uid; - - -- RAISE NOTICE 'New balance = %', v_balance; - return v_balance; -end; -$$; - -CREATE OR REPLACE FUNCTION bitcoin_outputs_changed() - RETURNS TRIGGER - LANGUAGE PLPGSQL -AS -$$ -BEGIN - -- RAISE NOTICE 'Triggered bitcoin_outputs_changed with account uid %', NEW.account_uid; - PERFORM update_balance(NEW.account_uid); - -- RAISE NOTICE 'End trigger bitcoin_outputs_changed'; - RETURN NEW; -end; -$$; - -CREATE OR REPLACE FUNCTION bitcoin_inputs_changed() - RETURNS TRIGGER - LANGUAGE PLPGSQL -AS -$$ -DECLARE - uid varchar(255); -BEGIN - -- RAISE NOTICE 'Triggered bitcoin_inputs_changed with uid %', NEW.uid; - select o.account_uid into uid from bitcoin_inputs as i - right outer join bitcoin_outputs as o on o.transaction_uid = i.previous_tx_uid - and o.idx = i.previous_output_idx - WHERE i.previous_tx_uid IS NOT NULL and i.uid=NEW.uid; - - IF (uid IS NOT NULL) THEN - -- RAISE NOTICE 'uid IS NOT NULL: %', uid; - PERFORM update_balance(uid); - END IF; - -- RAISE NOTICE 'End trigger bitcoin_inputs_changed'; - - return NEW; -end; -$$; - -CREATE OR REPLACE TRIGGER bitcoin_outputs_changed - AFTER INSERT OR UPDATE - ON bitcoin_outputs - FOR EACH ROW -EXECUTE PROCEDURE bitcoin_outputs_changed(); - -CREATE OR REPLACE TRIGGER bitcoin_inputs_changed - AFTER INSERT OR UPDATE - ON bitcoin_inputs - FOR EACH ROW -EXECUTE PROCEDURE bitcoin_inputs_changed(); - - --- Populate account balance for first time -DO -$$ - DECLARE - account bitcoin_accounts%rowtype; - BEGIN - FOR account in SELECT * FROM bitcoin_accounts - LOOP - PERFORM update_balance(account.uid); - END LOOP; - end; -$$; -)"; + sql << "ALTER TABLE bitcoin_accounts ADD COLUMN balance BIGINT DEFAULT -1;"; } template <> void rollback<32>(soci::session &sql, api::DatabaseBackendType /*type*/) { - sql << "DROP TRIGGER bitcoin_outputs_changed on bitcoin_outputs;"; - sql << "DROP TRIGGER bitcoin_inputs_changed on bitcoin_inputs;"; - sql << "DROP FUNCTION update_balance;"; - sql << "DROP FUNCTION bitcoin_outputs_changed;"; - sql << "DROP FUNCTION bitcoin_inputs_changed;"; sql << "ALTER TABLE bitcoin_accounts DROP COLUMN balance;"; } } // namespace core diff --git a/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.cpp b/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.cpp index 829d8301f4..d4aa31ba92 100644 --- a/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.cpp +++ b/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.cpp @@ -86,19 +86,47 @@ namespace ledger { return out.size(); } + constexpr auto uncachedBalanceQuery = R"( + SELECT sum(o.amount)::bigint + FROM bitcoin_outputs AS o + LEFT OUTER JOIN bitcoin_inputs AS i ON i.previous_tx_uid = o.transaction_uid + AND i.previous_output_idx = o.idx + WHERE i.previous_tx_uid IS NULL + AND o.account_uid = :uid)"; + + BigInt getUncachedBalance(soci::session &sql, const std::string &accountUid) { + const rowset rows = (sql.prepare << uncachedBalanceQuery, use(accountUid)); + for (auto &row : rows) { + if (row.get_indicator(0) != i_null) { + return row.get(0); + } + } + return BigInt(0); + } + BigInt BitcoinLikeUTXODatabaseHelper::getBalance(soci::session &sql, const std::string &accountUid) { - // balance is computed by a sql trigger in migrations.cpp step 31 const rowset rows = (sql.prepare << "SELECT balance from bitcoin_accounts WHERE uid=:uid", use(accountUid)); for (auto &row : rows) { if (row.get_indicator(0) != i_null) { - return row.get(0); + auto balance = row.get(0); + if (balance.isNegative()) { + // We compute balance from DB instead of updating it, to let only lama-bitcoin doing the update + return getUncachedBalance(sql, accountUid); + } + return balance; } } return BigInt(0); } + void BitcoinLikeUTXODatabaseHelper::updateBalance(session &sql, const std::string &accountUid) { + // Only used for tests (by libcore btc sync), production update of balance is made by lama-bitcoin + const rowset rows = (sql.prepare << std::string{"UPDATE bitcoin_accounts SET balance = ("} + uncachedBalanceQuery + ") WHERE uid = :uid;", + use(accountUid), use(accountUid)); + } + std::vector BitcoinLikeUTXODatabaseHelper::queryAllUtxos( soci::session &session, std::string const &accountUid, diff --git a/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h b/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h index 4bc9c9ddeb..e080b9d688 100644 --- a/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h +++ b/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h @@ -54,6 +54,10 @@ namespace ledger { static BigInt getBalance(soci::session &sql, const std::string &accountUid); + // Only used for tests (by libcore btc sync) + static void updateBalance(soci::session &sql, + const std::string &accountUid); + static std::size_t UTXOcount(soci::session &sql, const std::string &accountUid, int64_t dustAmount); diff --git a/core/src/wallet/bitcoin/synchronizers/BlockchainExplorerAccountSynchronizer.cpp b/core/src/wallet/bitcoin/synchronizers/BlockchainExplorerAccountSynchronizer.cpp index 585b5fd4b1..7b04c7923d 100644 --- a/core/src/wallet/bitcoin/synchronizers/BlockchainExplorerAccountSynchronizer.cpp +++ b/core/src/wallet/bitcoin/synchronizers/BlockchainExplorerAccountSynchronizer.cpp @@ -30,6 +30,8 @@ */ #include "BlockchainExplorerAccountSynchronizer.h" +#include "wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h" + #include #include #include @@ -535,6 +537,10 @@ namespace ledger { buddy->account->getWallet()->getName(), duration.count()); buddy->logger->error("Due to {}, {}", api::to_string(ex.getErrorCode()), ex.getMessage()); return buddy->context; + }) + .then(account->getContext(), [account, buddy]() { + soci::session sql(buddy->wallet->getDatabase()->getPool()); + BitcoinLikeUTXODatabaseHelper::updateBalance(sql, account->getAccountUid()); }); }; From 5adf3fb85e5e1b62879b5ca2c0e9d9a1092766f8 Mon Sep 17 00:00:00 2001 From: Bertrand Darbon Date: Thu, 6 Apr 2023 11:24:31 +0200 Subject: [PATCH 08/10] don't need to upgrade postgres --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 935c02d005..f2ddd949a0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,8 +86,8 @@ jobs: - name: Setup Macos run: | brew install openssl - brew install postgresql@14 - brew link --overwrite postgresql@14 + brew install postgresql@13 + brew link --overwrite postgresql@13 - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: From fab4ba85f7f0c93ef9a764579a79f61c8f0e6dd0 Mon Sep 17 00:00:00 2001 From: Bertrand Darbon Date: Thu, 6 Apr 2023 13:32:15 +0200 Subject: [PATCH 09/10] format --- .../src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h b/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h index e080b9d688..b54a4938ba 100644 --- a/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h +++ b/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.h @@ -56,7 +56,7 @@ namespace ledger { // Only used for tests (by libcore btc sync) static void updateBalance(soci::session &sql, - const std::string &accountUid); + const std::string &accountUid); static std::size_t UTXOcount(soci::session &sql, const std::string &accountUid, From c0c90441a602c42a93fd588a59d14bcc401478f7 Mon Sep 17 00:00:00 2001 From: Bertrand Darbon Date: Tue, 11 Apr 2023 11:31:01 +0200 Subject: [PATCH 10/10] Update misleading comments --- .../wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.cpp b/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.cpp index d4aa31ba92..447a429934 100644 --- a/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.cpp +++ b/core/src/wallet/bitcoin/database/BitcoinLikeUTXODatabaseHelper.cpp @@ -112,7 +112,7 @@ namespace ledger { if (row.get_indicator(0) != i_null) { auto balance = row.get(0); if (balance.isNegative()) { - // We compute balance from DB instead of updating it, to let only lama-bitcoin doing the update + // We compute balance from DB instead of updating it, to let only the sync (lama-bitcoin or libcore, depending on the coin) doing the update return getUncachedBalance(sql, accountUid); } return balance; @@ -122,7 +122,6 @@ namespace ledger { } void BitcoinLikeUTXODatabaseHelper::updateBalance(session &sql, const std::string &accountUid) { - // Only used for tests (by libcore btc sync), production update of balance is made by lama-bitcoin const rowset rows = (sql.prepare << std::string{"UPDATE bitcoin_accounts SET balance = ("} + uncachedBalanceQuery + ") WHERE uid = :uid;", use(accountUid), use(accountUid)); }