From e7260d9bed85df0bf0b0ead21e9c0050e3d243f3 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 13 Aug 2025 12:33:06 +0100 Subject: [PATCH] backend: fetch multiple ETH balances at once. To reduce the amount of API calls we make to Etherscan, we bundle together updates to the balance of up to 20 ETH accounts together. --- CHANGELOG.md | 1 + backend/accounts_test.go | 11 ++ backend/backend.go | 116 ++++++++++++++++--- backend/coins/eth/account.go | 137 +++++++++++++---------- backend/coins/eth/account_test.go | 3 + backend/coins/eth/coin.go | 5 + backend/coins/eth/etherscan/etherscan.go | 63 ++++++++++- 7 files changed, 257 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8817a7a814..50e87cc194 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Added a banner to remind user to backup their seed phrase when an account reaches a certain threshold. - Gracefully shut down Electrum connections upon closing the app - Show the selected coin's address on the confirmation screen +- Fetch balances of multiple ETH accounts at the same time, instead of one by one. ## v4.48.3 - Linux: fix compatiblity with some versions of Mesa also when using the AppImage diff --git a/backend/accounts_test.go b/backend/accounts_test.go index 72acd9b923..e07d4bc9fa 100644 --- a/backend/accounts_test.go +++ b/backend/accounts_test.go @@ -179,11 +179,13 @@ func TestSortAccounts(t *testing.T) { {Code: "acct-tbtc", CoinCode: coinpkg.CodeTBTC}, } backend := newBackend(t, testnetDisabled, regtestDisabled) + unlockFN := backend.accountsAndKeystoreLock.Lock() for i := range accountConfigs { c, err := backend.Coin(accountConfigs[i].CoinCode) require.NoError(t, err) backend.createAndAddAccount(c, accountConfigs[i]) } + unlockFN() expectedOrder := []accountsTypes.Code{ "acct-btc-1", @@ -747,6 +749,7 @@ func TestCreateAndAddAccount(t *testing.T) { // Add a Bitcoin account. coin, err := b.Coin(coinpkg.CodeBTC) require.NoError(t, err) + unlockFN := b.accountsAndKeystoreLock.Lock() b.createAndAddAccount( coin, &config.Account{ @@ -757,6 +760,7 @@ func TestCreateAndAddAccount(t *testing.T) { }, }, ) + unlockFN() require.Len(t, b.Accounts(), 1) // Check some properties of the newly added account. acct := b.Accounts()[0] @@ -767,6 +771,8 @@ func TestCreateAndAddAccount(t *testing.T) { // Add a Litecoin account. coin, err = b.Coin(coinpkg.CodeLTC) require.NoError(t, err) + + unlockFN = b.accountsAndKeystoreLock.Lock() b.createAndAddAccount(coin, &config.Account{ Code: "test-ltc-account-code", @@ -776,6 +782,7 @@ func TestCreateAndAddAccount(t *testing.T) { }, }, ) + unlockFN() require.Len(t, b.Accounts(), 2) // Check some properties of the newly added account. acct = b.Accounts()[1] @@ -786,6 +793,7 @@ func TestCreateAndAddAccount(t *testing.T) { // Add an Ethereum account with some active ERC20 tokens. coin, err = b.Coin(coinpkg.CodeETH) require.NoError(t, err) + unlockFN = b.accountsAndKeystoreLock.Lock() b.createAndAddAccount(coin, &config.Account{ Code: "test-eth-account-code", @@ -796,6 +804,7 @@ func TestCreateAndAddAccount(t *testing.T) { ActiveTokens: []string{"eth-erc20-mkr"}, }, ) + unlockFN() // 2 more accounts: the added ETH account plus the active token for the ETH account. require.Len(t, b.Accounts(), 4) // Check some properties of the newly added account. @@ -812,6 +821,7 @@ func TestCreateAndAddAccount(t *testing.T) { // Add another Ethereum account with some active ERC20 tokens. coin, err = b.Coin(coinpkg.CodeETH) require.NoError(t, err) + unlockFN = b.accountsAndKeystoreLock.Lock() b.createAndAddAccount(coin, &config.Account{ Code: "test-eth-account-code-2", @@ -823,6 +833,7 @@ func TestCreateAndAddAccount(t *testing.T) { ActiveTokens: []string{"eth-erc20-usdt", "eth-erc20-bat"}, }, ) + unlockFN() // 3 more accounts: the added ETH account plus the two active tokens for the ETH account. require.Len(t, b.Accounts(), 7) // Check some properties of the newly added accounts. diff --git a/backend/backend.go b/backend/backend.go index 4596e28c42..f39589039e 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -61,6 +61,7 @@ import ( "github.com/btcsuite/btcd/chaincfg" "github.com/ethereum/go-ethereum/params" "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) func init() { @@ -225,10 +226,11 @@ type Backend struct { socksProxy socksproxy.SocksProxy // can be a regular or, if Tor is enabled in the config, a SOCKS5 proxy client. - httpClient *http.Client - etherScanHTTPClient *http.Client - ratesUpdater *rates.RateUpdater - banners *banners.Banners + httpClient *http.Client + etherScanHTTPClient *http.Client + etherScanRateLimiter *rate.Limiter + ratesUpdater *rates.RateUpdater + banners *banners.Banners // For unit tests, called when `backend.checkAccountUsed()` is called. tstCheckAccountUsed func(accounts.Interface) bool @@ -240,6 +242,15 @@ type Backend struct { // isOnline indicates whether the backend is online, i.e. able to connect to the internet. isOnline atomic.Bool + + // quit is used to indicate to running goroutines that they should stop as the backend is being closed + quit chan struct{} + + // enqueueUpdateForAccount is used to enqueue an update for an account. + enqueueUpdateForAccount chan *eth.Account + + // updateETHAccountsCh is used to trigger an update of all ETH accounts. + updateETHAccountsCh chan struct{} } // NewBackend creates a new backend with the given arguments. @@ -262,11 +273,13 @@ func NewBackend(arguments *arguments.Arguments, environment Environment) (*Backe return nil, err } + accountUpdate := make(chan *eth.Account) + backend := &Backend{ arguments: arguments, environment: environment, config: backendConfig, - events: make(chan interface{}, 1000), + events: make(chan interface{}), devices: map[string]device.Interface{}, coins: map[coinpkg.Code]coinpkg.Coin{}, @@ -276,12 +289,15 @@ func NewBackend(arguments *arguments.Arguments, environment Environment) (*Backe return btc.NewAccount(config, coin, gapLimits, getAddress, log, hclient) }, makeEthAccount: func(config *accounts.AccountConfig, coin *eth.Coin, httpClient *http.Client, log *logrus.Entry) accounts.Interface { - return eth.NewAccount(config, coin, httpClient, log) + return eth.NewAccount(config, coin, httpClient, log, accountUpdate) }, log: log, - testing: backendConfig.AppConfig().Backend.StartInTestnet || arguments.Testing(), + testing: backendConfig.AppConfig().Backend.StartInTestnet || arguments.Testing(), + quit: make(chan struct{}), + etherScanRateLimiter: rate.NewLimiter(rate.Limit(etherscan.CallsPerSec), 1), + enqueueUpdateForAccount: accountUpdate, } // TODO: remove when connectivity check is present on all platforms backend.isOnline.Store(true) @@ -529,19 +545,19 @@ func (backend *Backend) Coin(code coinpkg.Code) (coinpkg.Coin, error) { coin = btc.NewCoin(coinpkg.CodeLTC, "Litecoin", "LTC", coinpkg.BtcUnitDefault, <c.MainNetParams, dbFolder, servers, "https://blockchair.com/litecoin/transaction/", backend.socksProxy) case code == coinpkg.CodeETH: - etherScan := etherscan.NewEtherScan("1", backend.etherScanHTTPClient) + etherScan := etherscan.NewEtherScan("1", backend.etherScanHTTPClient, backend.etherScanRateLimiter) coin = eth.NewCoin(etherScan, code, "Ethereum", "ETH", "ETH", params.MainnetChainConfig, "https://etherscan.io/tx/", etherScan, nil) case code == coinpkg.CodeSEPETH: - etherScan := etherscan.NewEtherScan("11155111", backend.etherScanHTTPClient) + etherScan := etherscan.NewEtherScan("11155111", backend.etherScanHTTPClient, backend.etherScanRateLimiter) coin = eth.NewCoin(etherScan, code, "Ethereum Sepolia", "SEPETH", "SEPETH", params.SepoliaChainConfig, "https://sepolia.etherscan.io/tx/", etherScan, nil) case erc20Token != nil: - etherScan := etherscan.NewEtherScan("1", backend.etherScanHTTPClient) + etherScan := etherscan.NewEtherScan("1", backend.etherScanHTTPClient, backend.etherScanRateLimiter) coin = eth.NewCoin(etherScan, erc20Token.code, erc20Token.name, erc20Token.unit, "ETH", params.MainnetChainConfig, "https://etherscan.io/tx/", etherScan, @@ -555,6 +571,74 @@ func (backend *Backend) Coin(code coinpkg.Code) (coinpkg.Coin, error) { return coin, nil } +func (backend *Backend) pollETHAccounts() { + timer := time.After(0) + + updateAll := func() { + if err := backend.updateETHAccounts(); err != nil { + backend.log.WithError(err).Error("could not update ETH accounts") + } + } + + for { + select { + case <-backend.quit: + return + default: + select { + case <-backend.quit: + return + case account := <-backend.enqueueUpdateForAccount: + go func() { + // A single ETH accounts needs an update. + ethCoin, ok := account.Coin().(*eth.Coin) + if !ok { + backend.log.WithField("account", account.Config().Config.Name).Errorf("expected ETH account to have ETH coin, got %T", account.Coin()) + } + etherScanClient := etherscan.NewEtherScan(ethCoin.ChainIDstr(), backend.etherScanHTTPClient, backend.etherScanRateLimiter) + if err := eth.UpdateBalances([]*eth.Account{account}, etherScanClient); err != nil { + backend.log.WithError(err).Errorf("could not update account %s", account.Config().Config.Name) + } + }() + case <-backend.updateETHAccountsCh: + go updateAll() + timer = time.After(eth.PollInterval) + case <-timer: + go updateAll() + timer = time.After(eth.PollInterval) + } + } + } +} + +func (backend *Backend) updateETHAccounts() error { + defer backend.accountsAndKeystoreLock.RLock()() + backend.log.Debug("Updating ETH accounts balances") + + accountsChainID := map[string][]*eth.Account{} + for _, account := range backend.accounts { + ethAccount, ok := account.(*eth.Account) + if ok { + ethCoin, ok := ethAccount.Coin().(*eth.Coin) + if !ok { + return errp.Newf("expected ETH account to have ETH coin, got %T", ethAccount.Coin()) + } + chainID := ethCoin.ChainIDstr() + accountsChainID[chainID] = append(accountsChainID[chainID], ethAccount) + } + + } + + for chainID, ethAccounts := range accountsChainID { + etherScanClient := etherscan.NewEtherScan(chainID, backend.etherScanHTTPClient, backend.etherScanRateLimiter) + if err := eth.UpdateBalances(ethAccounts, etherScanClient); err != nil { + backend.log.WithError(err).Errorf("could not update ETH accounts for chain ID %s", chainID) + } + } + + return nil +} + // ManualReconnect triggers reconnecting to Electrum servers if their connection is down. // Only coin connections that were previously established are reconnected. // Calling this is a no-op for coins that are already connected. @@ -592,13 +676,7 @@ func (backend *Backend) ManualReconnect(reconnectETH bool) { } if reconnectETH { backend.log.Info("Reconnecting ETH accounts") - for _, account := range backend.accounts { - ethAccount, ok := account.(*eth.Account) - if !ok { - continue - } - ethAccount.EnqueueUpdate() - } + backend.updateETHAccountsCh <- struct{}{} } } @@ -661,6 +739,8 @@ func (backend *Backend) Start() <-chan interface{} { backend.environment.OnAuthSettingChanged(backend.config.AppConfig().Backend.Authentication) + go backend.pollETHAccounts() + if backend.config.AppConfig().Backend.StartInTestnet { if err := backend.config.ModifyAppConfig(func(c *config.AppConfig) error { c.Backend.StartInTestnet = false; return nil }); err != nil { backend.log.WithError(err).Error("Can't set StartInTestnet to false") @@ -933,6 +1013,8 @@ func (backend *Backend) Close() error { if len(errors) > 0 { return errp.New(strings.Join(errors, "; ")) } + + close(backend.quit) return nil } diff --git a/backend/coins/eth/account.go b/backend/coins/eth/account.go index ab29d9b0a0..27e7ce26ab 100644 --- a/backend/coins/eth/account.go +++ b/backend/coins/eth/account.go @@ -49,7 +49,8 @@ import ( "github.com/sirupsen/logrus" ) -var pollInterval = 5 * time.Minute +// PollInterval is the interval at which the account is polled for updates. +var PollInterval = 5 * time.Minute func isMixedCase(s string) bool { return strings.ToLower(s) != s && strings.ToUpper(s) != s @@ -87,7 +88,7 @@ type Account struct { // enqueueUpdateCh is used to invoke an account update outside of the regular poll update // interval. - enqueueUpdateCh chan struct{} + enqueueUpdateCh chan *Account address Address @@ -100,11 +101,10 @@ type Account struct { // if not nil, SendTx() will sign and send this transaction. Set by TxProposal(). activeTxProposal *TxProposal - // quitChan is used to send a quit signal to the accounts long running routines that - // should listen to it. - quitChan chan struct{} - log *logrus.Entry + + // initDone is called when the account is initialized for the first time + initDone func() } // NewAccount creates a new account. @@ -113,6 +113,7 @@ func NewAccount( accountCoin *Coin, httpClient *http.Client, log *logrus.Entry, + enqueueUpdateCh chan *Account, ) *Account { log = log.WithField("group", "eth"). WithFields(logrus.Fields{"coin": accountCoin.String(), "code": config.Config.Code, "name": config.Config.Name}) @@ -126,8 +127,7 @@ func NewAccount( httpClient: httpClient, balance: coin.NewAmountFromInt64(0), - enqueueUpdateCh: make(chan struct{}), - quitChan: make(chan struct{}), + enqueueUpdateCh: enqueueUpdateCh, log: log, } @@ -207,41 +207,12 @@ func (account *Account) Initialize() error { ) account.coin.Initialize() - done := account.Synchronizer.IncRequestsCounter() - go account.poll(done) + account.initDone = account.Synchronizer.IncRequestsCounter() + go account.EnqueueUpdate() return account.BaseAccount.Initialize(accountIdentifier) } -func (account *Account) poll(initDone func()) { - timer := time.After(0) - for { - select { - case <-account.quitChan: - return - default: - select { - case <-account.quitChan: - return - case <-timer: - case <-account.enqueueUpdateCh: - account.log.Info("extraordinary account update invoked") - } - if err := account.update(); err != nil { - account.log.WithError(err).Error("error updating account") - account.SetOffline(err) - } else { - account.SetOffline(nil) - } - if initDone != nil { - initDone() - initDone = nil - } - timer = time.After(pollInterval) - } - } -} - // updateOutgoingTransactions updates the height of the stored outgoing transactions. // We update heights for tx with up to 12 confirmations, so re-orgs are taken into account. // tipHeight is the current blockchain height. @@ -375,7 +346,9 @@ func (account *Account) nextNonce() (uint64, error) { return nextNonce, nil } -func (account *Account) update() error { +// Update performs an Update of the account's transaction +// and its balance, which must be provided as an argument. +func (account *Account) Update(balance *big.Int) error { defer account.updateLock.Lock()() defer account.Synchronizer.IncRequestsCounter()() @@ -416,22 +389,14 @@ func (account *Account) update() error { } } - var balance *big.Int - if account.coin.erc20Token != nil { - balance, err = account.coin.client.ERC20Balance(account.address.Address, account.coin.erc20Token) - if err != nil { - return errp.WithStack(err) - } - } else { - balance, err = account.coin.client.Balance(context.TODO(), account.address.Address) - if err != nil { - return errp.WithStack(err) - } - } - pendingAmount := pendingTxsAmount(outgoingTransactionsData, account.coin.erc20Token != nil) account.balance = coin.NewAmount(balance.Sub(balance, pendingAmount)) + if account.initDone != nil { + account.initDone() + account.initDone = nil + } + return nil } @@ -472,7 +437,6 @@ func (account *Account) Close() { } account.log.Info("Closed DB") } - close(account.quitChan) account.closed = true account.Notify(observable.Event{ Subject: string(accountsTypes.EventStatusChanged), @@ -1052,8 +1016,67 @@ func (account *Account) MatchesAddress(address string) (bool, error) { // EnqueueUpdate enqueues an update for the account. func (account *Account) EnqueueUpdate() { - select { - case account.enqueueUpdateCh <- struct{}{}: - default: + account.enqueueUpdateCh <- account +} + +// UpdateBalances updates the balances of the accounts in the provided slice. +func UpdateBalances(accounts []*Account, etherScanClient *etherscan.EtherScan) error { + ethNonErc20Addresses := make([]ethcommon.Address, 0, len(accounts)) + for _, account := range accounts { + if account.isClosed() { + continue + } + address, err := account.Address() + if err != nil { + account.log.WithError(err).Errorf("Could not get address for account %s", account.Config().Config.Code) + account.SetOffline(err) + continue + } + if account.coin.erc20Token == nil { + ethNonErc20Addresses = append(ethNonErc20Addresses, address.Address) + } } + + balances, err := etherScanClient.Balances(context.TODO(), ethNonErc20Addresses) + if err != nil { + return errp.WithStack(err) + } + + for _, account := range accounts { + if account.isClosed() { + continue + } + address, err := account.Address() + if err != nil { + account.log.WithError(err).Errorf("Could not get address for account %s", account.Config().Config.Code) + account.SetOffline(err) + continue + } + var balance *big.Int + if account.coin.erc20Token != nil { + var err error + balance, err = account.coin.client.ERC20Balance(account.address.Address, account.coin.erc20Token) + if err != nil { + account.log.WithError(err).Errorf("Could not get ERC20 balance for address %s", address.Address.Hex()) + account.SetOffline(err) + continue + } + } else { + var ok bool + balance, ok = balances[address.Address] + if !ok { + errMsg := fmt.Sprintf("Could not find balance for address %s", address.Address.Hex()) + account.log.Error(errMsg) + account.SetOffline(errp.New(errMsg)) + continue + } + } + if err := account.Update(balance); err != nil { + account.log.WithError(err).Errorf("Could not update balance for address %s", address.Address.Hex()) + account.SetOffline(err) + } else { + account.SetOffline(nil) + } + } + return nil } diff --git a/backend/coins/eth/account_test.go b/backend/coins/eth/account_test.go index 471459c684..a0aa8292aa 100644 --- a/backend/coins/eth/account_test.go +++ b/backend/coins/eth/account_test.go @@ -106,6 +106,7 @@ func newAccount(t *testing.T) *Account { coin, &http.Client{}, log, + make(chan *Account), ) require.NoError(t, acct.Initialize()) return acct @@ -114,6 +115,7 @@ func newAccount(t *testing.T) *Account { func TestTxProposal(t *testing.T) { acct := newAccount(t) defer acct.Close() + require.NoError(t, acct.Update(big.NewInt(1e18))) require.Eventually(t, acct.Synced, time.Second, time.Millisecond*200) t.Run("valid", func(t *testing.T) { @@ -171,6 +173,7 @@ func TestTxProposal(t *testing.T) { func TestMatchesAddress(t *testing.T) { acct := newAccount(t) defer acct.Close() + require.NoError(t, acct.Update(big.NewInt(1e18))) require.Eventually(t, acct.Synced, time.Second, time.Millisecond*200) // Test invalid Ethereum address diff --git a/backend/coins/eth/coin.go b/backend/coins/eth/coin.go index 6b76f7bdaa..ad4162ffb1 100644 --- a/backend/coins/eth/coin.go +++ b/backend/coins/eth/coin.go @@ -107,6 +107,11 @@ func (coin *Coin) Net() *params.ChainConfig { return coin.net } // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-155.md#list-of-chain-ids func (coin *Coin) ChainID() uint64 { return coin.net.ChainID.Uint64() } +// ChainIDstr returns the chain ID of the network as a string. +func (coin *Coin) ChainIDstr() string { + return coin.net.ChainID.String() +} + // Initialize implements coin.Coin. func (coin *Coin) Initialize() {} diff --git a/backend/coins/eth/etherscan/etherscan.go b/backend/coins/eth/etherscan/etherscan.go index 3dffed71a9..6e67d4cbc8 100644 --- a/backend/coins/eth/etherscan/etherscan.go +++ b/backend/coins/eth/etherscan/etherscan.go @@ -23,7 +23,9 @@ import ( "math/big" "net/http" "net/url" + "slices" "strconv" + "strings" "time" "github.com/BitBoxSwiss/bitbox-wallet-app/backend/accounts" @@ -39,12 +41,15 @@ import ( "golang.org/x/time/rate" ) -// callsPerSec is thenumber of etherscanr equests allowed +// CallsPerSec is thenumber of etherscanr equests allowed // per second. // Etherscan rate limits to one request per 0.2 seconds. -var callsPerSec = 3.8 +var CallsPerSec = 3.8 -const apiKey = "X3AFAGQT2QCAFTFPIH9VJY88H9PIQ2UWP7" +const ( + apiKey = "X3AFAGQT2QCAFTFPIH9VJY88H9PIQ2UWP7" + maxAddressesForBalances = 20 +) // ERC20GasErr is the error message returned from etherscan when there is not enough ETH to pay the transaction fee. const ERC20GasErr = "insufficient funds for gas * price + value" @@ -58,11 +63,11 @@ type EtherScan struct { } // NewEtherScan creates a new instance of EtherScan. -func NewEtherScan(chainId string, httpClient *http.Client) *EtherScan { +func NewEtherScan(chainId string, httpClient *http.Client, limiter *rate.Limiter) *EtherScan { return &EtherScan{ url: "https://api.etherscan.io/v2/api", httpClient: httpClient, - limiter: rate.NewLimiter(rate.Limit(callsPerSec), 1), + limiter: limiter, chainId: chainId, } } @@ -460,6 +465,54 @@ func (etherScan *EtherScan) Balance(ctx context.Context, account common.Address) return balance, nil } +// Balances returns the balances for multiple addresses. +func (etherScan *EtherScan) Balances(ctx context.Context, accounts []common.Address) (map[common.Address]*big.Int, error) { + if len(accounts) == 0 { + return nil, nil + } + + params := url.Values{} + params.Set("module", "account") + params.Set("action", "balancemulti") + params.Set("tag", "latest") + + balances := make(map[common.Address]*big.Int) + + type balancesResult struct { + Status string + Message string + Result []struct { + Account string `json:"account"` + Balance jsonBigInt `json:"balance"` + } `json:"result"` + } + + for addressesChunk := range slices.Chunk(accounts, maxAddressesForBalances) { + + addresses := make([]string, len(addressesChunk)) + for i, account := range addressesChunk { + addresses[i] = account.Hex() + } + + params.Set("address", strings.Join(addresses, ",")) + + result := balancesResult{} + if err := etherScan.call(ctx, params, &result); err != nil { + return nil, err + } + if result.Status != "1" { + return nil, errp.New("unexpected response from EtherScan") + } + + for _, item := range result.Result { + account := common.HexToAddress(item.Account) + balance := item.Balance.BigInt() + balances[account] = balance + } + } + return balances, nil +} + // ERC20Balance implements rpc.Interface. func (etherScan *EtherScan) ERC20Balance(account common.Address, erc20Token *erc20.Token) (*big.Int, error) { var result struct {