diff --git a/CHANGELOG.md b/CHANGELOG.md index 8817a7a814..50e87cc194 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Added a banner to remind user to backup their seed phrase when an account reaches a certain threshold. - Gracefully shut down Electrum connections upon closing the app - Show the selected coin's address on the confirmation screen +- Fetch balances of multiple ETH accounts at the same time, instead of one by one. ## v4.48.3 - Linux: fix compatiblity with some versions of Mesa also when using the AppImage diff --git a/backend/accounts_test.go b/backend/accounts_test.go index 72acd9b923..e07d4bc9fa 100644 --- a/backend/accounts_test.go +++ b/backend/accounts_test.go @@ -179,11 +179,13 @@ func TestSortAccounts(t *testing.T) { {Code: "acct-tbtc", CoinCode: coinpkg.CodeTBTC}, } backend := newBackend(t, testnetDisabled, regtestDisabled) + unlockFN := backend.accountsAndKeystoreLock.Lock() for i := range accountConfigs { c, err := backend.Coin(accountConfigs[i].CoinCode) require.NoError(t, err) backend.createAndAddAccount(c, accountConfigs[i]) } + unlockFN() expectedOrder := []accountsTypes.Code{ "acct-btc-1", @@ -747,6 +749,7 @@ func TestCreateAndAddAccount(t *testing.T) { // Add a Bitcoin account. coin, err := b.Coin(coinpkg.CodeBTC) require.NoError(t, err) + unlockFN := b.accountsAndKeystoreLock.Lock() b.createAndAddAccount( coin, &config.Account{ @@ -757,6 +760,7 @@ func TestCreateAndAddAccount(t *testing.T) { }, }, ) + unlockFN() require.Len(t, b.Accounts(), 1) // Check some properties of the newly added account. acct := b.Accounts()[0] @@ -767,6 +771,8 @@ func TestCreateAndAddAccount(t *testing.T) { // Add a Litecoin account. coin, err = b.Coin(coinpkg.CodeLTC) require.NoError(t, err) + + unlockFN = b.accountsAndKeystoreLock.Lock() b.createAndAddAccount(coin, &config.Account{ Code: "test-ltc-account-code", @@ -776,6 +782,7 @@ func TestCreateAndAddAccount(t *testing.T) { }, }, ) + unlockFN() require.Len(t, b.Accounts(), 2) // Check some properties of the newly added account. acct = b.Accounts()[1] @@ -786,6 +793,7 @@ func TestCreateAndAddAccount(t *testing.T) { // Add an Ethereum account with some active ERC20 tokens. coin, err = b.Coin(coinpkg.CodeETH) require.NoError(t, err) + unlockFN = b.accountsAndKeystoreLock.Lock() b.createAndAddAccount(coin, &config.Account{ Code: "test-eth-account-code", @@ -796,6 +804,7 @@ func TestCreateAndAddAccount(t *testing.T) { ActiveTokens: []string{"eth-erc20-mkr"}, }, ) + unlockFN() // 2 more accounts: the added ETH account plus the active token for the ETH account. require.Len(t, b.Accounts(), 4) // Check some properties of the newly added account. @@ -812,6 +821,7 @@ func TestCreateAndAddAccount(t *testing.T) { // Add another Ethereum account with some active ERC20 tokens. coin, err = b.Coin(coinpkg.CodeETH) require.NoError(t, err) + unlockFN = b.accountsAndKeystoreLock.Lock() b.createAndAddAccount(coin, &config.Account{ Code: "test-eth-account-code-2", @@ -823,6 +833,7 @@ func TestCreateAndAddAccount(t *testing.T) { ActiveTokens: []string{"eth-erc20-usdt", "eth-erc20-bat"}, }, ) + unlockFN() // 3 more accounts: the added ETH account plus the two active tokens for the ETH account. require.Len(t, b.Accounts(), 7) // Check some properties of the newly added accounts. diff --git a/backend/backend.go b/backend/backend.go index 4596e28c42..f39589039e 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -61,6 +61,7 @@ import ( "github.com/btcsuite/btcd/chaincfg" "github.com/ethereum/go-ethereum/params" "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) func init() { @@ -225,10 +226,11 @@ type Backend struct { socksProxy socksproxy.SocksProxy // can be a regular or, if Tor is enabled in the config, a SOCKS5 proxy client. - httpClient *http.Client - etherScanHTTPClient *http.Client - ratesUpdater *rates.RateUpdater - banners *banners.Banners + httpClient *http.Client + etherScanHTTPClient *http.Client + etherScanRateLimiter *rate.Limiter + ratesUpdater *rates.RateUpdater + banners *banners.Banners // For unit tests, called when `backend.checkAccountUsed()` is called. tstCheckAccountUsed func(accounts.Interface) bool @@ -240,6 +242,15 @@ type Backend struct { // isOnline indicates whether the backend is online, i.e. able to connect to the internet. isOnline atomic.Bool + + // quit is used to indicate to running goroutines that they should stop as the backend is being closed + quit chan struct{} + + // enqueueUpdateForAccount is used to enqueue an update for an account. + enqueueUpdateForAccount chan *eth.Account + + // updateETHAccountsCh is used to trigger an update of all ETH accounts. + updateETHAccountsCh chan struct{} } // NewBackend creates a new backend with the given arguments. @@ -262,11 +273,13 @@ func NewBackend(arguments *arguments.Arguments, environment Environment) (*Backe return nil, err } + accountUpdate := make(chan *eth.Account) + backend := &Backend{ arguments: arguments, environment: environment, config: backendConfig, - events: make(chan interface{}, 1000), + events: make(chan interface{}), devices: map[string]device.Interface{}, coins: map[coinpkg.Code]coinpkg.Coin{}, @@ -276,12 +289,15 @@ func NewBackend(arguments *arguments.Arguments, environment Environment) (*Backe return btc.NewAccount(config, coin, gapLimits, getAddress, log, hclient) }, makeEthAccount: func(config *accounts.AccountConfig, coin *eth.Coin, httpClient *http.Client, log *logrus.Entry) accounts.Interface { - return eth.NewAccount(config, coin, httpClient, log) + return eth.NewAccount(config, coin, httpClient, log, accountUpdate) }, log: log, - testing: backendConfig.AppConfig().Backend.StartInTestnet || arguments.Testing(), + testing: backendConfig.AppConfig().Backend.StartInTestnet || arguments.Testing(), + quit: make(chan struct{}), + etherScanRateLimiter: rate.NewLimiter(rate.Limit(etherscan.CallsPerSec), 1), + enqueueUpdateForAccount: accountUpdate, } // TODO: remove when connectivity check is present on all platforms backend.isOnline.Store(true) @@ -529,19 +545,19 @@ func (backend *Backend) Coin(code coinpkg.Code) (coinpkg.Coin, error) { coin = btc.NewCoin(coinpkg.CodeLTC, "Litecoin", "LTC", coinpkg.BtcUnitDefault, <c.MainNetParams, dbFolder, servers, "https://blockchair.com/litecoin/transaction/", backend.socksProxy) case code == coinpkg.CodeETH: - etherScan := etherscan.NewEtherScan("1", backend.etherScanHTTPClient) + etherScan := etherscan.NewEtherScan("1", backend.etherScanHTTPClient, backend.etherScanRateLimiter) coin = eth.NewCoin(etherScan, code, "Ethereum", "ETH", "ETH", params.MainnetChainConfig, "https://etherscan.io/tx/", etherScan, nil) case code == coinpkg.CodeSEPETH: - etherScan := etherscan.NewEtherScan("11155111", backend.etherScanHTTPClient) + etherScan := etherscan.NewEtherScan("11155111", backend.etherScanHTTPClient, backend.etherScanRateLimiter) coin = eth.NewCoin(etherScan, code, "Ethereum Sepolia", "SEPETH", "SEPETH", params.SepoliaChainConfig, "https://sepolia.etherscan.io/tx/", etherScan, nil) case erc20Token != nil: - etherScan := etherscan.NewEtherScan("1", backend.etherScanHTTPClient) + etherScan := etherscan.NewEtherScan("1", backend.etherScanHTTPClient, backend.etherScanRateLimiter) coin = eth.NewCoin(etherScan, erc20Token.code, erc20Token.name, erc20Token.unit, "ETH", params.MainnetChainConfig, "https://etherscan.io/tx/", etherScan, @@ -555,6 +571,74 @@ func (backend *Backend) Coin(code coinpkg.Code) (coinpkg.Coin, error) { return coin, nil } +func (backend *Backend) pollETHAccounts() { + timer := time.After(0) + + updateAll := func() { + if err := backend.updateETHAccounts(); err != nil { + backend.log.WithError(err).Error("could not update ETH accounts") + } + } + + for { + select { + case <-backend.quit: + return + default: + select { + case <-backend.quit: + return + case account := <-backend.enqueueUpdateForAccount: + go func() { + // A single ETH accounts needs an update. + ethCoin, ok := account.Coin().(*eth.Coin) + if !ok { + backend.log.WithField("account", account.Config().Config.Name).Errorf("expected ETH account to have ETH coin, got %T", account.Coin()) + } + etherScanClient := etherscan.NewEtherScan(ethCoin.ChainIDstr(), backend.etherScanHTTPClient, backend.etherScanRateLimiter) + if err := eth.UpdateBalances([]*eth.Account{account}, etherScanClient); err != nil { + backend.log.WithError(err).Errorf("could not update account %s", account.Config().Config.Name) + } + }() + case <-backend.updateETHAccountsCh: + go updateAll() + timer = time.After(eth.PollInterval) + case <-timer: + go updateAll() + timer = time.After(eth.PollInterval) + } + } + } +} + +func (backend *Backend) updateETHAccounts() error { + defer backend.accountsAndKeystoreLock.RLock()() + backend.log.Debug("Updating ETH accounts balances") + + accountsChainID := map[string][]*eth.Account{} + for _, account := range backend.accounts { + ethAccount, ok := account.(*eth.Account) + if ok { + ethCoin, ok := ethAccount.Coin().(*eth.Coin) + if !ok { + return errp.Newf("expected ETH account to have ETH coin, got %T", ethAccount.Coin()) + } + chainID := ethCoin.ChainIDstr() + accountsChainID[chainID] = append(accountsChainID[chainID], ethAccount) + } + + } + + for chainID, ethAccounts := range accountsChainID { + etherScanClient := etherscan.NewEtherScan(chainID, backend.etherScanHTTPClient, backend.etherScanRateLimiter) + if err := eth.UpdateBalances(ethAccounts, etherScanClient); err != nil { + backend.log.WithError(err).Errorf("could not update ETH accounts for chain ID %s", chainID) + } + } + + return nil +} + // ManualReconnect triggers reconnecting to Electrum servers if their connection is down. // Only coin connections that were previously established are reconnected. // Calling this is a no-op for coins that are already connected. @@ -592,13 +676,7 @@ func (backend *Backend) ManualReconnect(reconnectETH bool) { } if reconnectETH { backend.log.Info("Reconnecting ETH accounts") - for _, account := range backend.accounts { - ethAccount, ok := account.(*eth.Account) - if !ok { - continue - } - ethAccount.EnqueueUpdate() - } + backend.updateETHAccountsCh <- struct{}{} } } @@ -661,6 +739,8 @@ func (backend *Backend) Start() <-chan interface{} { backend.environment.OnAuthSettingChanged(backend.config.AppConfig().Backend.Authentication) + go backend.pollETHAccounts() + if backend.config.AppConfig().Backend.StartInTestnet { if err := backend.config.ModifyAppConfig(func(c *config.AppConfig) error { c.Backend.StartInTestnet = false; return nil }); err != nil { backend.log.WithError(err).Error("Can't set StartInTestnet to false") @@ -933,6 +1013,8 @@ func (backend *Backend) Close() error { if len(errors) > 0 { return errp.New(strings.Join(errors, "; ")) } + + close(backend.quit) return nil } diff --git a/backend/coins/eth/account.go b/backend/coins/eth/account.go index ab29d9b0a0..27e7ce26ab 100644 --- a/backend/coins/eth/account.go +++ b/backend/coins/eth/account.go @@ -49,7 +49,8 @@ import ( "github.com/sirupsen/logrus" ) -var pollInterval = 5 * time.Minute +// PollInterval is the interval at which the account is polled for updates. +var PollInterval = 5 * time.Minute func isMixedCase(s string) bool { return strings.ToLower(s) != s && strings.ToUpper(s) != s @@ -87,7 +88,7 @@ type Account struct { // enqueueUpdateCh is used to invoke an account update outside of the regular poll update // interval. - enqueueUpdateCh chan struct{} + enqueueUpdateCh chan *Account address Address @@ -100,11 +101,10 @@ type Account struct { // if not nil, SendTx() will sign and send this transaction. Set by TxProposal(). activeTxProposal *TxProposal - // quitChan is used to send a quit signal to the accounts long running routines that - // should listen to it. - quitChan chan struct{} - log *logrus.Entry + + // initDone is called when the account is initialized for the first time + initDone func() } // NewAccount creates a new account. @@ -113,6 +113,7 @@ func NewAccount( accountCoin *Coin, httpClient *http.Client, log *logrus.Entry, + enqueueUpdateCh chan *Account, ) *Account { log = log.WithField("group", "eth"). WithFields(logrus.Fields{"coin": accountCoin.String(), "code": config.Config.Code, "name": config.Config.Name}) @@ -126,8 +127,7 @@ func NewAccount( httpClient: httpClient, balance: coin.NewAmountFromInt64(0), - enqueueUpdateCh: make(chan struct{}), - quitChan: make(chan struct{}), + enqueueUpdateCh: enqueueUpdateCh, log: log, } @@ -207,41 +207,12 @@ func (account *Account) Initialize() error { ) account.coin.Initialize() - done := account.Synchronizer.IncRequestsCounter() - go account.poll(done) + account.initDone = account.Synchronizer.IncRequestsCounter() + go account.EnqueueUpdate() return account.BaseAccount.Initialize(accountIdentifier) } -func (account *Account) poll(initDone func()) { - timer := time.After(0) - for { - select { - case <-account.quitChan: - return - default: - select { - case <-account.quitChan: - return - case <-timer: - case <-account.enqueueUpdateCh: - account.log.Info("extraordinary account update invoked") - } - if err := account.update(); err != nil { - account.log.WithError(err).Error("error updating account") - account.SetOffline(err) - } else { - account.SetOffline(nil) - } - if initDone != nil { - initDone() - initDone = nil - } - timer = time.After(pollInterval) - } - } -} - // updateOutgoingTransactions updates the height of the stored outgoing transactions. // We update heights for tx with up to 12 confirmations, so re-orgs are taken into account. // tipHeight is the current blockchain height. @@ -375,7 +346,9 @@ func (account *Account) nextNonce() (uint64, error) { return nextNonce, nil } -func (account *Account) update() error { +// Update performs an Update of the account's transaction +// and its balance, which must be provided as an argument. +func (account *Account) Update(balance *big.Int) error { defer account.updateLock.Lock()() defer account.Synchronizer.IncRequestsCounter()() @@ -416,22 +389,14 @@ func (account *Account) update() error { } } - var balance *big.Int - if account.coin.erc20Token != nil { - balance, err = account.coin.client.ERC20Balance(account.address.Address, account.coin.erc20Token) - if err != nil { - return errp.WithStack(err) - } - } else { - balance, err = account.coin.client.Balance(context.TODO(), account.address.Address) - if err != nil { - return errp.WithStack(err) - } - } - pendingAmount := pendingTxsAmount(outgoingTransactionsData, account.coin.erc20Token != nil) account.balance = coin.NewAmount(balance.Sub(balance, pendingAmount)) + if account.initDone != nil { + account.initDone() + account.initDone = nil + } + return nil } @@ -472,7 +437,6 @@ func (account *Account) Close() { } account.log.Info("Closed DB") } - close(account.quitChan) account.closed = true account.Notify(observable.Event{ Subject: string(accountsTypes.EventStatusChanged), @@ -1052,8 +1016,67 @@ func (account *Account) MatchesAddress(address string) (bool, error) { // EnqueueUpdate enqueues an update for the account. func (account *Account) EnqueueUpdate() { - select { - case account.enqueueUpdateCh <- struct{}{}: - default: + account.enqueueUpdateCh <- account +} + +// UpdateBalances updates the balances of the accounts in the provided slice. +func UpdateBalances(accounts []*Account, etherScanClient *etherscan.EtherScan) error { + ethNonErc20Addresses := make([]ethcommon.Address, 0, len(accounts)) + for _, account := range accounts { + if account.isClosed() { + continue + } + address, err := account.Address() + if err != nil { + account.log.WithError(err).Errorf("Could not get address for account %s", account.Config().Config.Code) + account.SetOffline(err) + continue + } + if account.coin.erc20Token == nil { + ethNonErc20Addresses = append(ethNonErc20Addresses, address.Address) + } } + + balances, err := etherScanClient.Balances(context.TODO(), ethNonErc20Addresses) + if err != nil { + return errp.WithStack(err) + } + + for _, account := range accounts { + if account.isClosed() { + continue + } + address, err := account.Address() + if err != nil { + account.log.WithError(err).Errorf("Could not get address for account %s", account.Config().Config.Code) + account.SetOffline(err) + continue + } + var balance *big.Int + if account.coin.erc20Token != nil { + var err error + balance, err = account.coin.client.ERC20Balance(account.address.Address, account.coin.erc20Token) + if err != nil { + account.log.WithError(err).Errorf("Could not get ERC20 balance for address %s", address.Address.Hex()) + account.SetOffline(err) + continue + } + } else { + var ok bool + balance, ok = balances[address.Address] + if !ok { + errMsg := fmt.Sprintf("Could not find balance for address %s", address.Address.Hex()) + account.log.Error(errMsg) + account.SetOffline(errp.New(errMsg)) + continue + } + } + if err := account.Update(balance); err != nil { + account.log.WithError(err).Errorf("Could not update balance for address %s", address.Address.Hex()) + account.SetOffline(err) + } else { + account.SetOffline(nil) + } + } + return nil } diff --git a/backend/coins/eth/account_test.go b/backend/coins/eth/account_test.go index 471459c684..a0aa8292aa 100644 --- a/backend/coins/eth/account_test.go +++ b/backend/coins/eth/account_test.go @@ -106,6 +106,7 @@ func newAccount(t *testing.T) *Account { coin, &http.Client{}, log, + make(chan *Account), ) require.NoError(t, acct.Initialize()) return acct @@ -114,6 +115,7 @@ func newAccount(t *testing.T) *Account { func TestTxProposal(t *testing.T) { acct := newAccount(t) defer acct.Close() + require.NoError(t, acct.Update(big.NewInt(1e18))) require.Eventually(t, acct.Synced, time.Second, time.Millisecond*200) t.Run("valid", func(t *testing.T) { @@ -171,6 +173,7 @@ func TestTxProposal(t *testing.T) { func TestMatchesAddress(t *testing.T) { acct := newAccount(t) defer acct.Close() + require.NoError(t, acct.Update(big.NewInt(1e18))) require.Eventually(t, acct.Synced, time.Second, time.Millisecond*200) // Test invalid Ethereum address diff --git a/backend/coins/eth/coin.go b/backend/coins/eth/coin.go index 6b76f7bdaa..ad4162ffb1 100644 --- a/backend/coins/eth/coin.go +++ b/backend/coins/eth/coin.go @@ -107,6 +107,11 @@ func (coin *Coin) Net() *params.ChainConfig { return coin.net } // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-155.md#list-of-chain-ids func (coin *Coin) ChainID() uint64 { return coin.net.ChainID.Uint64() } +// ChainIDstr returns the chain ID of the network as a string. +func (coin *Coin) ChainIDstr() string { + return coin.net.ChainID.String() +} + // Initialize implements coin.Coin. func (coin *Coin) Initialize() {} diff --git a/backend/coins/eth/etherscan/etherscan.go b/backend/coins/eth/etherscan/etherscan.go index 3dffed71a9..6e67d4cbc8 100644 --- a/backend/coins/eth/etherscan/etherscan.go +++ b/backend/coins/eth/etherscan/etherscan.go @@ -23,7 +23,9 @@ import ( "math/big" "net/http" "net/url" + "slices" "strconv" + "strings" "time" "github.com/BitBoxSwiss/bitbox-wallet-app/backend/accounts" @@ -39,12 +41,15 @@ import ( "golang.org/x/time/rate" ) -// callsPerSec is thenumber of etherscanr equests allowed +// CallsPerSec is thenumber of etherscanr equests allowed // per second. // Etherscan rate limits to one request per 0.2 seconds. -var callsPerSec = 3.8 +var CallsPerSec = 3.8 -const apiKey = "X3AFAGQT2QCAFTFPIH9VJY88H9PIQ2UWP7" +const ( + apiKey = "X3AFAGQT2QCAFTFPIH9VJY88H9PIQ2UWP7" + maxAddressesForBalances = 20 +) // ERC20GasErr is the error message returned from etherscan when there is not enough ETH to pay the transaction fee. const ERC20GasErr = "insufficient funds for gas * price + value" @@ -58,11 +63,11 @@ type EtherScan struct { } // NewEtherScan creates a new instance of EtherScan. -func NewEtherScan(chainId string, httpClient *http.Client) *EtherScan { +func NewEtherScan(chainId string, httpClient *http.Client, limiter *rate.Limiter) *EtherScan { return &EtherScan{ url: "https://api.etherscan.io/v2/api", httpClient: httpClient, - limiter: rate.NewLimiter(rate.Limit(callsPerSec), 1), + limiter: limiter, chainId: chainId, } } @@ -460,6 +465,54 @@ func (etherScan *EtherScan) Balance(ctx context.Context, account common.Address) return balance, nil } +// Balances returns the balances for multiple addresses. +func (etherScan *EtherScan) Balances(ctx context.Context, accounts []common.Address) (map[common.Address]*big.Int, error) { + if len(accounts) == 0 { + return nil, nil + } + + params := url.Values{} + params.Set("module", "account") + params.Set("action", "balancemulti") + params.Set("tag", "latest") + + balances := make(map[common.Address]*big.Int) + + type balancesResult struct { + Status string + Message string + Result []struct { + Account string `json:"account"` + Balance jsonBigInt `json:"balance"` + } `json:"result"` + } + + for addressesChunk := range slices.Chunk(accounts, maxAddressesForBalances) { + + addresses := make([]string, len(addressesChunk)) + for i, account := range addressesChunk { + addresses[i] = account.Hex() + } + + params.Set("address", strings.Join(addresses, ",")) + + result := balancesResult{} + if err := etherScan.call(ctx, params, &result); err != nil { + return nil, err + } + if result.Status != "1" { + return nil, errp.New("unexpected response from EtherScan") + } + + for _, item := range result.Result { + account := common.HexToAddress(item.Account) + balance := item.Balance.BigInt() + balances[account] = balance + } + } + return balances, nil +} + // ERC20Balance implements rpc.Interface. func (etherScan *EtherScan) ERC20Balance(account common.Address, erc20Token *erc20.Token) (*big.Int, error) { var result struct {