From 9d346bc6d544db69784fd0734d963fc02b67eb5a Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Thu, 25 Jun 2026 14:38:46 +0200 Subject: [PATCH] [Trading] handle lazy markets loading exchanges trading --- .../api/core.py | 35 ++++- .../tests/api/test_core.py | 89 +++++++++-- .../connectors/ccxt/ccxt_connector.py | 44 +++++- .../exchanges/types/rest_exchange.py | 15 ++ .../octobot_trading/modes/modes_util.py | 2 +- .../script_keywords/basic_keywords/price.py | 2 +- .../personal_data/orders/order_factory.py | 2 +- .../personal_data/orders/order_util.py | 2 +- .../orders/types/limit/limit_order.py | 6 +- packages/trading/requirements.txt | 2 +- .../connectors/ccxt/test_ccxt_connector.py | 146 ++++++++++++++++++ .../test_default_rest_exchange.py | 69 +++++++++ .../basic_keywords/test_price.py | 4 +- .../orders/test_order_factory.py | 2 +- .../real_exchanges/real_exchange_tester.py | 30 +++- 15 files changed, 412 insertions(+), 38 deletions(-) diff --git a/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/api/core.py b/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/api/core.py index 23dea5364d..f2deea9c33 100644 --- a/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/api/core.py +++ b/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/api/core.py @@ -107,11 +107,25 @@ async def _fill_market_making_data_by_symbol( elif dependency.symbol not in dependency_symbol_alias_by_symbol: dependency_symbol_alias_by_symbol[dependency.symbol] = None available_symbols = set(exchange_manager.exchange.get_all_available_symbols(active_only=True)) - symbols_to_skip_ticker_fetch = { - source.pair - for source in price_sources - if source.formula and source.pair not in available_symbols - } + lazy_load_markets = exchange_manager.exchange.get_option_value( + octobot_trading.enums.ExchangeClientOptions.LAZY_LOAD_MARKETS + ) + if lazy_load_markets: + symbols_to_skip_ticker_fetch = { + source.pair + for source in price_sources + if source.formula and source.pair not in available_symbols + } + else: + symbols_to_skip_ticker_fetch = { + symbol for symbol in (set(symbols) | set(dependency_symbol_alias_by_symbol.keys())) + if symbol not in available_symbols + } + if symbols_to_skip_ticker_fetch: + _get_logger().info( + f"Ignored unavailable pairs on [{exchange_internal_name}]: " + f"{sorted(symbols_to_skip_ticker_fetch)}" + ) symbols_to_fetch = (set(symbols) | set(dependency_symbol_alias_by_symbol.keys())) - symbols_to_skip_ticker_fetch tickers = { symbol: ticker @@ -121,7 +135,11 @@ async def _fill_market_making_data_by_symbol( tickers, ticker_updater = await _fetch_tickers( exchange_manager, tickers, list(symbols_to_fetch) ) - if missing_tickers_to_fetch := [symbol for symbol in symbols_to_fetch if symbol not in tickers]: + if missing_tickers_to_fetch := [ + symbol for symbol in symbols_to_fetch + if symbol not in tickers + and (lazy_load_markets or symbol in available_symbols) + ]: try: tickers.update(await ticker_updater.fetch_all_tickers(missing_tickers_to_fetch)) except octobot_trading.errors.NotSupported as err: @@ -694,6 +712,7 @@ async def _get_price_and_predicted_order_book( books_by_symbol = {} for pair, reference_price in reference_price_by_pair.items(): if not reference_price or reference_price.is_nan(): + error_by_pair[pair] = _get_unsupported_pair_message(pair, mm_exchange) continue mm_data = mm_data_by_exchange[mm_exchange].get(pair) _adapt_volume_if_necessary(mm_data, reference_price) @@ -742,6 +761,10 @@ def _get_missing_symbol_message(symbol: str, exchange: str) -> str: ) +def _get_unsupported_pair_message(pair: str, exchange: str) -> str: + return f"{pair} is not supported on {exchange}" + + def _format_format_market_making_volume(volume: typing.Union[dict, None], error: typing.Union[str, None]): return { constants.VOLUME_KEY: volume, diff --git a/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/tests/api/test_core.py b/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/tests/api/test_core.py index a26e2fbde2..22650e8dba 100644 --- a/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/tests/api/test_core.py +++ b/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/tests/api/test_core.py @@ -589,7 +589,7 @@ async def test_get_price_and_predicted_order_book_with_invalid_formula(profile_d async def test_get_price_and_predicted_order_book_with_error(profile_data_with_full_mm_config, mm_data_by_exchange): - """Test _get_price_and_predicted_order_book with error handling.""" + """Empty reference price sources yield an unsupported-pair error.""" profile_data_with_full_mm_config.tentacles[0].config[ simple_market_making_trading.SimpleMarketMakingTradingMode.CONFIG_PAIR_SETTINGS ][0][simple_market_making_trading.SimpleMarketMakingTradingMode.REFERENCE_PRICE] = [] @@ -602,7 +602,9 @@ async def test_get_price_and_predicted_order_book_with_error(profile_data_with_f assert result == { "BTC/USDT": { - market_making_constants.ERROR_KEY: "BTC/USDT reference price on binance can't be computed from the following price sources: {}" + market_making_constants.ERROR_KEY: market_making_core._get_unsupported_pair_message( + "BTC/USDT", "binance" + ), } } @@ -1059,10 +1061,29 @@ async def _mock_exchange_manager_context(exchange_manager): yield exchange_manager +def _configure_lazy_load_markets(exchange_manager, lazy_load_markets: bool): + exchange_manager.exchange.get_option_value = mock.Mock( + side_effect=lambda option, **kwargs: ( + lazy_load_markets + if option == trading_enums.ExchangeClientOptions.LAZY_LOAD_MARKETS + else False + ) + ) + + +def _get_symbols_passed_to_fetch_tickers(fetch_tickers_mock) -> set: + return { + symbol + for call_args in fetch_tickers_mock.call_args_list + for symbol in call_args.args[2] + } + + async def _call_fill_market_making_data_by_symbol( price_sources, available_symbols, formula_init_patches=None, + lazy_load_markets=True, ): profile_data = _profile_data_for_market_making_fill() mm_data_by_symbol_by_exchange = {} @@ -1070,10 +1091,12 @@ async def _call_fill_market_making_data_by_symbol( exchange_manager.exchange.get_all_available_symbols = mock.Mock( return_value=available_symbols ) + _configure_lazy_load_markets(exchange_manager, lazy_load_markets) ticker_updater = mock.Mock() ticker_updater.fetch_all_tickers = mock.AsyncMock(return_value={}) ticker_cache = mock.Mock() ticker_cache.get_all_tickers = mock.Mock(return_value={}) + fetch_tickers_mock = mock.AsyncMock(return_value=({}, ticker_updater)) patches = [ mock.patch.object( @@ -1099,7 +1122,7 @@ async def _call_fill_market_making_data_by_symbol( mock.patch.object( market_making_core, "_fetch_tickers", - mock.AsyncMock(return_value=({}, ticker_updater)), + fetch_tickers_mock, ), ] if formula_init_patches: @@ -1118,7 +1141,10 @@ async def _call_fill_market_making_data_by_symbol( auth=None, ) - return mm_data_by_symbol_by_exchange, ticker_updater.fetch_all_tickers + return mm_data_by_symbol_by_exchange, { + "fetch_tickers": fetch_tickers_mock, + "fetch_all_tickers": ticker_updater.fetch_all_tickers, + } class TestFillMarketMakingDataBySymbol: @@ -1141,12 +1167,13 @@ async def test_skips_ticker_fetch_for_unsupported_ref_price_symbol_with_formula( mock.patch.object(exchange_operators, "create_ohlcv_operators", return_value=[]), mock.patch.object(exchange_operators, "create_price_operators", return_value=[]), ] - mm_data_by_symbol_by_exchange, fetch_all_tickers_mock = await _call_fill_market_making_data_by_symbol( + mm_data_by_symbol_by_exchange, fetch_mocks = await _call_fill_market_making_data_by_symbol( price_sources, available_symbols={"BTC/ETH", "ETH/USDT"}, formula_init_patches=formula_init_patches, ) + fetch_all_tickers_mock = fetch_mocks["fetch_all_tickers"] assert fetch_all_tickers_mock.call_count == 0 or all( "BTC/USDT" not in call_args.args[0] for call_args in fetch_all_tickers_mock.call_args_list @@ -1189,12 +1216,13 @@ async def test_still_fetches_formula_dependency_symbols(self): return_value=exchange_operators.create_price_operators(exchange_manager, "BTC/USDT"), ), ] - mm_data_by_symbol_by_exchange, fetch_all_tickers_mock = await _call_fill_market_making_data_by_symbol( + mm_data_by_symbol_by_exchange, fetch_mocks = await _call_fill_market_making_data_by_symbol( price_sources, available_symbols={"BTC/ETH", "ETH/USDT"}, formula_init_patches=formula_init_patches, ) + fetch_all_tickers_mock = fetch_mocks["fetch_all_tickers"] assert fetch_all_tickers_mock.call_count == 1 fetched_symbols = fetch_all_tickers_mock.call_args.args[0] assert "BTC/USDT" not in fetched_symbols @@ -1203,6 +1231,7 @@ async def test_still_fetches_formula_dependency_symbols(self): assert set(mm_data_by_symbol_by_exchange["binance"]) == {"BTC/USDT", "BTC/ETH", "ETH/USDT"} async def test_fetches_ticker_for_unsupported_symbol_without_formula(self): + """Lazy-load exchanges still fetch direct pairs not listed in available_symbols.""" price_sources = [ advanced_reference_price_import.AdvancedPriceSource( exchange="binance", @@ -1212,13 +1241,55 @@ async def test_fetches_ticker_for_unsupported_symbol_without_formula(self): formula="", ) ] - mm_data_by_symbol_by_exchange, fetch_all_tickers_mock = await _call_fill_market_making_data_by_symbol( + mm_data_by_symbol_by_exchange, fetch_mocks = await _call_fill_market_making_data_by_symbol( price_sources, available_symbols={"BTC/ETH", "ETH/USDT"}, + lazy_load_markets=True, ) - assert fetch_all_tickers_mock.call_count == 1 - assert fetch_all_tickers_mock.call_args.args[0] == ["BTC/USDT"] + fetch_tickers_mock = fetch_mocks["fetch_tickers"] + assert fetch_tickers_mock.call_count == 1 + assert fetch_tickers_mock.call_args.args[2] == ["BTC/USDT"] + + async def test_skips_unavailable_direct_pair_on_cex(self): + price_sources = [ + advanced_reference_price_import.AdvancedPriceSource( + exchange="binance", + pair="BTC/USDT", + time_frame=advanced_reference_price_import.DEFAULT_TIME_FRAME, + weight=decimal.Decimal("1.0"), + formula="", + ) + ] + mm_data_by_symbol_by_exchange, fetch_mocks = await _call_fill_market_making_data_by_symbol( + price_sources, + available_symbols={"BTC/ETH", "ETH/USDT"}, + lazy_load_markets=False, + ) + + assert "BTC/USDT" not in _get_symbols_passed_to_fetch_tickers(fetch_mocks["fetch_tickers"]) + btc_usdt_data = mm_data_by_symbol_by_exchange["binance"]["BTC/USDT"] + assert btc_usdt_data.price.is_nan() + + async def test_still_fetches_unavailable_direct_pair_on_lazy_load_exchange(self): + price_sources = [ + advanced_reference_price_import.AdvancedPriceSource( + exchange="binance", + pair="BTC/USDT", + time_frame=advanced_reference_price_import.DEFAULT_TIME_FRAME, + weight=decimal.Decimal("1.0"), + formula="", + ) + ] + _, fetch_mocks = await _call_fill_market_making_data_by_symbol( + price_sources, + available_symbols={"BTC/ETH", "ETH/USDT"}, + lazy_load_markets=True, + ) + + fetch_tickers_mock = fetch_mocks["fetch_tickers"] + assert fetch_tickers_mock.call_count == 1 + assert fetch_tickers_mock.call_args.args[2] == ["BTC/USDT"] def _mock_create_price_operators(prices_by_symbol: dict[str, float]): diff --git a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py index 7f5727323d..d2beb874c2 100644 --- a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py +++ b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py @@ -170,10 +170,34 @@ def _ensure_successful_markets_fetch(self, client): f"No spot markets found for {self.exchange_manager.exchange_name}: {len(symbols)} fetched markets: {logged_symbols}" ) + def _persist_markets_cache( + self, + client=None, + authenticated_cache: typing.Optional[bool] = None, + ): + if client is None: + client = self.client + if authenticated_cache is None: + authenticated_cache = self.exchange_manager.exchange.requires_authentication_for_this_configuration_only() + ccxt_client_util.set_ccxt_client_cache(client, authenticated_cache) + + def _persist_markets_cache_if_new_symbols( + self, + previous_market_symbols: set[str], + client=None, + ): + if client is None: + client = self.client + current_market_symbols = set((client.markets or {}).keys()) + if current_market_symbols - previous_market_symbols: + self._persist_markets_cache(client) + async def load_markets_for_symbols(self, symbols: list[str]) -> list[dict]: if not self.client.has.get('obLoadMarketsForSymbols'): raise octobot_trading.errors.NotSupported("This exchange doesn't support lazyLoadMarkets") - return await self.client.ob_load_markets_for_symbols(symbols) + loaded_markets = await self.client.ob_load_markets_for_symbols(symbols) + self._persist_markets_cache() + return loaded_markets async def _filtered_if_necessary_load_markets( self, @@ -256,7 +280,7 @@ async def load_symbol_markets( ) try: await self._load_markets(self.client, reload, market_filter=market_filter) - ccxt_client_util.set_ccxt_client_cache(self.client, authenticated_cache) + self._persist_markets_cache() except ccxt.async_support.OBIPWhitelistError as err: raise octobot_trading.errors.InvalidAPIKeyIPWhitelistError( f"Invalid IP whitelist error: {html_util.get_html_summary_if_relevant(err)}" @@ -299,7 +323,7 @@ async def load_symbol_markets( try: unauth_client = self._client_factory(True)[0] await self._load_markets(unauth_client, reload, market_filter=market_filter) - ccxt_client_util.set_ccxt_client_cache(unauth_client, False) + self._persist_markets_cache(unauth_client, False) # apply markets to target client ccxt_client_util.load_markets_from_cache(self.client, False, market_filter=market_filter) self.logger.debug( @@ -648,15 +672,20 @@ async def get_recent_trades(self, symbol: str, limit: int = 50, **kwargs: dict) async def get_price_ticker(self, symbol: str, **kwargs: dict) -> typing.Optional[dict]: try: with self.error_describer(False): - return self.adapter.adapt_ticker( + previous_market_symbols = set((self.client.markets or {}).keys()) + ticker = self.adapter.adapt_ticker( await self.client.fetch_ticker(symbol, params=kwargs) ) + self._persist_markets_cache_if_new_symbols(previous_market_symbols) + return ticker + except ccxt.async_support.BadSymbol as err: + raise octobot_trading.errors.UnSupportedSymbolError(str(err)) from err except ccxt.async_support.NotSupported: raise octobot_trading.errors.NotSupported except ccxt.async_support.BaseError as e: raise octobot_trading.errors.FailedRequest( f"Failed to get_price_ticker {html_util.get_html_summary_if_relevant(e)}" - ) + ) from e @ccxt_client_util.converted_ccxt_common_errors async def get_all_currencies_price_ticker( @@ -664,9 +693,12 @@ async def get_all_currencies_price_ticker( ) -> typing.Optional[dict[str, dict]]: try: with self.error_describer(False): + previous_market_symbols = set((self.client.markets or {}).keys()) + fetched_tickers = await self.client.fetch_tickers(symbols, params=kwargs) + self._persist_markets_cache_if_new_symbols(previous_market_symbols) tickers = { symbol: self.adapter.adapt_ticker(ticker) - for symbol, ticker in (await self.client.fetch_tickers(symbols, params=kwargs)).items() + for symbol, ticker in fetched_tickers.items() } # self.all_currencies_price_ticker should always contain as many tickers as possible: don't override it # with less symbols when fetching only a few tickers diff --git a/packages/trading/octobot_trading/exchanges/types/rest_exchange.py b/packages/trading/octobot_trading/exchanges/types/rest_exchange.py index dcfd766a6e..4c85dd0971 100644 --- a/packages/trading/octobot_trading/exchanges/types/rest_exchange.py +++ b/packages/trading/octobot_trading/exchanges/types/rest_exchange.py @@ -582,6 +582,21 @@ async def load_markets_for_symbols(self, symbols: list[str]) -> list[dict]: ) return loaded_markets + def _is_lazy_market_loaded(self, symbol: str) -> bool: + client_markets = self.connector.client.markets or {} + return symbol in client_markets + + async def ensure_lazy_market_loaded(self, symbol: str) -> None: + if not self.lazy_load_markets(): + return + if self._is_lazy_market_loaded(symbol): + return + await self.load_markets_for_symbols([symbol]) + + async def get_market_status_including_lazy_load(self, symbol, price_example=None, with_fixer=True): + await self.ensure_lazy_market_loaded(symbol) + return self.get_market_status(symbol, price_example=price_example, with_fixer=with_fixer) + def get_market_status(self, symbol, price_example=None, with_fixer=True): """ Override using get_fixed_market_status in exchange tentacle if the default market status is not as expected diff --git a/packages/trading/octobot_trading/modes/modes_util.py b/packages/trading/octobot_trading/modes/modes_util.py index 97df0347e9..2adf2782ac 100644 --- a/packages/trading/octobot_trading/modes/modes_util.py +++ b/packages/trading/octobot_trading/modes/modes_util.py @@ -195,7 +195,7 @@ async def convert_with_market_or_limit_order( # get order quantity quantity = _get_available_or_target_quantity(exchange_mgr, symbol, order_type, price, asset_amount) - symbol_market = exchange_mgr.exchange.get_market_status(symbol, with_fixer=False) + symbol_market = await exchange_mgr.exchange.get_market_status_including_lazy_load(symbol, with_fixer=False) created_orders = [] for order_quantity, order_price in \ trading_personal_data.decimal_check_and_adapt_order_details_if_necessary( diff --git a/packages/trading/octobot_trading/modes/script_keywords/basic_keywords/price.py b/packages/trading/octobot_trading/modes/script_keywords/basic_keywords/price.py index 83b60e7b22..af792649e3 100644 --- a/packages/trading/octobot_trading/modes/script_keywords/basic_keywords/price.py +++ b/packages/trading/octobot_trading/modes/script_keywords/basic_keywords/price.py @@ -63,5 +63,5 @@ async def get_price_with_offset(context, offset_input, use_delta_type_as_flat_va f"1.2, -0.222, -0.222d, @65100, 5%, e5%, e500" ) - symbol_market = context.exchange_manager.exchange.get_market_status(context.symbol, with_fixer=False) + symbol_market = await context.exchange_manager.exchange.get_market_status_including_lazy_load(context.symbol, with_fixer=False) return personal_data.decimal_adapt_price(symbol_market, computed_price) diff --git a/packages/trading/octobot_trading/personal_data/orders/order_factory.py b/packages/trading/octobot_trading/personal_data/orders/order_factory.py index 2dd6e51ca9..056db0bf7b 100644 --- a/packages/trading/octobot_trading/personal_data/orders/order_factory.py +++ b/packages/trading/octobot_trading/personal_data/orders/order_factory.py @@ -473,7 +473,7 @@ async def create_base_orders_and_associated_elements( current_price = await personal_data.get_up_to_date_price( self.exchange_manager, symbol=symbol, timeout=constants.ORDER_DATA_FETCHING_TIMEOUT ) - symbol_market = self.exchange_manager.exchange.get_market_status(symbol, with_fixer=False) + symbol_market = await self.exchange_manager.exchange.get_market_status_including_lazy_load(symbol, with_fixer=False) ctx = script_keywords.get_base_context_from_exchange_manager(self.exchange_manager, symbol) # market orders have no price computed_price = current_price if price is None else await self._get_computed_price(ctx, price) diff --git a/packages/trading/octobot_trading/personal_data/orders/order_util.py b/packages/trading/octobot_trading/personal_data/orders/order_util.py index 3d8204bff7..5fffc1de2c 100644 --- a/packages/trading/octobot_trading/personal_data/orders/order_util.py +++ b/packages/trading/octobot_trading/personal_data/orders/order_util.py @@ -272,7 +272,7 @@ async def get_pre_order_data(exchange_manager, symbol: str, timeout: int = None, portfolio_type=commons_constants.PORTFOLIO_AVAILABLE, target_price=None): price = target_price or await get_up_to_date_price(exchange_manager, symbol, timeout=timeout) - symbol_market = exchange_manager.exchange.get_market_status(symbol, with_fixer=False) + symbol_market = await exchange_manager.exchange.get_market_status_including_lazy_load(symbol, with_fixer=False) currency_available, market_available, market_quantity = get_portfolio_amounts( exchange_manager, symbol, price, portfolio_type=portfolio_type ) diff --git a/packages/trading/octobot_trading/personal_data/orders/types/limit/limit_order.py b/packages/trading/octobot_trading/personal_data/orders/types/limit/limit_order.py index 0d50268bcb..0c5349bf41 100644 --- a/packages/trading/octobot_trading/personal_data/orders/types/limit/limit_order.py +++ b/packages/trading/octobot_trading/personal_data/orders/types/limit/limit_order.py @@ -34,12 +34,12 @@ async def update_price_if_outdated(self): try: current_price = await self.exchange_manager.exchange_symbols_data.get_exchange_symbol_data(self.symbol) \ .prices_manager.get_mark_price(timeout=constants.CHAINED_ORDER_PRICE_FETCHING_TIMEOUT) - self._update_limit_price_if_necessary(current_price) + await self._update_limit_price_if_necessary(current_price) except asyncio.TimeoutError: # price can't be checked return - def _update_limit_price_if_necessary(self, current_price): + async def _update_limit_price_if_necessary(self, current_price): updated_price = self.origin_price if self.side is enums.TradeOrderSide.BUY: highest_accepted_buy_price = ( @@ -58,7 +58,7 @@ def _update_limit_price_if_necessary(self, current_price): # => Increase it to the current price updated_price = lowest_accepted_sell_price if self.origin_price != updated_price: - symbol_market = self.exchange_manager.exchange.get_market_status(self.symbol, with_fixer=False) + symbol_market = await self.exchange_manager.exchange.get_market_status_including_lazy_load(self.symbol, with_fixer=False) self.origin_price = decimal_order_adapter.decimal_adapt_price(symbol_market, updated_price) async def update_order_status(self, force_refresh=False): diff --git a/packages/trading/requirements.txt b/packages/trading/requirements.txt index 033f729556..e8941f609e 100644 --- a/packages/trading/requirements.txt +++ b/packages/trading/requirements.txt @@ -1,7 +1,7 @@ numpy==2.4.2 # Exchange connection requirements -octobot-ccxt==0.0.2 # always ensure real exchanges tests (in tests_additional and authenticated exchange tests) are passing before changing the ccxt version +octobot-ccxt==0.0.3 # always ensure real exchanges tests (in tests_additional and authenticated exchange tests) are passing before changing the ccxt version cryptography # Never specify a version (managed by https://github.com/Drakkar-Software/OctoBot-PyPi-Linux-Deployer) diff --git a/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_connector.py b/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_connector.py index f7af200979..6b6dceded7 100644 --- a/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_connector.py +++ b/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_connector.py @@ -30,6 +30,7 @@ import octobot_trading.constants import octobot_trading.exchange_data.contracts as contracts import octobot_trading.exchanges.connectors.ccxt.ccxt_client_util as ccxt_client_util +import octobot_trading.exchanges.connectors.ccxt.ccxt_clients_cache as ccxt_clients_cache import pytest import octobot_trading.exchanges.connectors.ccxt.constants as ccxt_constants @@ -437,6 +438,151 @@ async def test_error_describer(ccxt_connector): set_first_consecutive_authentication_error_at_if_unset_mock.reset_mock() +class _LazyLoadMarketsMockCCXT: + def __init__(self, populate_markets_on_load: bool = True): + self.markets = {} + self.has = {'obLoadMarketsForSymbols': True} + self.urls = {'api': {'public': 'https://test.example'}} + self.apiKey = None + self.options = {} + self.name = 'test' + self.populate_markets_on_load = populate_markets_on_load + + async def ob_load_markets_for_symbols(self, symbols, reload=False, params=None): + if self.populate_markets_on_load: + for symbol in symbols: + self.markets[symbol] = {'symbol': symbol} + return [] + + +class TestCcxtConnectorLoadMarketsForSymbols: + + async def test_persists_markets_cache_after_lazy_symbol_load(self, ccxt_connector): + symbol = "BTC/ETH" + with ( + mock.patch.object(ccxt_connector, 'client', new=_LazyLoadMarketsMockCCXT()), + mock.patch.object(ccxt_connector, '_persist_markets_cache') as persist_markets_cache_mock, + ): + await ccxt_connector.load_markets_for_symbols([symbol]) + + persist_markets_cache_mock.assert_called_once() + + async def test_skips_cache_persist_when_lazy_load_leaves_markets_empty(self, ccxt_connector): + lazy_client = _LazyLoadMarketsMockCCXT(populate_markets_on_load=False) + with ( + ccxt_clients_cache.isolated_empty_cache(), + mock.patch.object(octobot_trading.constants, "USE_CCXT_SHARED_MARKETS_CACHE", False), + mock.patch.object(ccxt_connector, 'client', new=lazy_client), + ): + await ccxt_connector.load_markets_for_symbols(["BTC/ETH"]) + client_key = ccxt_clients_cache.get_client_key(lazy_client, False) + with pytest.raises(KeyError): + ccxt_clients_cache.get_exchange_parsed_markets(client_key) + + async def test_accumulates_markets_in_cache_across_lazy_loads(self, ccxt_connector): + symbol_a = "BTC/ETH" + symbol_b = "ETH/USDT" + lazy_client = _LazyLoadMarketsMockCCXT() + with ( + ccxt_clients_cache.isolated_empty_cache(), + mock.patch.object(octobot_trading.constants, "USE_CCXT_SHARED_MARKETS_CACHE", False), + mock.patch.object(ccxt_connector, 'client', new=lazy_client), + ): + await ccxt_connector.load_markets_for_symbols([symbol_a]) + await ccxt_connector.load_markets_for_symbols([symbol_b]) + client_key = ccxt_clients_cache.get_client_key(lazy_client, False) + cached_markets = ccxt_clients_cache.get_exchange_parsed_markets(client_key) + cached_symbols = {market['symbol'] for market in cached_markets} + assert cached_symbols == {symbol_a, symbol_b} + + +class _TickerMarketsMockCCXT: + def __init__(self, symbols_to_add_on_fetch: list[str] | None = None): + self.markets = {} + self.urls = {'api': {'public': 'https://test.example'}} + self.apiKey = None + self.symbols_to_add_on_fetch = symbols_to_add_on_fetch or [] + + async def fetch_ticker(self, symbol, params=None): + for symbol_to_add in self.symbols_to_add_on_fetch: + self.markets[symbol_to_add] = {'symbol': symbol_to_add} + return {'symbol': symbol} + + async def fetch_tickers(self, symbols, params=None): + for symbol_to_add in self.symbols_to_add_on_fetch: + self.markets[symbol_to_add] = {'symbol': symbol_to_add} + return { + symbol_to_add: {'symbol': symbol_to_add} + for symbol_to_add in self.symbols_to_add_on_fetch + } + + +class TestCcxtConnectorGetPriceTicker: + + async def test_persists_markets_cache_when_ticker_adds_symbol(self, ccxt_connector): + symbol = "BTC/ETH" + ticker_client = _TickerMarketsMockCCXT(symbols_to_add_on_fetch=[symbol]) + with ( + mock.patch.object(ccxt_connector, 'client', new=ticker_client), + mock.patch.object(ccxt_connector.adapter, 'adapt_ticker', side_effect=lambda ticker: ticker), + mock.patch.object( + ccxt_connector, + '_persist_markets_cache_if_new_symbols', + ) as persist_markets_cache_if_new_symbols_mock, + ): + await ccxt_connector.get_price_ticker(symbol) + + persist_markets_cache_if_new_symbols_mock.assert_called_once_with(set()) + + async def test_skips_cache_persist_when_ticker_does_not_add_symbol(self, ccxt_connector): + symbol = "BTC/ETH" + ticker_client = _TickerMarketsMockCCXT() + ticker_client.markets = {symbol: {'symbol': symbol}} + with ( + mock.patch.object(ccxt_connector, 'client', new=ticker_client), + mock.patch.object(ccxt_connector.adapter, 'adapt_ticker', side_effect=lambda ticker: ticker), + mock.patch.object(ccxt_connector, '_persist_markets_cache') as persist_markets_cache_mock, + ): + await ccxt_connector.get_price_ticker(symbol) + + persist_markets_cache_mock.assert_not_called() + + +class TestCcxtConnectorGetAllCurrenciesPriceTicker: + + async def test_persists_markets_cache_when_tickers_add_symbols(self, ccxt_connector): + symbol_a = "BTC/ETH" + symbol_b = "ETH/USDT" + ticker_client = _TickerMarketsMockCCXT(symbols_to_add_on_fetch=[symbol_a, symbol_b]) + with ( + mock.patch.object(ccxt_connector, 'client', new=ticker_client), + mock.patch.object(ccxt_connector.adapter, 'adapt_ticker', side_effect=lambda ticker: ticker), + mock.patch.object( + ccxt_connector, + '_persist_markets_cache_if_new_symbols', + ) as persist_markets_cache_if_new_symbols_mock, + ): + await ccxt_connector.get_all_currencies_price_ticker(symbols=[symbol_a, symbol_b]) + + persist_markets_cache_if_new_symbols_mock.assert_called_once_with(set()) + + async def test_skips_cache_persist_when_tickers_do_not_add_symbols(self, ccxt_connector): + symbol_a = "BTC/ETH" + symbol_b = "ETH/USDT" + ticker_client = _TickerMarketsMockCCXT() + ticker_client.markets = { + symbol_a: {'symbol': symbol_a}, + symbol_b: {'symbol': symbol_b}, + } + with ( + mock.patch.object(ccxt_connector, 'client', new=ticker_client), + mock.patch.object(ccxt_connector.adapter, 'adapt_ticker', side_effect=lambda ticker: ticker), + mock.patch.object(ccxt_connector, '_persist_markets_cache') as persist_markets_cache_mock, + ): + await ccxt_connector.get_all_currencies_price_ticker(symbols=[symbol_a, symbol_b]) + + persist_markets_cache_mock.assert_not_called() + def _get_fees(type, currency, rate, cost): return { diff --git a/packages/trading/tests/exchanges/implementations/test_default_rest_exchange.py b/packages/trading/tests/exchanges/implementations/test_default_rest_exchange.py index cbcafefef3..c8fb6671f8 100644 --- a/packages/trading/tests/exchanges/implementations/test_default_rest_exchange.py +++ b/packages/trading/tests/exchanges/implementations/test_default_rest_exchange.py @@ -164,3 +164,72 @@ async def test_accepts_withdrawals_when_funds_transfer_is_enabled(self, default_ ), ): assert await default_rest_exchange.ensure_api_key_permissions() is None + + +class TestDefaultRestExchangeLazyMarketLoading: + + async def test_ensure_lazy_market_loaded_calls_load_markets_for_symbols(self, default_rest_exchange): + symbol = "BTC/ETH" + default_rest_exchange.connector.client.markets = {} + with ( + mock.patch.object(default_rest_exchange, "lazy_load_markets", mock.Mock(return_value=True)), + mock.patch.object( + default_rest_exchange, + "load_markets_for_symbols", + mock.AsyncMock(return_value=[{"symbol": symbol}]), + ) as load_markets_for_symbols_mock, + ): + await default_rest_exchange.ensure_lazy_market_loaded(symbol) + + load_markets_for_symbols_mock.assert_awaited_once_with([symbol]) + + async def test_ensure_lazy_market_loaded_skips_when_symbol_already_loaded(self, default_rest_exchange): + symbol = "BTC/ETH" + default_rest_exchange.connector.client.markets = {symbol: {"symbol": symbol}} + with ( + mock.patch.object(default_rest_exchange, "lazy_load_markets", mock.Mock(return_value=True)), + mock.patch.object( + default_rest_exchange, + "load_markets_for_symbols", + mock.AsyncMock(), + ) as load_markets_for_symbols_mock, + ): + await default_rest_exchange.ensure_lazy_market_loaded(symbol) + + load_markets_for_symbols_mock.assert_not_awaited() + + async def test_ensure_lazy_market_loaded_noop_when_lazy_load_disabled(self, default_rest_exchange): + symbol = "BTC/ETH" + default_rest_exchange.connector.client.markets = {} + with ( + mock.patch.object(default_rest_exchange, "lazy_load_markets", mock.Mock(return_value=False)), + mock.patch.object( + default_rest_exchange, + "load_markets_for_symbols", + mock.AsyncMock(), + ) as load_markets_for_symbols_mock, + ): + await default_rest_exchange.ensure_lazy_market_loaded(symbol) + + load_markets_for_symbols_mock.assert_not_awaited() + + async def test_get_market_status_including_lazy_load_ensures_before_get_market_status(self, default_rest_exchange): + symbol = "BTC/ETH" + market_status = {"symbol": symbol, "limits": {}} + with ( + mock.patch.object( + default_rest_exchange, + "ensure_lazy_market_loaded", + mock.AsyncMock(), + ) as ensure_lazy_market_loaded_mock, + mock.patch.object( + default_rest_exchange, + "get_market_status", + mock.Mock(return_value=market_status), + ) as get_market_status_mock, + ): + result = await default_rest_exchange.get_market_status_including_lazy_load(symbol, with_fixer=False) + + ensure_lazy_market_loaded_mock.assert_awaited_once_with(symbol) + get_market_status_mock.assert_called_once_with(symbol, price_example=None, with_fixer=False) + assert result is market_status diff --git a/packages/trading/tests/modes/script_keywords/basic_keywords/test_price.py b/packages/trading/tests/modes/script_keywords/basic_keywords/test_price.py index 3856a5c4f7..2b21970d4c 100644 --- a/packages/trading/tests/modes/script_keywords/basic_keywords/test_price.py +++ b/packages/trading/tests/modes/script_keywords/basic_keywords/test_price.py @@ -36,7 +36,7 @@ async def test_get_price_with_offset(null_context): null_context.symbol = "blop/plop" null_context.exchange_manager = mock.Mock( exchange=mock.Mock( - get_market_status=mock.Mock(return_value={ + get_market_status_including_lazy_load=mock.AsyncMock(return_value={ Ecmsc.PRECISION.value: { Ecmsc.PRECISION_PRICE.value: 2 } @@ -61,7 +61,7 @@ async def test_get_price_with_offset(null_context): null_context.exchange_manager, null_context.symbol, timeout=constants.ORDER_DATA_FETCHING_TIMEOUT ) parse_quantity_mock.assert_called_once_with(10) - null_context.exchange_manager.exchange.get_market_status.assert_called_once_with( + null_context.exchange_manager.exchange.get_market_status_including_lazy_load.assert_awaited_once_with( "blop/plop", with_fixer=False ) current_price_mock.reset_mock() diff --git a/packages/trading/tests/personal_data/orders/test_order_factory.py b/packages/trading/tests/personal_data/orders/test_order_factory.py index 3996fd8aee..8e4a40cc42 100644 --- a/packages/trading/tests/personal_data/orders/test_order_factory.py +++ b/packages/trading/tests/personal_data/orders/test_order_factory.py @@ -774,7 +774,7 @@ async def test_order_factory_create_base_orders_and_associated_elements_success( get_exchange_symbol_data=mock.Mock(return_value=symbol_data), ) exchange = mock.Mock( - get_market_status=mock.Mock(return_value=symbol_market), + get_market_status_including_lazy_load=mock.AsyncMock(return_value=symbol_market), get_exchange_current_time=mock.Mock(return_value=1234567890), ) exchange_manager = mock.Mock( diff --git a/packages/trading/tests_additional/real_exchanges/real_exchange_tester.py b/packages/trading/tests_additional/real_exchanges/real_exchange_tester.py index 8618566635..c2f9896b62 100644 --- a/packages/trading/tests_additional/real_exchanges/real_exchange_tester.py +++ b/packages/trading/tests_additional/real_exchanges/real_exchange_tester.py @@ -532,14 +532,22 @@ async def get_market_statuses(self): # return 2 different market status with different traded pairs to reduce possible # side effects using only one pair. async with self.get_exchange_manager() as exchange_manager: - self._ensure_market_status_cachability(exchange_manager) + await self._ensure_market_status_cachability(exchange_manager) return exchange_manager.exchange.get_market_status(self.SYMBOL), \ exchange_manager.exchange.get_market_status(self.SYMBOL_2), \ exchange_manager.exchange.get_market_status(self.SYMBOL_3) - async def assert_get_market_status_not_loaded(self): - for market_status in await self.get_market_statuses(): - assert market_status == {}, f"market status must be empty, got {market_status!r}" + async def assert_get_market_status_not_loaded(self, can_have_cache: bool = False): + statuses = await self.get_market_statuses() + missing_markets = [ + market_status + for market_status in statuses + if market_status == {} + ] + if can_have_cache: + assert 0 <= len(missing_markets) <= 2, f"market status must be empty or have one or 2 market status, got {len(missing_markets)} market status: {list(missing_markets)}" + else: + assert len(missing_markets) == len(statuses), f"market status must be empty, got {len(missing_markets)} market status: {list(missing_markets)}" async def assert_lazy_loaded_markets( self, symbols: list[str], @@ -552,9 +560,10 @@ async def assert_lazy_loaded_markets( enable_price_and_cost_comparison=True, has_price_limits=True, extra_checks: typing.Optional[typing.Callable[[dict], None]] = None, + can_have_cache: bool = False, ): # 1. markets are not loaded - await self.assert_get_market_status_not_loaded() + await self.assert_get_market_status_not_loaded(can_have_cache) # 2. fetching markets for symbols initializes market statuses async with self.get_exchange_manager() as exchange_manager: fetched_markets = await exchange_manager.exchange.load_markets_for_symbols(symbols) @@ -689,7 +698,7 @@ async def assert_market_status(self): f"LIMITS subtree missing AMOUNT/PRICE/COST buckets ({symbol=} keys={list(limits)})" ) - def _ensure_market_status_cachability(self, exchange_manager): + async def _ensure_market_status_cachability(self, exchange_manager): exchange_class = ccxt_client_util.ccxt_exchange_class_factory(self.EXCHANGE_NAME) client_using_cached_markets = exchange_class( ccxt_client_util.get_custom_domain_config(exchange_class) # use custom domain config if set @@ -698,6 +707,8 @@ def _ensure_market_status_cachability(self, exchange_manager): with pytest.raises(KeyError): ccxt_client_util.load_markets_from_cache(client_using_cached_markets, False) return + if exchange_manager.exchange.get_option_value(trading_enums.ExchangeClientOptions.LAZY_LOAD_MARKETS): + await self._ensure_populated_lazy_loaded_market_status(exchange_manager, client_using_cached_markets) ccxt_client_util.load_markets_from_cache(client_using_cached_markets, False) cached = client_using_cached_markets.markets actual = exchange_manager.exchange.connector.client.markets @@ -706,6 +717,13 @@ def _ensure_market_status_cachability(self, exchange_manager): f"(sizes {len(actual)} vs {len(cached)})" ) + async def _ensure_populated_lazy_loaded_market_status(self, exchange_manager, client_using_cached_markets): + await exchange_manager.exchange.ensure_lazy_market_loaded(self.SYMBOL) + assert self.SYMBOL in exchange_manager.exchange.connector.client.markets + await exchange_manager.exchange.ensure_lazy_market_loaded(self.SYMBOL_2) + assert self.SYMBOL in exchange_manager.exchange.connector.client.markets + assert self.SYMBOL_2 in exchange_manager.exchange.connector.client.markets + async def get_symbol_prices(self, limit=None, **kwargs): async with self.get_exchange_manager() as exchange_manager: return await exchange_manager.exchange.get_symbol_prices(