[Trading] handle lazy markets loading exchanges trading

packages/tentacles/Trading/Mode/simple_market_making_trading_mode/api/core.py

@@ -107,11 +107,25 @@ async def _fill_market_making_data_by_symbol(
                 elif dependency.symbol not in dependency_symbol_alias_by_symbol:
                     dependency_symbol_alias_by_symbol[dependency.symbol] = None
             available_symbols = set(exchange_manager.exchange.get_all_available_symbols(active_only=True))
-            symbols_to_skip_ticker_fetch = {
-                source.pair
-                for source in price_sources
-                if source.formula and source.pair not in available_symbols
-            }
+            lazy_load_markets = exchange_manager.exchange.get_option_value(
+                octobot_trading.enums.ExchangeClientOptions.LAZY_LOAD_MARKETS
+            )
+            if lazy_load_markets:
+                symbols_to_skip_ticker_fetch = {
+                    source.pair
+                    for source in price_sources
+                    if source.formula and source.pair not in available_symbols
+                }
+            else:
+                symbols_to_skip_ticker_fetch = {
+                    symbol for symbol in (set(symbols) | set(dependency_symbol_alias_by_symbol.keys()))
+                    if symbol not in available_symbols
+                }
+            if symbols_to_skip_ticker_fetch:
+                _get_logger().info(
+                    f"Ignored unavailable pairs on [{exchange_internal_name}]: "
+                    f"{sorted(symbols_to_skip_ticker_fetch)}"
+                )
             symbols_to_fetch = (set(symbols) | set(dependency_symbol_alias_by_symbol.keys())) - symbols_to_skip_ticker_fetch
             tickers = {
                 symbol: ticker
@@ -121,7 +135,11 @@ async def _fill_market_making_data_by_symbol(
             tickers, ticker_updater = await _fetch_tickers(
                 exchange_manager, tickers, list(symbols_to_fetch)
             )
-            if missing_tickers_to_fetch := [symbol for symbol in symbols_to_fetch if symbol not in tickers]:
+            if missing_tickers_to_fetch := [
+                symbol for symbol in symbols_to_fetch
+                if symbol not in tickers
+                and (lazy_load_markets or symbol in available_symbols)
+            ]:
                 try:
                     tickers.update(await ticker_updater.fetch_all_tickers(missing_tickers_to_fetch))
                 except octobot_trading.errors.NotSupported as err:
@@ -694,6 +712,7 @@ async def _get_price_and_predicted_order_book(
     books_by_symbol = {}
     for pair, reference_price in reference_price_by_pair.items():
         if not reference_price or reference_price.is_nan():
+            error_by_pair[pair] = _get_unsupported_pair_message(pair, mm_exchange)
             continue
         mm_data = mm_data_by_exchange[mm_exchange].get(pair)
         _adapt_volume_if_necessary(mm_data, reference_price)
@@ -742,6 +761,10 @@ def _get_missing_symbol_message(symbol: str, exchange: str) -> str:
     )
 
 
+def _get_unsupported_pair_message(pair: str, exchange: str) -> str:
+    return f"{pair} is not supported on {exchange}"
+
+
 def _format_format_market_making_volume(volume: typing.Union[dict, None], error: typing.Union[str, None]):
     return {
         constants.VOLUME_KEY: volume,

packages/tentacles/Trading/Mode/simple_market_making_trading_mode/tests/api/test_core.py

@@ -589,7 +589,7 @@ async def test_get_price_and_predicted_order_book_with_invalid_formula(profile_d
 
 
 async def test_get_price_and_predicted_order_book_with_error(profile_data_with_full_mm_config, mm_data_by_exchange):
-    """Test _get_price_and_predicted_order_book with error handling."""
+    """Empty reference price sources yield an unsupported-pair error."""
     profile_data_with_full_mm_config.tentacles[0].config[
         simple_market_making_trading.SimpleMarketMakingTradingMode.CONFIG_PAIR_SETTINGS
     ][0][simple_market_making_trading.SimpleMarketMakingTradingMode.REFERENCE_PRICE] = []
@@ -602,7 +602,9 @@ async def test_get_price_and_predicted_order_book_with_error(profile_data_with_f
 
     assert result == {
         "BTC/USDT": {
-            market_making_constants.ERROR_KEY: "BTC/USDT reference price on binance can't be computed from the following price sources: {}"
+            market_making_constants.ERROR_KEY: market_making_core._get_unsupported_pair_message(
+                "BTC/USDT", "binance"
+            ),
         }
     }
 
@@ -1059,21 +1061,42 @@ async def _mock_exchange_manager_context(exchange_manager):
     yield exchange_manager
 
 
+def _configure_lazy_load_markets(exchange_manager, lazy_load_markets: bool):
+    exchange_manager.exchange.get_option_value = mock.Mock(
+        side_effect=lambda option, **kwargs: (
+            lazy_load_markets
+            if option == trading_enums.ExchangeClientOptions.LAZY_LOAD_MARKETS
+            else False
+        )
+    )
+
+
+def _get_symbols_passed_to_fetch_tickers(fetch_tickers_mock) -> set:
+    return {
+        symbol
+        for call_args in fetch_tickers_mock.call_args_list
+        for symbol in call_args.args[2]
+    }
+
+
 async def _call_fill_market_making_data_by_symbol(
     price_sources,
     available_symbols,
     formula_init_patches=None,
+    lazy_load_markets=True,
 ):
     profile_data = _profile_data_for_market_making_fill()
     mm_data_by_symbol_by_exchange = {}
     exchange_manager = mock.Mock()
     exchange_manager.exchange.get_all_available_symbols = mock.Mock(
         return_value=available_symbols
     )
+    _configure_lazy_load_markets(exchange_manager, lazy_load_markets)
     ticker_updater = mock.Mock()
     ticker_updater.fetch_all_tickers = mock.AsyncMock(return_value={})
     ticker_cache = mock.Mock()
     ticker_cache.get_all_tickers = mock.Mock(return_value={})
+    fetch_tickers_mock = mock.AsyncMock(return_value=({}, ticker_updater))
 
     patches = [
         mock.patch.object(
@@ -1099,7 +1122,7 @@ async def _call_fill_market_making_data_by_symbol(
         mock.patch.object(
             market_making_core,
             "_fetch_tickers",
-            mock.AsyncMock(return_value=({}, ticker_updater)),
+            fetch_tickers_mock,
         ),
     ]
     if formula_init_patches:
@@ -1118,7 +1141,10 @@ async def _call_fill_market_making_data_by_symbol(
             auth=None,
         )
 
-    return mm_data_by_symbol_by_exchange, ticker_updater.fetch_all_tickers
+    return mm_data_by_symbol_by_exchange, {
+        "fetch_tickers": fetch_tickers_mock,
+        "fetch_all_tickers": ticker_updater.fetch_all_tickers,
+    }
 
 
 class TestFillMarketMakingDataBySymbol:
@@ -1141,12 +1167,13 @@ async def test_skips_ticker_fetch_for_unsupported_ref_price_symbol_with_formula(
             mock.patch.object(exchange_operators, "create_ohlcv_operators", return_value=[]),
             mock.patch.object(exchange_operators, "create_price_operators", return_value=[]),
         ]
-        mm_data_by_symbol_by_exchange, fetch_all_tickers_mock = await _call_fill_market_making_data_by_symbol(
+        mm_data_by_symbol_by_exchange, fetch_mocks = await _call_fill_market_making_data_by_symbol(
             price_sources,
             available_symbols={"BTC/ETH", "ETH/USDT"},
             formula_init_patches=formula_init_patches,
         )
 
+        fetch_all_tickers_mock = fetch_mocks["fetch_all_tickers"]
         assert fetch_all_tickers_mock.call_count == 0 or all(
             "BTC/USDT" not in call_args.args[0]
             for call_args in fetch_all_tickers_mock.call_args_list
@@ -1189,12 +1216,13 @@ async def test_still_fetches_formula_dependency_symbols(self):
                 return_value=exchange_operators.create_price_operators(exchange_manager, "BTC/USDT"),
             ),
         ]
-        mm_data_by_symbol_by_exchange, fetch_all_tickers_mock = await _call_fill_market_making_data_by_symbol(
+        mm_data_by_symbol_by_exchange, fetch_mocks = await _call_fill_market_making_data_by_symbol(
             price_sources,
             available_symbols={"BTC/ETH", "ETH/USDT"},
             formula_init_patches=formula_init_patches,
         )
 
+        fetch_all_tickers_mock = fetch_mocks["fetch_all_tickers"]
         assert fetch_all_tickers_mock.call_count == 1
         fetched_symbols = fetch_all_tickers_mock.call_args.args[0]
         assert "BTC/USDT" not in fetched_symbols
@@ -1203,6 +1231,7 @@ async def test_still_fetches_formula_dependency_symbols(self):
         assert set(mm_data_by_symbol_by_exchange["binance"]) == {"BTC/USDT", "BTC/ETH", "ETH/USDT"}
 
     async def test_fetches_ticker_for_unsupported_symbol_without_formula(self):
+        """Lazy-load exchanges still fetch direct pairs not listed in available_symbols."""
         price_sources = [
             advanced_reference_price_import.AdvancedPriceSource(
                 exchange="binance",
@@ -1212,13 +1241,55 @@ async def test_fetches_ticker_for_unsupported_symbol_without_formula(self):
                 formula="",
             )
         ]
-        mm_data_by_symbol_by_exchange, fetch_all_tickers_mock = await _call_fill_market_making_data_by_symbol(
+        mm_data_by_symbol_by_exchange, fetch_mocks = await _call_fill_market_making_data_by_symbol(
             price_sources,
             available_symbols={"BTC/ETH", "ETH/USDT"},
+            lazy_load_markets=True,
         )
 
-        assert fetch_all_tickers_mock.call_count == 1
-        assert fetch_all_tickers_mock.call_args.args[0] == ["BTC/USDT"]
+        fetch_tickers_mock = fetch_mocks["fetch_tickers"]
+        assert fetch_tickers_mock.call_count == 1
+        assert fetch_tickers_mock.call_args.args[2] == ["BTC/USDT"]
+
+    async def test_skips_unavailable_direct_pair_on_cex(self):
+        price_sources = [
+            advanced_reference_price_import.AdvancedPriceSource(
+                exchange="binance",
+                pair="BTC/USDT",
+                time_frame=advanced_reference_price_import.DEFAULT_TIME_FRAME,
+                weight=decimal.Decimal("1.0"),
+                formula="",
+            )
+        ]
+        mm_data_by_symbol_by_exchange, fetch_mocks = await _call_fill_market_making_data_by_symbol(
+            price_sources,
+            available_symbols={"BTC/ETH", "ETH/USDT"},
+            lazy_load_markets=False,
+        )
+
+        assert "BTC/USDT" not in _get_symbols_passed_to_fetch_tickers(fetch_mocks["fetch_tickers"])
+        btc_usdt_data = mm_data_by_symbol_by_exchange["binance"]["BTC/USDT"]
+        assert btc_usdt_data.price.is_nan()
+
+    async def test_still_fetches_unavailable_direct_pair_on_lazy_load_exchange(self):
+        price_sources = [
+            advanced_reference_price_import.AdvancedPriceSource(
+                exchange="binance",
+                pair="BTC/USDT",
+                time_frame=advanced_reference_price_import.DEFAULT_TIME_FRAME,
+                weight=decimal.Decimal("1.0"),
+                formula="",
+            )
+        ]
+        _, fetch_mocks = await _call_fill_market_making_data_by_symbol(
+            price_sources,
+            available_symbols={"BTC/ETH", "ETH/USDT"},
+            lazy_load_markets=True,
+        )
+
+        fetch_tickers_mock = fetch_mocks["fetch_tickers"]
+        assert fetch_tickers_mock.call_count == 1
+        assert fetch_tickers_mock.call_args.args[2] == ["BTC/USDT"]
 
 
 def _mock_create_price_operators(prices_by_symbol: dict[str, float]):

packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py

@@ -170,10 +170,34 @@ def _ensure_successful_markets_fetch(self, client):
                     f"No spot markets found for {self.exchange_manager.exchange_name}: {len(symbols)} fetched markets: {logged_symbols}"
                 )
 
+    def _persist_markets_cache(
+        self,
+        client=None,
+        authenticated_cache: typing.Optional[bool] = None,
+    ):
+        if client is None:
+            client = self.client
+        if authenticated_cache is None:
+            authenticated_cache = self.exchange_manager.exchange.requires_authentication_for_this_configuration_only()
+        ccxt_client_util.set_ccxt_client_cache(client, authenticated_cache)
+
+    def _persist_markets_cache_if_new_symbols(
+        self,
+        previous_market_symbols: set[str],
+        client=None,
+    ):
+        if client is None:
+            client = self.client
+        current_market_symbols = set((client.markets or {}).keys())
+        if current_market_symbols - previous_market_symbols:
+            self._persist_markets_cache(client)
+
     async def load_markets_for_symbols(self, symbols: list[str]) -> list[dict]:
         if not self.client.has.get('obLoadMarketsForSymbols'):
             raise octobot_trading.errors.NotSupported("This exchange doesn't support lazyLoadMarkets")
-        return await self.client.ob_load_markets_for_symbols(symbols)
+        loaded_markets = await self.client.ob_load_markets_for_symbols(symbols)
+        self._persist_markets_cache()
+        return loaded_markets
 
     async def _filtered_if_necessary_load_markets(
         self,
@@ -256,7 +280,7 @@ async def load_symbol_markets(
             )
             try:
                 await self._load_markets(self.client, reload, market_filter=market_filter)
-                ccxt_client_util.set_ccxt_client_cache(self.client, authenticated_cache)
+                self._persist_markets_cache()
             except ccxt.async_support.OBIPWhitelistError as err:
                 raise octobot_trading.errors.InvalidAPIKeyIPWhitelistError(
                     f"Invalid IP whitelist error: {html_util.get_html_summary_if_relevant(err)}"
@@ -299,7 +323,7 @@ async def load_symbol_markets(
                     try:
                         unauth_client = self._client_factory(True)[0]
                         await self._load_markets(unauth_client, reload, market_filter=market_filter)
-                        ccxt_client_util.set_ccxt_client_cache(unauth_client, False)
+                        self._persist_markets_cache(unauth_client, False)
                         # apply markets to target client
                         ccxt_client_util.load_markets_from_cache(self.client, False, market_filter=market_filter)
                         self.logger.debug(
@@ -648,25 +672,33 @@ async def get_recent_trades(self, symbol: str, limit: int = 50, **kwargs: dict)
     async def get_price_ticker(self, symbol: str, **kwargs: dict) -> typing.Optional[dict]:
         try:
             with self.error_describer(False):
-                return self.adapter.adapt_ticker(
+                previous_market_symbols = set((self.client.markets or {}).keys())
+                ticker = self.adapter.adapt_ticker(
                     await self.client.fetch_ticker(symbol, params=kwargs)
                 )
+                self._persist_markets_cache_if_new_symbols(previous_market_symbols)
+                return ticker
+        except ccxt.async_support.BadSymbol as err:
+            raise octobot_trading.errors.UnSupportedSymbolError(str(err)) from err
         except ccxt.async_support.NotSupported:
             raise octobot_trading.errors.NotSupported
         except ccxt.async_support.BaseError as e:
             raise octobot_trading.errors.FailedRequest(
                 f"Failed to get_price_ticker {html_util.get_html_summary_if_relevant(e)}"
-            )
+            ) from e
 
     @ccxt_client_util.converted_ccxt_common_errors
     async def get_all_currencies_price_ticker(
         self, symbols: typing.Optional[list[str]] = None, can_try_to_fix_missing_tickers: bool = True, **kwargs: dict
     ) -> typing.Optional[dict[str, dict]]:
         try:
             with self.error_describer(False):
+                previous_market_symbols = set((self.client.markets or {}).keys())
+                fetched_tickers = await self.client.fetch_tickers(symbols, params=kwargs)
+                self._persist_markets_cache_if_new_symbols(previous_market_symbols)
                 tickers = {
                     symbol: self.adapter.adapt_ticker(ticker)
-                    for symbol, ticker in (await self.client.fetch_tickers(symbols, params=kwargs)).items()
+                    for symbol, ticker in fetched_tickers.items()
                 }
                 # self.all_currencies_price_ticker should always contain as many tickers as possible: don't override it
                 # with less symbols when fetching only a few tickers

packages/trading/octobot_trading/exchanges/types/rest_exchange.py

@@ -582,6 +582,21 @@ async def load_markets_for_symbols(self, symbols: list[str]) -> list[dict]:
         )
         return loaded_markets
 
+    def _is_lazy_market_loaded(self, symbol: str) -> bool:
+        client_markets = self.connector.client.markets or {}
+        return symbol in client_markets
+
+    async def ensure_lazy_market_loaded(self, symbol: str) -> None:
+        if not self.lazy_load_markets():
+            return
+        if self._is_lazy_market_loaded(symbol):
+            return
+        await self.load_markets_for_symbols([symbol])
+
+    async def get_market_status_including_lazy_load(self, symbol, price_example=None, with_fixer=True):
+        await self.ensure_lazy_market_loaded(symbol)
+        return self.get_market_status(symbol, price_example=price_example, with_fixer=with_fixer)
+
     def get_market_status(self, symbol, price_example=None, with_fixer=True):
         """
         Override using get_fixed_market_status in exchange tentacle if the default market status is not as expected

packages/trading/octobot_trading/modes/modes_util.py

@@ -195,7 +195,7 @@ async def convert_with_market_or_limit_order(
 
     # get order quantity
     quantity = _get_available_or_target_quantity(exchange_mgr, symbol, order_type, price, asset_amount)
-    symbol_market = exchange_mgr.exchange.get_market_status(symbol, with_fixer=False)
+    symbol_market = await exchange_mgr.exchange.get_market_status_including_lazy_load(symbol, with_fixer=False)
     created_orders = []
     for order_quantity, order_price in \
             trading_personal_data.decimal_check_and_adapt_order_details_if_necessary(

packages/trading/octobot_trading/modes/script_keywords/basic_keywords/price.py

@@ -63,5 +63,5 @@ async def get_price_with_offset(context, offset_input, use_delta_type_as_flat_va
             f"1.2, -0.222, -0.222d, @65100, 5%, e5%, e500"
         )
 
-    symbol_market = context.exchange_manager.exchange.get_market_status(context.symbol, with_fixer=False)
+    symbol_market = await context.exchange_manager.exchange.get_market_status_including_lazy_load(context.symbol, with_fixer=False)
     return personal_data.decimal_adapt_price(symbol_market, computed_price)

packages/trading/octobot_trading/personal_data/orders/order_factory.py

@@ -473,7 +473,7 @@ async def create_base_orders_and_associated_elements(
         current_price = await personal_data.get_up_to_date_price(
             self.exchange_manager, symbol=symbol, timeout=constants.ORDER_DATA_FETCHING_TIMEOUT
         )
-        symbol_market = self.exchange_manager.exchange.get_market_status(symbol, with_fixer=False)
+        symbol_market = await self.exchange_manager.exchange.get_market_status_including_lazy_load(symbol, with_fixer=False)
         ctx = script_keywords.get_base_context_from_exchange_manager(self.exchange_manager, symbol)
             # market orders have no price
         computed_price = current_price if price is None else await self._get_computed_price(ctx, price)

packages/trading/octobot_trading/personal_data/orders/order_util.py

@@ -272,7 +272,7 @@ async def get_pre_order_data(exchange_manager, symbol: str, timeout: int = None,
                              portfolio_type=commons_constants.PORTFOLIO_AVAILABLE,
                              target_price=None):
     price = target_price or await get_up_to_date_price(exchange_manager, symbol, timeout=timeout)
-    symbol_market = exchange_manager.exchange.get_market_status(symbol, with_fixer=False)
+    symbol_market = await exchange_manager.exchange.get_market_status_including_lazy_load(symbol, with_fixer=False)
     currency_available, market_available, market_quantity = get_portfolio_amounts(
         exchange_manager, symbol, price, portfolio_type=portfolio_type
     )