diff --git a/docs/content/guides/octobot-configuration/profiles.md b/docs/content/guides/octobot-configuration/profiles.md index 475f2dbfe8..d3f1b29ea2 100644 --- a/docs/content/guides/octobot-configuration/profiles.md +++ b/docs/content/guides/octobot-configuration/profiles.md @@ -6,10 +6,15 @@ sidebar_position: 1 # Profiles -OctoBot's trading configuration is using profiles (located into -user/profiles). This allows for quick switches between previously set +OctoBot's trading configuration is using profiles. This allows for quick switches between previously set configurations. Each profile defines a [Trading Mode](/guides/octobot-trading-modes/trading-modes) configuration as well as other settings. +Bundled default profiles (for example `default` and `non-trading`) remain on the filesystem under +`user/profiles/`. When a wallet is configured, user-created profiles are stored in the sync +`StrategyProvider` collection as `GenericProcessConfiguration.profile_data` (same profile id as the +strategy). OctoBot still exposes the same profile API to the web UI and configuration layer; only the +profile module selects the storage backend. + ![octobot trading mode details from profiles](/images/guides/configuration/octobot-trading-mode-details-from-profiles.png) Profiles include: diff --git a/octobot/cli.py b/octobot/cli.py index 7023c4be76..c7936d0e1e 100644 --- a/octobot/cli.py +++ b/octobot/cli.py @@ -134,23 +134,30 @@ def _create_configuration(): return config -def _create_startup_config(logger, default_config_file): +def _create_startup_config(logger, default_config_file, *, is_process_child: bool = False): logger.info("Loading config files...") config = _create_configuration() is_first_startup = config.is_config_file_empty_or_missing() if is_first_startup: - logger.info("No configuration found creating default configuration...") - configuration_manager.init_config(from_config_file=default_config_file) - config.read(should_raise=False) + user_config_path = configuration.get_user_config() + if is_process_child: + raise errors.ConfigError( + f"Process OctoBot child expected prepared {common_constants.CONFIG_FILE} at " + f"{user_config_path!r} (under {common_constants.USER_AUTOMATIONS_FOLDER}//). " + f"The executor must materialize the automation layout before spawn; if this file is missing, " + f"check ensure_user_profile_and_layout timing or parallel functional test races." + ) + logger.info( + f"No configuration found in {user_config_path}. " + f"Creating default configuration..." + ) + configuration_manager.init_config( + config_file=user_config_path, + from_config_file=default_config_file, + ) + config.read(should_raise=False, activate_profile=False) else: - _read_config(config, logger) - try: - commands.ensure_profile(config) - _validate_config(config, logger) - except (errors.NoProfileError, errors.ConfigError): - # real issue if tentacles exist otherwise continue - if os.path.isdir(tentacles_manager_constants.TENTACLES_PATH): - raise + _read_config(config, logger, activate_profile=False) distribution = configuration_manager.get_distribution(config.config) if distribution is not enums.OctoBotDistribution.DEFAULT: logger.info(f"Using {distribution.value} OctoBot distribution.") @@ -183,7 +190,7 @@ async def _apply_db_bot_config(logger, config, community_auth) -> bool: constants.COMMUNITY_BOT_ID, constants.USER_AUTH_KEY, ) - profile = await profiles.import_profile_data_as_profile( + profile = await config.profile_storage.import_profile_data( profile_data, constants.PROFILE_FILE_SCHEMA, None, @@ -277,17 +284,31 @@ def _apply_forced_configs(community_auth, logger, config, is_first_startup): _apply_env_variables_to_config(logger, config) -def _read_config(config, logger): +def _read_config(config, logger, *, activate_profile=True): try: - config.read(should_raise=True, fill_missing_fields=True) + config.read( + should_raise=True, + fill_missing_fields=True, + activate_profile=activate_profile, + ) except errors.NoProfileError: _repair_with_default_profile(config, logger) config = _create_configuration() - config.read(should_raise=False, fill_missing_fields=True) + config.read( + should_raise=False, + fill_missing_fields=True, + activate_profile=activate_profile, + ) except Exception as e: raise errors.ConfigError(e) +def _activate_saved_profile_after_sync(config, logger): + config.activate_saved_profile() + commands.ensure_profile(config) + _validate_config(config, logger) + + def _validate_config(config, logger): try: config.validate() @@ -314,9 +335,7 @@ def _load_or_create_tentacles(community_auth, config, logger): ): # when tentacles folder already exists config.load_profiles_if_possible_and_necessary() - tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config( - config.get_tentacles_config_path() - ) + tentacles_setup_config = config.get_active_tentacles_setup_config() commands.run_update_or_repair_tentacles_if_necessary(community_auth, config, tentacles_setup_config) else: # when no tentacles folder has been found @@ -338,6 +357,55 @@ def _init_cli_overriden_folders(args): return overrides, logs_folder +def _assert_process_child_folder_overrides(args) -> None: + """When ``--dump-state`` is set (DSL process children), require an automation-scoped ``--user-folder``.""" + if not args.dump_state or not str(args.dump_state).strip(): + return + user_folder = args.user_folder + if not user_folder or not str(user_folder).strip(): + raise errors.ConfigError( + "Process OctoBot children require --user-folder under " + f"{common_constants.USER_AUTOMATIONS_FOLDER}// when --dump-state is set." + ) + path_segments = tuple( + segment + for segment in str(user_folder).replace("\\", "/").split("/") + if segment + ) + expected_prefix = ( + common_constants.USER_FOLDER, + common_constants.AUTOMATIONS_FOLDER, + ) + if len(path_segments) < len(expected_prefix) + 1: + raise errors.ConfigError( + "Process OctoBot children require --user-folder under " + f"{common_constants.USER_AUTOMATIONS_FOLDER}//, got " + f"{user_folder!r}." + ) + if path_segments[: len(expected_prefix)] != expected_prefix: + raise errors.ConfigError( + "Process OctoBot children require --user-folder to start with " + f"{common_constants.USER_AUTOMATIONS_FOLDER}/, got {user_folder!r}." + ) + if ".." in path_segments: + raise errors.ConfigError( + f"Invalid --user-folder for process child: parent segments are not allowed ({user_folder!r})." + ) + +def _configure_profile_sync_user(config, community_auth, *, is_process_child: bool = False): + if is_process_child: + sync_user_id = constants.PROCESS_BOT_SYNC_USER_ID + if not sync_user_id: + raise errors.ConfigError( + "Process OctoBot children require " + f"{constants.ENV_PROCESS_BOT_SYNC_USER_ID} to be set by the executor." + ) + config.profile_storage.bind_process_child_sync_user_id(sync_user_id) + return + if community_auth is not None and community_auth.auto_init_sync_client(): + config.profile_storage.configure_sync_user(community_auth.sync_user_id) + + def start_octobot(args, default_config_file=None): logger = None try: @@ -346,6 +414,7 @@ def start_octobot(args, default_config_file=None): return overrides, logs_folder = _init_cli_overriden_folders(args) + _assert_process_child_folder_overrides(args) logger = octobot_logger.init_logger(logs_folder=logs_folder) startup_messages = [] @@ -364,8 +433,11 @@ def start_octobot(args, default_config_file=None): octobot_community.init_sentry_tracker() # load configuration + is_process_child = bool(args.dump_state and str(args.dump_state).strip()) config, is_first_startup = _create_startup_config( - logger, default_config_file or constants.DEFAULT_CONFIG_FILE + logger, + default_config_file or constants.DEFAULT_CONFIG_FILE, + is_process_child=is_process_child, ) # check config loading @@ -387,6 +459,14 @@ def start_octobot(args, default_config_file=None): _get_authenticated_community_if_possible(config, logger) ) + _configure_profile_sync_user( + config, + community_auth, + is_process_child=is_process_child, + ) + + _activate_saved_profile_after_sync(config, logger) + # tries to load, install or repair tentacles _load_or_create_tentacles(community_auth, config, logger) diff --git a/octobot/commands.py b/octobot/commands.py index 25f4261582..fa492ae387 100644 --- a/octobot/commands.py +++ b/octobot/commands.py @@ -333,7 +333,7 @@ async def start_bot(bot, logger, catch=False): def stop_bot(bot, force=False): - bot.task_manager.stop_tasks() + bot.task_manager.stop_tasks(stop_managed_child_processes=True, force=force) if force: os._exit(0) diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index db5a35049a..20d4479d5c 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -44,6 +44,7 @@ import octobot_commons.authentication as authentication import octobot_commons.configuration as commons_configuration import octobot_commons.profiles as commons_profiles +import octobot_commons.user_root_folder_provider as user_root_folder_provider import octobot_trading.enums as trading_enums import octobot_sync.client as sync_client import octobot_sync.chain as sync_chain @@ -129,10 +130,10 @@ def __init__(self, config=None, backend_url=None, backend_key=None, use_as_singl self._fetch_account_task: typing.Optional[asyncio.Task] = None self._sync_client = None - self._sync_user_id: str = "" + self.sync_user_id: str = "" self._sync_client_lock = threading.Lock() self._wallet_backend: wallet_backend.WalletBackend = wallet_backend.WalletBackend( - self.configuration_storage.sync_storage, self.logger + self._get_wallet_sync_storage(), self.logger ) @staticmethod @@ -144,6 +145,23 @@ def create(configuration: commons_configuration.Configuration, **kwargs): def update(self, configuration: commons_configuration.Configuration): self.configuration_storage.set_configuration(configuration) + self._wallet_backend = wallet_backend.WalletBackend( + self._get_wallet_sync_storage(), self.logger + ) + + def _get_wallet_sync_storage(self): + sync_data_root = os.path.normpath(user_root_folder_provider.get_sync_data_root()) + user_root = os.path.normpath(user_root_folder_provider.get_user_root_folder()) + if sync_data_root != user_root: + master_config_path = os.path.join(sync_data_root, commons_constants.CONFIG_FILE) + master_config = commons_configuration.Configuration( + master_config_path, + os.path.join(sync_data_root, commons_constants.PROFILES_FOLDER), + ) + if os.path.isfile(master_config_path): + master_config.read(should_raise=False, activate_profile=False) + return supabase_backend.SyncConfigurationStorage(master_config) + return self.configuration_storage.sync_storage def get_logged_in_email(self): if self.user_account.has_user_data(): @@ -712,7 +730,7 @@ def init_sync_client_for_wallet(self, address: str) -> None: self.logger.debug("No sync server URL configured, skipping sync client init") return wallet = self.get_wallet(address) - self._sync_client, self._sync_user_id = sync_client.create_sync_client( + self._sync_client, self.sync_user_id = sync_client.create_sync_client( private_key=wallet.private_key, sync_url=sync_url, ) @@ -745,7 +763,7 @@ def auto_init_sync_client(self) -> bool: "No sync server URL configured, skipping auto sync client init" ) return False - self._sync_client, self._sync_user_id = sync_client.create_sync_client( + self._sync_client, self.sync_user_id = sync_client.create_sync_client( private_key=wallet.private_key, sync_url=sync_url, ) diff --git a/octobot/configuration_manager.py b/octobot/configuration_manager.py index 8b2e9d9dd9..087b3878de 100644 --- a/octobot/configuration_manager.py +++ b/octobot/configuration_manager.py @@ -115,7 +115,7 @@ def config_health_check(config: configuration.Configuration, in_backtesting: boo def init_config( - config_file=configuration.get_user_config(), + config_file=None, from_config_file=constants.DEFAULT_CONFIG_FILE ): """ @@ -123,6 +123,8 @@ def init_config( :param config_file: the config file path :param from_config_file: the default config file path """ + if config_file is None: + config_file = configuration.get_user_config() try: user_root = user_root_folder_provider.get_user_root_folder() if not os.path.exists(user_root): diff --git a/octobot/constants.py b/octobot/constants.py index 9074a541a4..ac41c88a3b 100644 --- a/octobot/constants.py +++ b/octobot/constants.py @@ -215,6 +215,9 @@ # Process bot state JSON next to user config (--dump-state); liveness for run_octobot_process PROCESS_BOT_STATE_FILE_NAME = "process_bot_state.json" ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS = "OCTOBOT_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS" +ENV_PROCESS_BOT_SYNC_USER_ID = "OCTOBOT_PROCESS_BOT_SYNC_USER_ID" +PROCESS_BOT_SYNC_USER_ID = os.environ.get(ENV_PROCESS_BOT_SYNC_USER_ID, "").strip() + PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS = float( os.getenv(ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS, "30") ) @@ -292,6 +295,15 @@ ENABLE_RUN_DATABASE_LIMIT = os_util.parse_boolean_environment_var("ENABLE_RUN_DATABASE_LIMIT", "True") MAX_TOTAL_RUN_DATABASES_SIZE = int(os.getenv("MAX_TOTAL_RUN_DATABASES_SIZE", DEFAULT_MAX_TOTAL_RUN_DATABASES_SIZE)) +# Managed child OctoBot processes (run_octobot_process) +MANAGED_CHILD_GRACEFUL_STOP_TIMEOUT_SECONDS = float( + os.getenv("OCTOBOT_MANAGED_CHILD_STOP_TIMEOUT", "10.0") +) + +OCTOBOT_STOP_TIMEOUT_SECONDS = float( + os.getenv("OCTOBOT_STOP_TIMEOUT_SECONDS", "30.0") +) + # Channel OCTOBOT_CHANNEL = "OctoBot" diff --git a/octobot/initializer.py b/octobot/initializer.py index ced0aa1628..ccfcbfab11 100644 --- a/octobot/initializer.py +++ b/octobot/initializer.py @@ -13,7 +13,6 @@ # # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . -import octobot_tentacles_manager.api as tentacles_manager_api import octobot.constants as constants import octobot_commons.databases as databases import octobot_commons.logging as logging @@ -31,9 +30,8 @@ def __init__(self, octobot): async def create(self, init_bot_storage): # initialize tentacle configuration - tentacles_config_path = self.octobot.get_startup_config(constants.CONFIG_KEY, dict_only=False).\ - get_tentacles_config_path() - self.octobot.tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config(tentacles_config_path) + startup_config = self.octobot.get_startup_config(constants.CONFIG_KEY, dict_only=False) + self.octobot.tentacles_setup_config = startup_config.get_active_tentacles_setup_config() if init_bot_storage: try: diff --git a/octobot/limits.py b/octobot/limits.py index 71277dccc3..1c8206fcc1 100644 --- a/octobot/limits.py +++ b/octobot/limits.py @@ -21,7 +21,6 @@ import octobot_commons.enums as common_enums import octobot_commons.logging as logging import octobot_commons.time_frame_manager as time_frame_manager -import octobot_tentacles_manager.api as tentacles_manager_api import octobot_evaluators.api as evaluators_api import octobot_trading.api as trading_api @@ -79,7 +78,7 @@ def _apply_symbols_limits(dict_config, logger, limit): def _apply_time_frames_limits(full_config, logger, limit): - tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config(full_config.get_tentacles_config_path()) + tentacles_setup_config = full_config.get_active_tentacles_setup_config() has_disabled_time_frames = False all_enabled_time_frames = [] # patch time frames config diff --git a/octobot/storage/process_bot_state_dumper.py b/octobot/storage/process_bot_state_dumper.py index ff51f7a9f1..bed8c0a257 100644 --- a/octobot/storage/process_bot_state_dumper.py +++ b/octobot/storage/process_bot_state_dumper.py @@ -82,7 +82,7 @@ async def _write_state_file_async( bot, ), ) - content = state.to_dict(include_default_values=False) + content = json_util.sanitize(state.to_dict(include_default_values=False)) str_content = json_util.dump_formatted_json(content) full_path = os.path.abspath(state_file_path) directory = os.path.dirname(full_path) diff --git a/octobot/task_manager.py b/octobot/task_manager.py index af81034491..ba5e3d27b9 100644 --- a/octobot/task_manager.py +++ b/octobot/task_manager.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . import asyncio +import os import threading import concurrent.futures as thread import traceback @@ -22,6 +23,7 @@ import octobot_commons.asyncio_tools as asyncio_tools import octobot_commons.logging as logging import octobot_commons.constants as commons_constants +import octobot_commons.managed_child_process_registry as managed_child_process_registry import octobot.constants as constants import octobot.storage.process_bot_state_dumper as process_bot_state_dumper @@ -93,7 +95,7 @@ def run_forever(self, coroutine): while self.loop_forever_thread.is_alive(): self.loop_forever_thread.join(timeout=1) - def stop_tasks(self, stop_octobot=True): + def stop_tasks(self, stop_octobot=True, stop_managed_child_processes=False, force=False): self.logger.info("Stopping tasks...") async def stop_timeout(timeout): @@ -101,7 +103,9 @@ async def stop_timeout(timeout): stop_coroutines = [] if stop_octobot: - allowed_seconds_to_stop = 10 + allowed_seconds_to_stop = constants.OCTOBOT_STOP_TIMEOUT_SECONDS + if stop_managed_child_processes: + stop_coroutines.append(self._graceful_stop_managed_child_processes()) stop_coroutines.append(self.octobot.stop()) stop_coroutines.append(stop_timeout(allowed_seconds_to_stop)) @@ -120,16 +124,19 @@ async def _await_timeouted_gather(tasks): # await this gather to be sure to complete each stop call or timeout try: await asyncio.gather(*tasks) - except asyncio.exceptions.TimeoutError: + except TimeoutError: self.logger.warning(f"Timeout while stopping tasks, forcing stop.") raise if stop_coroutines: try: asyncio_tools.run_coroutine_in_asyncio_loop(_await_timeouted_gather(stop_coroutines), self.async_loop) - except asyncio.exceptions.TimeoutError: + except TimeoutError: self.logger.info(f"Remaining threads: {self._get_remaining_threads()}") - sys.exit(-1) + if force: + os._exit(1) + else: + sys.exit(-1) self.async_loop.stop() # ensure there is at least one element in the event loop tasks # not to block on base_event.py#self._selector.select(timeout) which prevents run_forever() from completing @@ -138,6 +145,11 @@ async def _await_timeouted_gather(tasks): self.logger.debug(f"Remaining threads: {self._get_remaining_threads()}") self.logger.info("Tasks stopped.") + async def _graceful_stop_managed_child_processes(self): + await managed_child_process_registry.ManagedChildProcessRegistry.instance().graceful_stop_all( + timeout_seconds=constants.MANAGED_CHILD_GRACEFUL_STOP_TIMEOUT_SECONDS, + ) + def _get_remaining_threads(self): return [ f"{alive_thread.name}{'[daemon]' if alive_thread.daemon else ''}" diff --git a/packages/commons/octobot_commons/authentication.py b/packages/commons/octobot_commons/authentication.py index 937b3f59e0..37f403c43d 100644 --- a/packages/commons/octobot_commons/authentication.py +++ b/packages/commons/octobot_commons/authentication.py @@ -20,6 +20,7 @@ import octobot_commons.logging as bot_logging import octobot_commons.singleton as singleton +import octobot_sync.chain as sync_chain class Authenticator(singleton.Singleton): @@ -178,6 +179,12 @@ def has_open_source_package(self) -> bool: """ raise NotImplementedError + def get_wallet_by_user_id(self, user_id: str) -> sync_chain.Wallet: + """ + Returns the wallet by user id + """ + raise NotImplementedError + @staticmethod async def wait_and_check_has_open_source_package(raise_on_timeout=False) -> bool: """ diff --git a/packages/commons/octobot_commons/configuration/config_file_manager.py b/packages/commons/octobot_commons/configuration/config_file_manager.py index f7348a0cb7..3fe7565464 100644 --- a/packages/commons/octobot_commons/configuration/config_file_manager.py +++ b/packages/commons/octobot_commons/configuration/config_file_manager.py @@ -88,6 +88,7 @@ def dump( ) raise global_exception + logging.get_logger(LOGGER_NAME).info(f"Saving config to {config_file}") json_util.safe_dump(config, config_file) diff --git a/packages/commons/octobot_commons/configuration/configuration.py b/packages/commons/octobot_commons/configuration/configuration.py index 5d0deb0f8a..709cbfb184 100644 --- a/packages/commons/octobot_commons/configuration/configuration.py +++ b/packages/commons/octobot_commons/configuration/configuration.py @@ -17,12 +17,13 @@ import os import functools import copy -import shutil +import typing import octobot_commons.logging as logging import octobot_commons.errors as errors import octobot_commons.constants as commons_constants import octobot_commons.profiles as profiles +import octobot_commons.profiles.profile_storage as profile_storage_module import octobot_commons.json_util as json_util import octobot_commons.configuration.config_file_manager as config_file_manager import octobot_commons.configuration.config_operations as config_operations @@ -54,6 +55,10 @@ def __init__( self._read_config: dict = None self.profile: profiles.Profile = None self.profile_by_id: dict = {} + self.profile_storage = profile_storage_module.ProfileStorage( + profiles_path, + profile_schema_path, + ) def validate(self) -> None: """ @@ -63,13 +68,20 @@ def validate(self) -> None: json_util.validate(self._read_config, self.config_schema_path) self.profile.validate() - def read(self, should_raise=True, fill_missing_fields=False) -> None: + def read( + self, + should_raise=True, + fill_missing_fields=False, + *, + activate_profile=True, + ) -> None: """ Reads the configuration from self.config_path and load the current profile Overall config is stored into self.config and consists of a merger from the user config and activated profile :param should_raise: will raise upon exception when True :param fill_missing_fields: will try to fill in missing fields when true + :param activate_profile: when False, only load config.json without selecting a profile :return: None """ self._read_config = config_file_manager.load( @@ -78,6 +90,13 @@ def read(self, should_raise=True, fill_missing_fields=False) -> None: fill_missing_fields=fill_missing_fields, ) self.config = copy.deepcopy(self._read_config) + if activate_profile: + self.load_profiles_if_possible_and_necessary() + + def activate_saved_profile(self) -> None: + """ + Load all profiles and select CONFIG_PROFILE from the last read config.json. + """ self.load_profiles_if_possible_and_necessary() def load_profiles_if_possible_and_necessary(self) -> None: @@ -97,6 +116,7 @@ def select_profile(self, profile_id) -> None: """ self.config[commons_constants.CONFIG_PROFILE] = profile_id self.profile = self.profile_by_id[profile_id] + self.profile_storage.activate_profile(self.profile) self.logger.info(f"Using {self.profile.name} profile.") self._generate_config_from_user_config_and_profile() @@ -110,8 +130,13 @@ def remove_profile(self, profile_id: str) -> None: if profile.read_only and not profile.imported: raise errors.ProfileRemovalError(f"{profile.name} profile can't be removed") try: - shutil.rmtree(profile.path) + self.profile_storage.delete_profile( + profile_id, + profile=profile, + ) self.profile_by_id.pop(profile_id, None) + except errors.ProfileRemovalError: + raise except Exception as err: raise errors.ProfileRemovalError() from err @@ -128,7 +153,8 @@ def _generate_config_from_user_config_and_profile(self): def save( self, schema_file=None, - sync_all_profiles=False, + sync_all_profiles: bool = False, + save_profile: typing.Optional[bool] = None, ) -> None: """ Save the current self.config and self.profile. @@ -141,11 +167,52 @@ def save( config_to_save, schema_file=schema_file, ) - if self.profile is not None: + if save_profile is None: + save_profile = self._profile_managed_elements_changed() + if ( + save_profile + and self.profile is not None + and not self.profile_storage.is_master_overlay_profile(self.profile) + ): self.profile.save_config(self.config) if sync_all_profiles: self._sync_other_profiles() + def _profile_managed_elements_changed(self) -> bool: + if self.profile is None: + return False + for element in self.profile.FULLY_MANAGED_ELEMENTS: + if element in self.config: + if self.config[element] != self.profile.config.get(element): + return True + for element in self.profile.PARTIALLY_MANAGED_ELEMENTS: + if self._partially_managed_element_would_change(element): + return True + return False + + def _partially_managed_element_would_change(self, element: str) -> bool: + if element not in self.config: + return False + allowed_keys = profiles.Profile.PARTIALLY_MANAGED_ELEMENTS_ALLOWED_KEYS.get( + element + ) + if allowed_keys is None: + return self.config[element] != self.profile.config.get(element) + global_element = self.config[element] + profile_element = self.profile.config.get(element, {}) + for exchange_name, global_exchange_config in global_element.items(): + if not isinstance(global_exchange_config, dict): + continue + profile_exchange_config = profile_element.get(exchange_name, {}) + if not isinstance(profile_exchange_config, dict): + return True + for allowed_key in allowed_keys: + if global_exchange_config.get(allowed_key) != profile_exchange_config.get( + allowed_key + ): + return True + return False + def _sync_other_profiles(self): """ Update profile partially managed elements for all profiles except self.profile @@ -193,8 +260,12 @@ def are_profiles_empty_or_missing(self) -> bool: Checks if self.profiles_path exists and contains folders :return: True if profiles folder is not empty """ + self._prepare_profile_storage() return not ( - os.path.isdir(self.profiles_path) and os.listdir(self.profiles_path) + self.profile_storage.has_any_profiles() + or ( + os.path.isdir(self.profiles_path) and os.listdir(self.profiles_path) + ) ) def get_non_imported_profiles(self) -> list: @@ -211,6 +282,23 @@ def get_tentacles_config_path(self) -> str: """ return self.profile.get_tentacles_config_path() + def get_active_tentacles_setup_config(self): + """ + :return: The tentacles setup config for the activated profile. + Profile-data-backed profiles use in-memory setup; filesystem profiles use their config path. + """ + if self.profile.is_profile_data_tentacle_backed(): + if self.profile.tentacles_setup_config is None: + self.profile.init_tentacles_setup_config() + tentacles_setup_config = self.profile.tentacles_setup_config + return self.profile.bind_tentacles_setup_config(tentacles_setup_config) + import octobot_tentacles_manager.api as tentacles_manager_api + + return tentacles_manager_api.get_tentacles_setup_config( + self.get_tentacles_config_path(), + profile=self.profile, + ) + def get_metrics_enabled(self) -> bool: """ Check if metrics are enabled @@ -302,19 +390,62 @@ def _get_selected_profile(self): selected_profile_id != commons_constants.DEFAULT_PROFILE and commons_constants.DEFAULT_PROFILE in self.profile_by_id ): + self.logger.warning( + "Profile %r from config.json is not available yet; falling back to %r. " + "This can happen when sync profiles are not loaded.", + selected_profile_id, + commons_constants.DEFAULT_PROFILE, + ) return commons_constants.DEFAULT_PROFILE raise errors.NoProfileError + def _prepare_profile_storage(self) -> None: + self.profile_storage.configure_paths( + self.profiles_path, self.profile_schema_path + ) + if self.config: + readonly_profiles_path = self.config.get( + commons_constants.CONFIG_READONLY_PROFILES_PATH + ) + if readonly_profiles_path: + self.profile_storage.configure_readonly_profiles_path( + readonly_profiles_path + ) + def load_profiles(self) -> None: """ Loads the available profiles :return: None """ - for profile in profiles.Profile.get_all_profiles( - self.profiles_path, schema_path=self.profile_schema_path - ): - if profile.profile_id not in self.profile_by_id: - self.profile_by_id[profile.profile_id] = profile + self._prepare_profile_storage() + loaded_profiles = self.profile_storage.load_all_profiles() + for profile_id, profile in loaded_profiles.items(): + if profile_id not in self.profile_by_id: + self.profile_by_id[profile_id] = profile + + def refresh_sync_profiles(self) -> None: + """ + Reload sync-backed profiles from storage into profile_by_id. + Used when the sync collection may have changed externally. + """ + if not self.profile_storage.is_sync_available(): + return + self._prepare_profile_storage() + loaded_sync_profiles = self.profile_storage.list_sync_profiles() + loaded_sync_profile_ids = set(loaded_sync_profiles) + for profile_id, profile in loaded_sync_profiles.items(): + self.profile_by_id[profile_id] = profile + removed_sync_profile_ids = [ + profile_id + for profile_id, profile in list(self.profile_by_id.items()) + if profile.is_sync_backed() and profile_id not in loaded_sync_profile_ids + ] + for profile_id in removed_sync_profile_ids: + self.profile_by_id.pop(profile_id, None) + if self.profile is not None and self.profile.is_sync_backed(): + refreshed_profile = self.profile_by_id.get(self.profile.profile_id) + if refreshed_profile is not None: + self.profile = refreshed_profile def _get_config_without_profile_elements(self) -> dict: filtered_config = copy.deepcopy(self.config) diff --git a/packages/commons/octobot_commons/constants.py b/packages/commons/octobot_commons/constants.py index 650686fcc8..ac13662888 100644 --- a/packages/commons/octobot_commons/constants.py +++ b/packages/commons/octobot_commons/constants.py @@ -76,7 +76,12 @@ def parse_boolean_environment_var(env_key: str, default_value: str) -> bool: # profiles PROFILES_FOLDER = "profiles" +PROFILES_MIGRATED_FOLDER = "profiles_migrated" USER_PROFILES_FOLDER = f"{USER_FOLDER}/{PROFILES_FOLDER}" +USER_PROFILES_MIGRATED_FOLDER = f"{USER_FOLDER}/profiles_migrated" +SYNC_PROFILE_RUNTIME_FOLDER = f"{USER_FOLDER}/sync_profile_runtime" +ENV_OCTOBOT_SYNC_DATA_ROOT = "OCTOBOT_SYNC_DATA_ROOT" +CONFIG_READONLY_PROFILES_PATH = "readonly_profiles_path" PROFILE_CONFIG_FILE = "profile.json" CONFIG_PROFILE = "profile" CONFIG_BACKTESTING_PROFILE = "backtesting_profile" diff --git a/packages/commons/octobot_commons/dsl_interpreter/operators/process_bound_operator_mixin.py b/packages/commons/octobot_commons/dsl_interpreter/operators/process_bound_operator_mixin.py index ec8bb2a242..a3740e5efd 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/operators/process_bound_operator_mixin.py +++ b/packages/commons/octobot_commons/dsl_interpreter/operators/process_bound_operator_mixin.py @@ -17,7 +17,6 @@ import asyncio import pathlib import subprocess -import time import typing import octobot_commons.dsl_interpreter.operator as dsl_interpreter_operator @@ -52,7 +51,10 @@ def request_graceful_stop( raise commons_errors.DSLInterpreterError( "No process id set; cannot request graceful stop." ) - return process_util.request_graceful_stop_via_sigterm(self.pid, logger=logger) + try: + return process_util.request_graceful_stop_via_sigterm(self.pid, logger=logger) + except commons_errors.ProcessError as err: + raise commons_errors.DSLInterpreterError(str(err)) from err async def wait_until_pid_stopped( self, @@ -63,27 +65,15 @@ async def wait_until_pid_stopped( poll_interval: float = 0.2, ) -> None: """Poll until ``pid`` is gone or ``timeout_seconds`` elapses (after e.g. SIGTERM).""" - resolved_logger = logger or commons_logging.get_logger(self.__class__.__name__) - if pid <= 0: - resolved_logger.info( - "wait_until_pid_stopped: pid=%s treated as already stopped (non-positive)", + try: + await process_util.wait_until_pid_stopped_async( pid, + logger=logger or commons_logging.get_logger(self.__class__.__name__), + timeout_seconds=timeout_seconds, + poll_interval=poll_interval, ) - return - resolved_logger.info( - "wait_until_pid_stopped: waiting for pid=%s to exit (timeout=%ss)", - pid, - timeout_seconds, - ) - deadline = time.monotonic() + timeout_seconds - while time.monotonic() < deadline: - if not process_util.pid_is_running(pid): - resolved_logger.info("wait_until_pid_stopped: pid=%s exited", pid) - return - await asyncio.sleep(poll_interval) - raise commons_errors.DSLInterpreterError( - f"Timed out after {timeout_seconds}s waiting for pid={pid} to exit." - ) + except commons_errors.ProcessError as err: + raise commons_errors.DSLInterpreterError(str(err)) from err def spawn_subprocess( self, diff --git a/packages/commons/octobot_commons/enums.py b/packages/commons/octobot_commons/enums.py index 70fcdcfa32..3bffe4b2ca 100644 --- a/packages/commons/octobot_commons/enums.py +++ b/packages/commons/octobot_commons/enums.py @@ -523,6 +523,12 @@ class ProfileType(enum.Enum): BACKTESTING = "backtesting" +class ProfileSource(enum.Enum): + FILESYSTEM = "filesystem" + SYNC = "sync" + EPHEMERAL = "ephemeral" + + class SignalHistoryTypes(enum.Enum): GPT = "gpt" diff --git a/packages/commons/octobot_commons/errors.py b/packages/commons/octobot_commons/errors.py index 9b1254c17a..ff01a254c1 100644 --- a/packages/commons/octobot_commons/errors.py +++ b/packages/commons/octobot_commons/errors.py @@ -193,3 +193,9 @@ class AmbiguousTradedSymbolsTradingTypeError(ValueError): """ Raised when traded symbols map to more than one exchange trading type. """ + + +class ProcessError(Exception): + """ + Raised when a process error occurs + """ diff --git a/packages/commons/octobot_commons/managed_child_process_registry.py b/packages/commons/octobot_commons/managed_child_process_registry.py new file mode 100644 index 0000000000..781dd4e6dd --- /dev/null +++ b/packages/commons/octobot_commons/managed_child_process_registry.py @@ -0,0 +1,135 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import asyncio +import threading +import time +import typing + +import octobot_commons.errors as commons_errors +import octobot_commons.logging as commons_logging +import octobot_commons.process_util as process_util +import octobot_commons.singleton.singleton_class as singleton_class + + +class ManagedChildProcessRegistry(singleton_class.Singleton): + """ + Thread-safe registry of OS pids spawned by spawn_managed_subprocess. + Used at parent shutdown to SIGTERM (then force-kill) child OctoBot processes. + """ + + def __init__(self) -> None: + self._pids: set[int] = set() + self._lock = threading.Lock() + self._logger = commons_logging.get_logger(self.__class__.__name__) + + def register(self, pid: int) -> None: + """Record a managed child pid. No-op for pid <= 0.""" + if pid <= 0: + return + with self._lock: + self._pids.add(pid) + registered_count = len(self._pids) + self._logger.debug( + "Registered managed child pid=%s (count=%s)", + pid, + registered_count, + ) + + def unregister(self, pid: int) -> None: + """Remove a pid from the registry. No-op if not present.""" + with self._lock: + self._pids.discard(pid) + self._logger.debug("Unregistered managed child pid=%s", pid) + + def snapshot_running_pids(self) -> frozenset[int]: + """Prune dead pids, return a point-in-time copy of still-running entries.""" + with self._lock: + dead_pids = { + pid for pid in self._pids if not process_util.pid_is_running(pid) + } + self._pids -= dead_pids + return frozenset(self._pids) + + async def graceful_stop_all( + self, + *, + timeout_seconds: float, + poll_interval: float = 0.2, + ) -> dict[int, str]: + """SIGTERM all snapshot pids, wait, force-kill survivors. Returns per-pid outcome.""" + pids = self.snapshot_running_pids() + if not pids: + return {} + + self._logger.info( + "Graceful stop for %s managed child process(es): %s", + len(pids), + sorted(pids), + ) + + signal_outcomes: dict[int, dict[str, typing.Any]] = {} + for pid in pids: + try: + signal_outcomes[pid] = process_util.request_graceful_stop_via_sigterm( + pid, + logger=self._logger, + ) + except commons_errors.ProcessError as err: + self._logger.warning( + "Failed to send SIGTERM to managed child pid=%s: %s", + pid, + err, + ) + signal_outcomes[pid] = {"status": "failed", "reason": str(err)} + + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if not any(process_util.pid_is_running(pid) for pid in pids): + break + await asyncio.sleep(poll_interval) + + outcomes: dict[int, str] = {} + for pid in pids: + if not process_util.pid_is_running(pid): + sig_status = signal_outcomes.get(pid, {}).get("status", "stopped") + outcomes[pid] = ( + "already_stopped" if sig_status == "already_stopped" else "stopped" + ) + self.unregister(pid) + continue + try: + process_util.request_force_kill(pid, logger=self._logger) + except commons_errors.ProcessError as err: + self._logger.error( + "Force kill failed for managed child pid=%s: %s", + pid, + err, + ) + outcomes[pid] = "failed" + continue + if not process_util.pid_is_running(pid): + outcomes[pid] = "force_killed" + self.unregister(pid) + else: + self._logger.error( + "Managed child pid=%s still running after force kill", + pid, + ) + outcomes[pid] = "failed" + + self._logger.info("Managed child graceful stop outcomes: %s", outcomes) + return outcomes diff --git a/packages/commons/octobot_commons/process_util.py b/packages/commons/octobot_commons/process_util.py index 08b7d61b97..ecee81d2e9 100644 --- a/packages/commons/octobot_commons/process_util.py +++ b/packages/commons/octobot_commons/process_util.py @@ -14,10 +14,12 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. +import asyncio import os import signal import subprocess import sys +import time import typing import octobot_commons.errors as commons_errors @@ -60,7 +62,7 @@ def spawn_managed_subprocess( else: child_stdout = subprocess.DEVNULL child_stderr = subprocess.DEVNULL - return subprocess.Popen( + proc = subprocess.Popen( argv, cwd=working_directory, env=resolved_env, @@ -68,6 +70,9 @@ def spawn_managed_subprocess( stdout=child_stdout, stderr=child_stderr, ) + import octobot_commons.managed_child_process_registry as managed_child_process_registry + managed_child_process_registry.ManagedChildProcessRegistry.instance().register(proc.pid) + return proc def pid_is_running(pid: int) -> bool: # pylint: disable=too-many-return-statements @@ -108,12 +113,12 @@ def request_graceful_stop_via_sigterm( """ resolved_logger = logger or commons_logging.get_logger(__name__) if pid <= 0: - raise commons_errors.DSLInterpreterError( + raise commons_errors.ProcessError( "Invalid pid for graceful stop via SIGTERM." ) sigterm = getattr(signal, "SIGTERM", None) if sigterm is None: - raise commons_errors.DSLInterpreterError( + raise commons_errors.ProcessError( "SIGTERM is not available on this platform." ) if not pid_is_running(pid): @@ -135,8 +140,91 @@ def request_graceful_stop_via_sigterm( resolved_logger.warning( "Graceful stop: failed to signal pid=%s: %s", pid, err ) - raise commons_errors.DSLInterpreterError( + raise commons_errors.ProcessError( f"Failed to send stop signal to pid={pid}: {err}" ) from err resolved_logger.info("Sent graceful stop signal (sigterm) to pid=%s", pid) return {"status": "stopped", "signal": "sigterm"} + + +def request_force_kill( + pid: int, + *, + logger: typing.Optional[typing.Any] = None, +) -> dict[str, typing.Any]: + """Force-kill the process identified by ``pid`` (SIGKILL / TerminateProcess).""" + resolved_logger = logger or commons_logging.get_logger(__name__) + if pid <= 0: + raise commons_errors.ProcessError("Invalid pid for force kill.") + if not pid_is_running(pid): + resolved_logger.info( + "Force kill: pid=%s not running, treating as already stopped", + pid, + ) + return {"status": "already_stopped", "reason": "not_running"} + try: + psutil.Process(pid).kill() + except psutil.NoSuchProcess: + resolved_logger.info( + "Force kill: pid=%s gone before kill", + pid, + ) + return {"status": "already_stopped", "reason": "not_running"} + except Exception as err: + if not pid_is_running(pid): + resolved_logger.info( + "Force kill: pid=%s gone after failed kill: %s", + pid, + err, + ) + return {"status": "already_stopped", "reason": str(err)} + resolved_logger.warning("Force kill failed for pid=%s: %s", pid, err) + raise commons_errors.ProcessError( + f"Failed to force kill pid={pid}: {err}" + ) from err + resolved_logger.info("Force killed pid=%s", pid) + return {"status": "force_killed"} + + +async def wait_until_pid_stopped_async( + pid: int, + *, + logger: typing.Optional[typing.Any] = None, + timeout_seconds: float, + poll_interval: float = 0.2, +) -> None: + """Poll until ``pid`` is gone or ``timeout_seconds`` elapses.""" + resolved_logger = logger or commons_logging.get_logger(__name__) + if pid <= 0: + resolved_logger.info( + "wait_until_pid_stopped_async: pid=%s treated as already stopped (non-positive)", + pid, + ) + return + resolved_logger.info( + "wait_until_pid_stopped_async: waiting for pid=%s to exit (timeout=%ss)", + pid, + timeout_seconds, + ) + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if not pid_is_running(pid): + resolved_logger.info("wait_until_pid_stopped_async: pid=%s exited", pid) + return + await asyncio.sleep(poll_interval) + raise commons_errors.ProcessError( + f"Timed out after {timeout_seconds}s waiting for pid={pid} to exit." + ) + + +async def graceful_stop_managed_children( + *, + timeout_seconds: float, + poll_interval: float = 0.2, +) -> dict[int, str]: + """Gracefully stop all registered managed children (see ManagedChildProcessRegistry).""" + import octobot_commons.managed_child_process_registry as managed_child_process_registry + return await managed_child_process_registry.ManagedChildProcessRegistry.instance().graceful_stop_all( + timeout_seconds=timeout_seconds, + poll_interval=poll_interval, + ) diff --git a/packages/commons/octobot_commons/profiles/__init__.py b/packages/commons/octobot_commons/profiles/__init__.py index 8c5b764d06..023cbafe9d 100644 --- a/packages/commons/octobot_commons/profiles/__init__.py +++ b/packages/commons/octobot_commons/profiles/__init__.py @@ -15,11 +15,8 @@ # License along with this library. -from octobot_commons.profiles import profile - -from octobot_commons.profiles.profile import ( - Profile, -) +from octobot_commons.profiles.profile_types import Profile +from octobot_commons.profiles.profile_types import EphemeralProfile from octobot_commons.profiles import profile_sharing from octobot_commons.profiles.profile_sharing import ( @@ -41,9 +38,15 @@ OptionsData, ) -from octobot_commons.profiles import profile_sync +from octobot_commons.profiles import profile_storage + +from octobot_commons.profiles.profile_storage import ( + ProfileStorage, +) + +from octobot_commons.profiles import profile_synchronizer -from octobot_commons.profiles.profile_sync import ( +from octobot_commons.profiles.profile_synchronizer import ( start_profile_synchronizer, stop_profile_synchronizer, ) @@ -69,6 +72,8 @@ __all__ = [ "Profile", + "EphemeralProfile", + "ProfileStorage", "export_profile", "install_profile", "import_profile", diff --git a/packages/commons/octobot_commons/profiles/backends/__init__.py b/packages/commons/octobot_commons/profiles/backends/__init__.py new file mode 100644 index 0000000000..7221cb5a40 --- /dev/null +++ b/packages/commons/octobot_commons/profiles/backends/__init__.py @@ -0,0 +1,19 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +from octobot_commons.profiles.backends.abstract_profile_backend import AbstractProfileBackend +from octobot_commons.profiles.backends.filesystem_profile_backend import FilesystemProfileBackend +from octobot_commons.profiles.backends.sync_profile_backend import SyncProfileBackend diff --git a/packages/commons/octobot_commons/profiles/backends/abstract_profile_backend.py b/packages/commons/octobot_commons/profiles/backends/abstract_profile_backend.py new file mode 100644 index 0000000000..26d4fea260 --- /dev/null +++ b/packages/commons/octobot_commons/profiles/backends/abstract_profile_backend.py @@ -0,0 +1,93 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import abc +import typing + +import octobot_commons.enums as enums +import octobot_commons.profiles.profile_types.profile as profile_module + + +class AbstractProfileBackend(abc.ABC): + def __init__( + self, + profiles_path: str = None, + profile_schema_path: str = None, + sync_user_id: str = None, + ) -> None: + self._profiles_path = profiles_path + self._profile_schema_path = profile_schema_path + self._sync_user_id = sync_user_id + + def _is_sync_available(self) -> bool: + return bool(self._sync_user_id and str(self._sync_user_id).strip()) + + def _resolve_schema_path(self, schema_path: str = None) -> str: + return schema_path if schema_path is not None else self._profile_schema_path + + @property + @abc.abstractmethod + def source(self) -> enums.ProfileSource: + raise NotImplementedError + + @abc.abstractmethod + def list_profiles( + self, + schema_path: str = None, + ) -> dict[str, profile_module.Profile]: + raise NotImplementedError + + @abc.abstractmethod + def get_profile( + self, + profile_id: str, + schema_path: str = None, + ) -> typing.Optional[profile_module.Profile]: + raise NotImplementedError + + @abc.abstractmethod + def save_profile( + self, + profile: profile_module.Profile, + global_config: dict, + ) -> None: + raise NotImplementedError + + @abc.abstractmethod + def delete_profile( + self, + profile_id: str, + profile: typing.Optional[profile_module.Profile] = None, + ) -> None: + raise NotImplementedError + + @abc.abstractmethod + def list_profile_ids( + self, + ignore: str = None, + schema_path: str = None, + ) -> list[str]: + raise NotImplementedError + + @abc.abstractmethod + def duplicate_profile( + self, + profile: profile_module.Profile, + name: str = None, + description: str = None, + schema_path: str = None, + ) -> profile_module.Profile: + raise NotImplementedError diff --git a/packages/commons/octobot_commons/profiles/backends/filesystem_profile_backend.py b/packages/commons/octobot_commons/profiles/backends/filesystem_profile_backend.py new file mode 100644 index 0000000000..b726b215fb --- /dev/null +++ b/packages/commons/octobot_commons/profiles/backends/filesystem_profile_backend.py @@ -0,0 +1,193 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import os +import shutil +import typing + +import octobot_commons.constants as constants +import octobot_commons.errors as errors +import octobot_commons.enums as enums +import octobot_commons.json_util as json_util +import octobot_commons.logging as commons_logging +import octobot_commons.profiles.backends.abstract_profile_backend as abstract_profile_backend_module +import octobot_commons.profiles.profile_types.profile as profile_module + + +class FilesystemProfileBackend(abstract_profile_backend_module.AbstractProfileBackend): + @property + def source(self) -> enums.ProfileSource: + return enums.ProfileSource.FILESYSTEM + + @staticmethod + def config_file_path(profile_path: str) -> str: + return os.path.join(profile_path, constants.PROFILE_CONFIG_FILE) + + @staticmethod + def tentacles_config_path(profile_path: str) -> str: + return os.path.join(profile_path, constants.CONFIG_TENTACLES_FILE) + + def list_profiles( + self, + schema_path: str = None, + ) -> dict[str, profile_module.Profile]: + resolved_schema_path = self._resolve_schema_path(schema_path) + profiles = {} + for profile in self._scan_profiles(self._profiles_path, resolved_schema_path): + profiles[profile.profile_id] = profile + return profiles + + def get_profile( + self, + profile_id: str, + schema_path: str = None, + ) -> typing.Optional[profile_module.Profile]: + resolved_schema_path = self._resolve_schema_path(schema_path) + for profile in self._scan_profiles(self._profiles_path, resolved_schema_path): + if profile.profile_id == profile_id: + return profile + return None + + def load_profile( + self, + profile_id: str, + schema_path: str = None, + ) -> profile_module.Profile: + profile = self.get_profile(profile_id, schema_path) + if profile is None: + raise errors.NoProfileError(f"No profile with id: {profile_id}") + return profile + + def list_profile_ids( + self, + ignore: str = None, + schema_path: str = None, + ) -> list[str]: + resolved_schema_path = self._resolve_schema_path(schema_path) + return [ + profile.profile_id + for profile in self._scan_profiles( + self._profiles_path, resolved_schema_path, ignore=ignore + ) + ] + + def filesystem_profile_ids(self) -> set[str]: + if not os.path.isdir(self._profiles_path): + return set() + return { + entry + for entry in os.listdir(self._profiles_path) + if os.path.isdir(os.path.join(self._profiles_path, entry)) + } + + def read_profile_from_path( + self, + profile_path: str, + schema_path: str = None, + ) -> profile_module.Profile: + profile = self._load_profile_from_folder(profile_path, schema_path) + if profile is None: + raise errors.ProfileDataError( + f"No profile configuration found at '{profile_path}'" + ) + return profile + + def write_profile_config(self, profile: profile_module.Profile) -> None: + if profile.is_sync_backed(): + raise errors.ProfileDataError( + "FilesystemProfileBackend cannot write sync-backed profiles" + ) + json_util.safe_dump(profile.as_dict(), self.config_file_path(profile.path)) + + def resolve_avatar_path(self, profile: profile_module.Profile) -> None: + if profile.avatar and profile.path: + avatar_path = os.path.join(profile.path, profile.avatar) + if os.path.isfile(avatar_path): + profile.avatar_path = avatar_path + + def save_profile( + self, + profile: profile_module.Profile, + global_config: dict, + ) -> None: + self.write_profile_config(profile) + + def duplicate_profile( + self, + profile: profile_module.Profile, + name: str = None, + description: str = None, + schema_path: str = None, + ) -> profile_module.Profile: + raise NotImplementedError("FilesystemProfileBackend cannot duplicate profiles") + + def delete_profile( + self, + profile_id: str, + profile: typing.Optional[profile_module.Profile] = None, + ) -> None: + if profile is not None and not profile.is_sync_backed(): + shutil.rmtree(profile.path) + return + profile_path = os.path.join(self._profiles_path, profile_id) + if os.path.isdir(profile_path): + shutil.rmtree(profile_path) + + def _scan_profiles( + self, + profiles_path: str, + schema_path: str = None, + ignore: str = None, + ) -> list[profile_module.Profile]: + profiles = [] + if not os.path.isdir(profiles_path): + return profiles + ignored_path = None if ignore is None else os.path.normpath(ignore) + for profile_entry in os.scandir(profiles_path): + if ( + ignored_path is not None + and os.path.normpath(profile_entry.path) == ignored_path + ): + continue + profile = self._load_profile_from_folder(profile_entry.path, schema_path) + if profile is not None: + profiles.append(profile) + return profiles + + def _load_profile_from_folder( + self, + profile_path: str, + schema_path: str = None, + ) -> typing.Optional[profile_module.Profile]: + logger = commons_logging.get_logger("ProfileExplorer") + config_path = self.config_file_path(profile_path) + if not os.path.isfile(config_path): + logger.debug( + f"Ignored {profile_path} as it does not contain a profile configuration" + ) + return None + profile = profile_module.Profile(profile_path, schema_path=schema_path) + try: + profile.from_dict(json_util.read_file(config_path)) + self.resolve_avatar_path(profile) + return profile + except Exception as err: + logger.exception( + err, + True, + f"Ignored profile due to an error upon reading '{profile_path}': {err}", + ) + return None diff --git a/packages/commons/octobot_commons/profiles/backends/sync_profile_backend.py b/packages/commons/octobot_commons/profiles/backends/sync_profile_backend.py new file mode 100644 index 0000000000..79c52cf3de --- /dev/null +++ b/packages/commons/octobot_commons/profiles/backends/sync_profile_backend.py @@ -0,0 +1,350 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import datetime +import os +import shutil +import typing +import uuid + +import octobot_commons.constants as constants +import octobot_commons.errors as errors +import octobot_commons.enums as enums +import octobot_commons.logging as logging +import octobot_commons.profiles.backends.abstract_profile_backend as abstract_profile_backend_module +import octobot_commons.profiles.profile_types.profile as profile_module +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_types.sync_profile as sync_profile_module + + +_LOGGER = None + + +def _get_logger(): + global _LOGGER + if _LOGGER is None: + _LOGGER = logging.get_logger("SyncProfileBackend") + return _LOGGER + + +class SyncProfileBackend(abstract_profile_backend_module.AbstractProfileBackend): + @property + def source(self) -> enums.ProfileSource: + return enums.ProfileSource.SYNC + + def list_profiles( + self, + schema_path: str = None, + ) -> dict[str, profile_module.Profile]: + if not self._is_sync_available(): + return {} + resolved_schema_path = self._resolve_schema_path(schema_path) + profiles = {} + try: + for strategy in self._list_profile_strategies(): + profile = self._strategy_to_profile(strategy, resolved_schema_path) + if profile is not None: + profiles[profile.profile_id] = profile + except Exception as err: + _get_logger().exception( + "Failed to list sync profiles for user %r: %s", + self._sync_user_id, + err, + ) + return {} + return profiles + + def get_profile( + self, + profile_id: str, + schema_path: str = None, + ) -> typing.Optional[profile_module.Profile]: + if not self._is_sync_available(): + return None + strategy = self._get_strategy(profile_id) + if strategy is None: + return None + return self._strategy_to_profile(strategy, self._resolve_schema_path(schema_path)) + + def save_profile( + self, + profile: profile_module.Profile, + global_config: dict, + ) -> None: + if not self._is_sync_available(): + raise errors.ProfileDataError( + "Sync profile save requires a configured wallet user id" + ) + if not profile.is_sync_backed(): + raise errors.ProfileDataError( + "SyncProfileBackend cannot save filesystem profiles" + ) + profile_data = self._profile_to_profile_data(profile, global_config) + self._validate_profile_data(profile_data) + strategy = self._profile_data_to_strategy(profile_data, profile) + strategy_provider = self._get_strategy_provider() + import octobot_sync.sync.collection_backend.errors as collection_errors + + try: + strategy_provider.update_item(self._sync_user_id, strategy) + except collection_errors.ItemNotFoundError: + strategy_provider.create_item(self._sync_user_id, strategy) + sync_profile = typing.cast(sync_profile_module.SyncProfile, profile) + sync_profile.set_profile_data(profile_data) + + def delete_profile( + self, + profile_id: str, + profile: typing.Optional[profile_module.Profile] = None, + ) -> None: + if not self._is_sync_available(): + raise errors.ProfileDataError( + "Sync profile delete requires a configured wallet user id" + ) + self._get_strategy_provider().delete_item(self._sync_user_id, profile_id) + runtime_path = self._runtime_profile_path(profile_id) + if os.path.isdir(runtime_path): + shutil.rmtree(runtime_path) + + def list_profile_ids( + self, + ignore: str = None, + schema_path: str = None, + ) -> list[str]: + if not self._is_sync_available(): + return [] + return [strategy.id for strategy in self._list_profile_strategies()] + + def duplicate_profile( + self, + profile: profile_module.Profile, + name: str = None, + description: str = None, + schema_path: str = None, + ) -> sync_profile_module.SyncProfile: + if not self._is_sync_available(): + raise errors.ProfileDataError( + "Sync profile duplicate requires a configured wallet user id" + ) + if profile.is_sync_backed(): + profile_data = self._profile_to_profile_data( + profile, profile._global_config_from_profile() + ) + else: + profile_data = profile_data_module.ProfileData.from_filesystem_profile(profile) + profile_data.profile_details.id = uuid.uuid4().hex + duplicated_name = name or profile.name + duplicated_description = description if description is not None else profile.description + duplicate = self.import_profile_data( + profile_data, + schema_path=schema_path, + name=duplicated_name, + description=duplicated_description, + risk=profile.risk, + auto_update=False, + slug=profile.slug, + ) + duplicate.read_only = False + duplicate.imported = False + duplicate.origin_url = None + duplicate.description = duplicated_description + return duplicate + + def import_profile_data( + self, + profile_data: profile_data_module.ProfileData, + schema_path: str = None, + name: str = None, + description: str = None, + risk=None, + auto_update: bool = False, + slug: str = None, + force_simulator: bool = False, + ) -> sync_profile_module.SyncProfile: + if not self._is_sync_available(): + raise errors.ProfileDataError( + "Sync profile import requires a configured wallet user id" + ) + resolved_schema_path = self._resolve_schema_path(schema_path) + if profile_data.profile_details.id is None: + profile_data.profile_details.id = uuid.uuid4().hex + if name: + profile_data.profile_details.name = name + profile = self._profile_data_to_runtime_profile( + profile_data, resolved_schema_path + ) + if description is not None: + profile.description = description + if risk is not None: + profile.risk = risk + if slug is not None: + profile.slug = slug + profile.auto_update = auto_update + if force_simulator: + profile.config[constants.CONFIG_TRADER][ + constants.CONFIG_ENABLED_OPTION + ] = False + profile.config[constants.CONFIG_SIMULATOR][ + constants.CONFIG_ENABLED_OPTION + ] = True + self.save_profile(profile, profile._global_config_from_profile()) + return profile + + def _get_strategy_provider(self): + import octobot_sync.sync.collection_providers as collection_providers + + return collection_providers.StrategyProvider.instance() + + def _list_profile_strategies(self): + import octobot_sync.sync.collection_backend.errors as collection_errors + + try: + strategies = self._get_strategy_provider().list_items(self._sync_user_id) + except collection_errors.CollectionNoDataError: + return [] + return [ + strategy + for strategy in strategies + if self._is_profile_strategy(strategy) + ] + + def _get_strategy(self, profile_id: str): + try: + strategy = self._get_strategy_provider().get_item( + self._sync_user_id, profile_id + ) + except Exception: + return None + if not self._is_profile_strategy(strategy): + return None + return strategy + + def _is_profile_strategy(self, strategy) -> bool: + configuration = strategy.configuration + if configuration is None or configuration.actual_instance is None: + return False + import octobot_protocol.models.generic_process_configuration as generic_process_configuration + + return isinstance( + configuration.actual_instance, + generic_process_configuration.GenericProcessConfiguration, + ) and configuration.actual_instance.profile_data is not None + + def _strategy_to_profile( + self, + strategy, + schema_path: str, + ) -> typing.Optional[sync_profile_module.SyncProfile]: + configuration = strategy.configuration.actual_instance + profile_data_dict = configuration.profile_data + if profile_data_dict is None: + return None + profile_data = profile_data_module.ProfileData.from_dict(profile_data_dict) + if profile_data.profile_details.id != strategy.id: + profile_data.profile_details.id = strategy.id + runtime_path = self._runtime_profile_path(strategy.id) + profile = sync_profile_module.SyncProfile( + profile_data, + runtime_path, + schema_path=schema_path, + strategy_version=strategy.version, + ) + if strategy.name: + profile.name = strategy.name + if strategy.description: + profile.description = strategy.description + return profile + + def _profile_data_to_runtime_profile( + self, + profile_data: profile_data_module.ProfileData, + schema_path: str, + ) -> sync_profile_module.SyncProfile: + runtime_path = self._runtime_profile_path( + profile_data.profile_details.id + ) + return sync_profile_module.SyncProfile( + profile_data, runtime_path, schema_path=schema_path + ) + + def _profile_data_to_strategy( + self, + profile_data: profile_data_module.ProfileData, + profile: profile_module.Profile, + ): + import octobot_protocol.models as protocol_models + import octobot_protocol.models.action_configuration_type as action_configuration_type + import octobot_protocol.models.generic_process_configuration as generic_process_configuration + import octobot_protocol.models.strategy_configuration as strategy_configuration + + strategy_version = "1" + if isinstance(profile, sync_profile_module.SyncProfile): + strategy_version = profile.get_strategy_version() + generic_configuration = generic_process_configuration.GenericProcessConfiguration( + configuration_type=action_configuration_type.ActionConfigurationType.GENERIC_PROCESS, + profile_data=profile_data.to_dict(), + ) + now = datetime.datetime.now(datetime.timezone.utc) + return protocol_models.Strategy( + id=profile_data.profile_details.id, + version=strategy_version, + name=profile.name or profile_data.profile_details.name, + description=profile.description, + created_at=now, + updated_at=now, + reference_market=profile_data.trading.reference_market + or constants.DEFAULT_REFERENCE_MARKET, + configuration=strategy_configuration.StrategyConfiguration( + actual_instance=generic_configuration + ), + ) + + def _profile_to_profile_data( + self, + profile: profile_module.Profile, + global_config: dict, + ) -> profile_data_module.ProfileData: + profile_data = profile_data_module.ProfileData.from_profile(profile) + profile_data.profile_details.id = profile.profile_id + profile_data.profile_details.name = profile.name + tentacles_data = profile.get_tentacles_data() + if tentacles_data is not None: + profile_data.tentacles = tentacles_data + elif profile.is_sync_backed(): + sync_profile = typing.cast(sync_profile_module.SyncProfile, profile) + profile_data.tentacles = list(sync_profile.get_profile_data().tentacles) + return profile_data + + def _validate_profile_data( + self, profile_data: profile_data_module.ProfileData + ) -> None: + if not profile_data.profile_details.id: + raise errors.ProfileDataError("profile_data.profile_details.id is required") + if ( + profile_data.profile_details.name is None + or profile_data.profile_details.name == "" + ): + raise errors.ProfileDataError("profile_data.profile_details.name is required") + + def _runtime_profile_path(self, profile_id: str) -> str: + import octobot_commons.user_root_folder_provider as user_root_folder_provider + + user_root = user_root_folder_provider.get_sync_data_root() + return os.path.join( + user_root, + constants.SYNC_PROFILE_RUNTIME_FOLDER, + profile_id, + ) diff --git a/packages/commons/octobot_commons/profiles/profile_data.py b/packages/commons/octobot_commons/profiles/profile_data.py index 8676b557eb..caaf9c530d 100644 --- a/packages/commons/octobot_commons/profiles/profile_data.py +++ b/packages/commons/octobot_commons/profiles/profile_data.py @@ -16,9 +16,10 @@ # License along with this library. import copy import dataclasses +import os import typing -import octobot_commons.profiles.profile as profile_import +import octobot_commons.profiles.profile_types.profile as profile_import import octobot_commons.dataclasses import octobot_commons.constants as constants @@ -154,6 +155,7 @@ class TentaclesData( ): name: typing.Optional[str] = None config: dict = dataclasses.field(default_factory=dict) + activated: bool = True @dataclasses.dataclass @@ -221,13 +223,33 @@ def __post_init__(self): ) @classmethod - def from_profile(cls, profile: profile_import.Profile): + def exchanges_from_profile_config(cls, profile_config: dict) -> list[ExchangeData]: + """ + Build enabled exchanges from a profile config dict. + """ + exchanges_config = profile_config.get(constants.CONFIG_EXCHANGES, {}) + enabled_exchanges = [] + for exchange_name, exchange_details in exchanges_config.items(): + if not exchange_details.get(constants.CONFIG_ENABLED_OPTION, False): + continue + enabled_exchanges.append( + ExchangeData( + internal_name=exchange_name, + exchange_type=exchange_details.get( + constants.CONFIG_EXCHANGE_TYPE, constants.DEFAULT_EXCHANGE_TYPE + ), + ) + ) + return enabled_exchanges + + @classmethod + def from_profile(cls, profile: "profile_import.Profile"): """ Creates a cls instance from the given profile """ profile_dict = profile.as_dict() content = profile_dict[constants.PROFILE_CONFIG] - return cls.from_dict( + profile_data = cls.from_dict( { "profile_details": { "id": profile_dict[constants.CONFIG_PROFILE][constants.CONFIG_ID], @@ -275,14 +297,27 @@ def from_profile(cls, profile: profile_import.Profile): "tentacles": [], } ) + profile_data.exchanges = cls.exchanges_from_profile_config(content) + return profile_data - def to_profile(self, to_create_profile_path: str) -> profile_import.Profile: + @classmethod + def from_filesystem_profile(cls, profile: "profile_import.Profile") -> "ProfileData": """ - Returns a new Profile from self + Build a complete ProfileData from a filesystem profile, including tentacles. """ - profile = profile_import.Profile(to_create_profile_path) - profile.from_dict(self._to_profile_dict()) - return profile + profile_data = cls.from_profile(profile) + profile_data.profile_details.id = profile.profile_id + profile_data.profile_details.name = profile.name + try: + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + except ImportError: + return profile_data + tentacles = profile_tentacles_util.collect_tentacles_data_from_filesystem_profile( + profile + ) + if tentacles is not None: + profile_data.tentacles = tentacles + return profile_data def set_tentacles_config(self, config_by_tentacle: dict): """ diff --git a/packages/commons/octobot_commons/profiles/profile_data_import.py b/packages/commons/octobot_commons/profiles/profile_data_import.py index 24082d07ce..5553910fc9 100644 --- a/packages/commons/octobot_commons/profiles/profile_data_import.py +++ b/packages/commons/octobot_commons/profiles/profile_data_import.py @@ -19,9 +19,9 @@ import uuid import octobot_commons.profiles.profile_data as profile_data_import -import octobot_commons.profiles.profile as profile_import +import octobot_commons.profiles.profile_types.profile as profile_import +import octobot_commons.profiles.backends as profile_backends_module import octobot_commons.logging as bot_logging -import octobot_commons.json_util as json_util import octobot_commons.constants as constants import octobot_commons.aiohttp_util as aiohttp_util import octobot_commons.enums as enums @@ -86,14 +86,21 @@ async def convert_profile_data_to_profile_directory( # when updating profile, keep existing registered tentacles import_registered_tentacles = profile_to_update is not None # tentacles_config.json - tentacles_setup_config = _get_tentacles_setup_config( - profile_data, output_path, import_registered_tentacles - ) - if tentacles_setup_config.save_config(is_config_update=True): - changed = True - # specific_config - if _save_specific_config(profile_data, output_path, bool(profile_to_update)): - changed = True + try: + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + tentacles_setup_config = profile_tentacles_util.build_setup_config_from_profile_data( + profile_data, output_path, import_registered_tentacles + ) + if tentacles_setup_config.save_config(is_config_update=True): + changed = True + # specific_config + if profile_tentacles_util.write_specific_configs_to_profile_folder( + profile_data, output_path, bool(profile_to_update) + ): + changed = True + except ImportError: + raise # avatar file if avatar_url: try: @@ -106,7 +113,11 @@ async def convert_profile_data_to_profile_directory( ) # finish with profile.json to include edits from previous methods if changed: - profile.save() + profile_storage = profile.get_profile_storage() + if profile_storage is not None: + profile.save() + else: + profile_backends_module.FilesystemProfileBackend().write_profile_config(profile) return changed @@ -119,7 +130,7 @@ def _get_profile( slug: str, force_simulator: bool, ): - profile = profile_data.to_profile(output_path) + profile = profile_import.Profile.from_profile_data(profile_data, output_path) if force_simulator: profile.config[constants.CONFIG_TRADER][constants.CONFIG_ENABLED_OPTION] = False profile.config[constants.CONFIG_SIMULATOR][ @@ -146,7 +157,7 @@ def get_updated_profile( :param profile_data: the profile_data to get the update from :return: True if something changed in the updated profile """ - updated_profile = profile_data.to_profile("") + updated_profile = profile_import.Profile.from_profile_data(profile_data, "") changed = False # update traded currencies (add new currencies) origin_currencies = copy.deepcopy( @@ -179,78 +190,6 @@ def get_updated_profile( return changed -def _get_tentacles_setup_config( - profile_data: profile_data_import.ProfileData, - output_path: str, - import_registered_tentacles: bool, -): - try: - import octobot_tentacles_manager.api - import octobot_tentacles_manager.constants - - classes = [ - octobot_tentacles_manager.api.get_tentacle_class_from_string( - tentacle_data.name - ).__name__ - for tentacle_data in profile_data.tentacles - if tentacle_data.name not in ( - octobot_tentacles_manager.constants.IGNORED_TENTACLES_NAMES_IN_TENTACLES_SETUP_CONFIG - ) - ] - config_path = os.path.join(output_path, constants.CONFIG_TENTACLES_FILE) - tentacles_setup_config = ( - octobot_tentacles_manager.api.create_tentacles_setup_config_with_tentacles( - *classes, config_path=config_path - ) - ) - use_reference_registered_tentacles = ( - not tentacles_setup_config.registered_tentacles - ) - octobot_tentacles_manager.api.fill_with_installed_tentacles( - tentacles_setup_config, - import_registered_tentacles=import_registered_tentacles, - use_reference_registered_tentacles=use_reference_registered_tentacles, - ) - return tentacles_setup_config - except ImportError: - raise - - -def _save_specific_config( - profile_data: profile_data_import.ProfileData, - output_path: str, - is_config_update: bool, -) -> bool: - changed = False - try: - import octobot_tentacles_manager.constants - - specific_config_dir = os.path.join( - output_path, - octobot_tentacles_manager.constants.TENTACLES_SPECIFIC_CONFIG_FOLDER, - ) - if not os.path.exists(specific_config_dir): - os.mkdir(specific_config_dir) - for tentacle_config in profile_data.tentacles: - file_path = os.path.join( - specific_config_dir, - f"{tentacle_config.name}{octobot_tentacles_manager.constants.CONFIG_EXT}", - ) - if is_config_update and json_util.has_same_content( - file_path, tentacle_config.config - ): - # nothing to do - continue - changed = True - json_util.safe_dump( - tentacle_config.config, - file_path, - ) - except ImportError: - raise - return changed - - async def _download_and_set_avatar( profile, avatar_url, output_path: str, aiohttp_session ): diff --git a/packages/commons/octobot_commons/profiles/profile_migration.py b/packages/commons/octobot_commons/profiles/profile_migration.py new file mode 100644 index 0000000000..2c0e73dc37 --- /dev/null +++ b/packages/commons/octobot_commons/profiles/profile_migration.py @@ -0,0 +1,69 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +from __future__ import annotations + +import os +import shutil + +import octobot_commons.constants as constants +import octobot_commons.errors as errors +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_storage as profile_storage_module + + +def migrate_user_profiles_to_sync( + profile_storage: profile_storage_module.ProfileStorage, +) -> list[str]: + if not profile_storage.is_sync_available(): + raise errors.ProfileDataError( + "Profile migration requires a configured wallet user id" + ) + migrated_profile_ids = [] + sync_backend = profile_storage._sync_backend + filesystem_ids = profile_storage.filesystem_profile_ids() + for profile_id in list(filesystem_ids): + profile = profile_storage.find_profile(profile_id) + if profile is None or profile.is_sync_backed(): + continue + if profile.read_only and not profile.imported: + continue + profile_data = profile_data_module.ProfileData.from_filesystem_profile(profile) + profile_data.profile_details.id = profile.profile_id + sync_backend.import_profile_data( + profile_data, schema_path=profile_storage.profile_schema_path + ) + _archive_filesystem_profile(profile_storage.profiles_path, profile_id) + migrated_profile_ids.append(profile_id) + return migrated_profile_ids + + +def _archive_filesystem_profile( + profiles_path: str, + profile_id: str, +) -> None: + source_path = os.path.join(profiles_path, profile_id) + if not os.path.isdir(source_path): + return + import octobot_commons.user_root_folder_provider as user_root_folder_provider + + user_root = user_root_folder_provider.get_sync_data_root() + migrated_root = os.path.join(user_root, constants.PROFILES_MIGRATED_FOLDER) + os.makedirs(migrated_root, exist_ok=True) + destination_path = os.path.join(migrated_root, profile_id) + if os.path.exists(destination_path): + shutil.rmtree(destination_path) + shutil.move(source_path, destination_path) diff --git a/packages/commons/octobot_commons/profiles/profile_sharing.py b/packages/commons/octobot_commons/profiles/profile_sharing.py index b978485e3f..0138340933 100644 --- a/packages/commons/octobot_commons/profiles/profile_sharing.py +++ b/packages/commons/octobot_commons/profiles/profile_sharing.py @@ -44,9 +44,11 @@ def __init__(self, *args): else: raise # avoid cyclic import -from octobot_commons.profiles.profile import Profile +from octobot_commons.profiles.profile_types.profile import Profile +import octobot_commons.profiles.backends as profile_backends_module import octobot_commons.profiles.profile_data as profile_data_import import octobot_commons.profiles.profile_data_import as profile_data_importer +import octobot_commons.profiles.profile_storage as profile_storage_module import octobot_commons.user_root_folder_provider as user_root_folder_provider @@ -121,7 +123,10 @@ def install_profile( if not quite: logger.info(f"{action}ing {profile_name} profile.") _import_profile_files(import_path, target_import_path) - profile = Profile(target_import_path, schema_path=profile_schema).read_config() + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + profile = filesystem_backend.read_profile_from_path( + target_import_path, schema_path=profile_schema + ) profile.imported = is_imported profile.origin_url = origin_url _ensure_unique_profile_id(profile) @@ -133,7 +138,7 @@ def install_profile( raise errors.ProfileImportError( f"Invalid imported profile: {err.message} in '{'/'.join(err.absolute_path)}'" ) from err - profile.save() + filesystem_backend.write_profile_config(profile) if not quite: logger.info(f"{action}ed {profile.name} ({profile_name}) profile.") return profile @@ -165,8 +170,6 @@ def import_profile( origin_url=origin_url, profile_schema=profile_schema, ) - if profile.name != temp_profile_name: - profile.rename_folder(_get_unique_profile_folder_from_name(profile), False) return profile @@ -182,6 +185,7 @@ async def import_profile_data_as_profile( logo_url: str = None, auto_update: bool = False, force_simulator: bool = False, + profile_storage=None, ) -> Profile: """ Imports the given ProfileData into the user's profile directory with the "imported_" prefix @@ -198,6 +202,22 @@ async def import_profile_data_as_profile( :param force_simulator: True if trader simulator should be forced in config :return: The created profile """ + + if profile_storage is not None and profile_storage.is_sync_available(): + return await profile_storage.import_profile_data( + profile_data, + profile_schema, + bot_install_path, + name=name, + description=description, + risk=risk, + auto_update=auto_update, + slug=profile_data.profile_details.name, + logo_url=logo_url, + force_simulator=force_simulator, + aiohttp_session=aiohttp_session, + origin_url=origin_url, + ) logger = bot_logging.get_logger("ProfileSharing") import_path = f"{name}-{uuid.uuid4().hex}" try: @@ -423,10 +443,10 @@ def _ensure_unique_profile_id(profile) -> None: :param profile: the installed profile :return: None """ - ids = Profile.get_all_profiles_ids( - pathlib.Path(profile.path).parent, ignore=profile.path - ) + profiles_parent = str(pathlib.Path(profile.path).parent) + profile_storage = profile_storage_module.ProfileStorage(profiles_parent) + existing_ids = set(profile_storage.list_profile_ids(ignore=profile.path)) iteration = 1 - while profile.profile_id in ids and iteration < 100: + while profile.profile_id in existing_ids and iteration < 100: profile.profile_id = str(uuid.uuid4()) iteration += 1 diff --git a/packages/commons/octobot_commons/profiles/profile_storage.py b/packages/commons/octobot_commons/profiles/profile_storage.py new file mode 100644 index 0000000000..e9af4b4f4b --- /dev/null +++ b/packages/commons/octobot_commons/profiles/profile_storage.py @@ -0,0 +1,332 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import os +import typing + +import octobot_commons.authentication as authentication_module +import octobot_commons.enums as enums +import octobot_commons.errors as errors +import octobot_commons.logging as logging +import octobot_commons.profiles.backends as profile_backends_module +import octobot_commons.profiles.profile_types.profile as profile_module +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_migration as profile_migration +import octobot_commons.profiles.profile_sharing as profile_sharing + + +class ProfileStorage: + def __init__( + self, + profiles_path: str, + profile_schema_path: str = None, + filesystem_backend: profile_backends_module.FilesystemProfileBackend = None, + sync_backend: profile_backends_module.SyncProfileBackend = None, + ) -> None: + self._profiles_path = profiles_path + self._profile_schema_path = profile_schema_path + self._sync_user_id: typing.Optional[str] = None + self._readonly_profiles_path: typing.Optional[str] = None + self._readonly_filesystem_backend: typing.Optional[ + profile_backends_module.FilesystemProfileBackend + ] = None + self._filesystem_backend = ( + filesystem_backend + or profile_backends_module.FilesystemProfileBackend( + profiles_path, profile_schema_path + ) + ) + self._sync_backend = sync_backend or profile_backends_module.SyncProfileBackend( + profiles_path, profile_schema_path, sync_user_id=None + ) + + @property + def profiles_path(self) -> str: + return self._profiles_path + + @property + def profile_schema_path(self) -> str: + return self._profile_schema_path + + def configure_sync_user(self, user_id: str) -> None: + try: + authentication_module.Authenticator.instance().get_wallet_by_user_id(user_id) + except Exception as error: + raise errors.ProfileDataError( + f"Unknown sync user id: {user_id}" + ) from error + self._sync_user_id = user_id + self._sync_backend._sync_user_id = user_id + + def bind_process_child_sync_user_id(self, user_id: str) -> None: + if not user_id or not str(user_id).strip(): + raise errors.ProfileDataError("Process child sync user id must be non-empty") + self._sync_user_id = str(user_id) + self._sync_backend._sync_user_id = self._sync_user_id + + def is_master_overlay_profile(self, profile: profile_module.Profile) -> bool: + if not self._readonly_profiles_path or profile.path is None: + return False + normalized_profile_path = os.path.normpath(profile.path) + readonly_prefix = self._readonly_profiles_path + if not readonly_prefix.endswith(os.sep): + readonly_prefix = f"{readonly_prefix}{os.sep}" + return normalized_profile_path == self._readonly_profiles_path or normalized_profile_path.startswith( + readonly_prefix + ) + + def configure_readonly_profiles_path(self, path: str) -> None: + normalized_path = os.path.normpath(path) + self._readonly_profiles_path = normalized_path + self._readonly_filesystem_backend = profile_backends_module.FilesystemProfileBackend( + normalized_path, self._profile_schema_path + ) + + def configure_paths( + self, + profiles_path: str, + profile_schema_path: str = None, + ) -> None: + self._profiles_path = profiles_path + if profile_schema_path is not None: + self._profile_schema_path = profile_schema_path + self._filesystem_backend._profiles_path = profiles_path + self._sync_backend._profiles_path = profiles_path + if profile_schema_path is not None: + self._filesystem_backend._profile_schema_path = profile_schema_path + self._sync_backend._profile_schema_path = profile_schema_path + if self._readonly_filesystem_backend is not None: + self._readonly_filesystem_backend._profile_schema_path = profile_schema_path + + def is_sync_available(self) -> bool: + return bool(self._sync_user_id and str(self._sync_user_id).strip()) + + def load_all_profiles(self) -> dict[str, profile_module.Profile]: + if self._profiles_path is None: + raise errors.ProfileDataError("profiles_path is required to load profiles") + loaded_profiles = self._list_profiles() + for profile in loaded_profiles.values(): + profile.bind_profile_storage(self) + return loaded_profiles + + def list_sync_profiles(self) -> dict[str, profile_module.Profile]: + profiles = self._sync_backend.list_profiles(self._profile_schema_path) + for profile in profiles.values(): + profile.bind_profile_storage(self) + return profiles + + def find_profile( + self, + profile_id: str, + ) -> typing.Optional[profile_module.Profile]: + return self._resolve_profile(profile_id) + + def get_profile( + self, + profile_id: str, + ) -> typing.Optional[profile_module.Profile]: + profile = self.find_profile(profile_id) + if profile is not None: + profile.bind_profile_storage(self) + return profile + + def load_profile_by_id(self, profile_id: str) -> profile_module.Profile: + profile = self.get_profile(profile_id) + if profile is None: + raise errors.NoProfileError(f"No profile with id: {profile_id}") + return profile + + def filesystem_profile_ids(self) -> set[str]: + return self._filesystem_backend.filesystem_profile_ids() + + def list_profile_ids(self, ignore: str = None) -> list[str]: + filesystem_ids = self._filesystem_backend.list_profile_ids(ignore=ignore) + overlay_ids = self._master_overlay_profile_ids(ignore=ignore) + sync_ids = self._sync_backend.list_profile_ids(ignore=ignore) + return list(dict.fromkeys(filesystem_ids + overlay_ids + sync_ids)) + + def duplicate_profile( + self, + profile: profile_module.Profile, + name: str = None, + description: str = None, + ) -> profile_module.Profile: + if not self.is_sync_available(): + raise errors.ProfileDataError( + "Profile duplicate requires a configured wallet user id" + ) + clone = self._sync_backend.duplicate_profile( + profile, + name=name, + description=description, + ) + clone.bind_profile_storage(self) + return clone + + def activate_profile(self, profile: profile_module.Profile) -> None: + profile.bind_profile_storage(self) + profile.init_tentacles_setup_config() + + def save_active_profile( + self, + profile: profile_module.Profile, + global_config: dict, + ) -> None: + if self.is_master_overlay_profile(profile): + raise errors.ProfileDataError( + f"{profile.name} profile is shared from the master and can't be saved" + ) + backend = self._get_backend_for_profile(profile) + logging.get_logger(self.__class__.__name__).info( + f"Saving {profile.name} {profile.__class__.__name__} with " + f"{backend.__class__.__name__}" + ) + backend.save_profile(profile, global_config) + if profile.get_storage_source() == enums.ProfileSource.FILESYSTEM: + tentacles_setup_config = profile.tentacles_setup_config + if tentacles_setup_config is not None: + tentacles_setup_config.save_config(is_config_update=True) + + def delete_profile( + self, + profile_id: str, + profile: profile_module.Profile = None, + ) -> None: + if profile is None: + profile = self.find_profile(profile_id) + if profile is None: + raise errors.ProfileRemovalError(f"Profile {profile_id} not found") + if self.is_master_overlay_profile(profile): + raise errors.ProfileRemovalError( + f"{profile.name} profile is shared from the master and can't be removed" + ) + backend = self._get_backend_for_profile(profile) + if profile.read_only and not profile.imported: + raise errors.ProfileRemovalError(f"{profile.name} profile can't be removed") + backend.delete_profile(profile_id, profile=profile) + + def has_any_profiles(self) -> bool: + if self._profiles_path is None: + return False + return self._has_any_profiles() + + async def import_profile_data( + self, + profile_data: profile_data_module.ProfileData, + profile_schema: str, + bot_install_path: str, + name: str = None, + description: str = None, + risk=None, + auto_update: bool = False, + slug: str = None, + logo_url: str = None, + force_simulator: bool = False, + aiohttp_session=None, + origin_url: str = None, + ) -> profile_module.Profile: + if self.is_sync_available(): + profile = self._sync_backend.import_profile_data( + profile_data, + schema_path=profile_schema, + name=name, + description=description, + risk=risk, + auto_update=auto_update, + slug=slug, + force_simulator=force_simulator, + ) + profile.bind_profile_storage(self) + return profile + return await profile_sharing.import_profile_data_as_profile( + profile_data, + profile_schema, + aiohttp_session, + name=name, + description=description, + risk=risk, + bot_install_path=bot_install_path, + origin_url=origin_url, + logo_url=logo_url, + auto_update=auto_update, + force_simulator=force_simulator, + profile_storage=self, + ) + + def migrate_filesystem_profiles_to_sync(self) -> list[str]: + return profile_migration.migrate_user_profiles_to_sync(self) + + def _should_expose_master_profile(self, profile: profile_module.Profile) -> bool: + return profile.read_only + + def _master_overlay_profiles(self) -> dict[str, profile_module.Profile]: + if self._readonly_filesystem_backend is None: + return {} + overlay_profiles = {} + for profile_id, profile in self._readonly_filesystem_backend.list_profiles().items(): + if self._should_expose_master_profile(profile): + overlay_profiles[profile_id] = profile + return overlay_profiles + + def _master_overlay_profile_ids(self, ignore: str = None) -> list[str]: + overlay_ids = [] + for profile_id in self._master_overlay_profiles(): + if ignore is not None and profile_id == ignore: + continue + overlay_ids.append(profile_id) + return overlay_ids + + def _list_profiles(self) -> dict[str, profile_module.Profile]: + profiles = self._sync_backend.list_profiles() + for profile_id, profile in self._master_overlay_profiles().items(): + if profile_id not in profiles: + profiles[profile_id] = profile + profiles.update(self._filesystem_backend.list_profiles()) + return profiles + + def _resolve_profile( + self, + profile_id: str, + ) -> typing.Optional[profile_module.Profile]: + filesystem_profile = self._filesystem_backend.get_profile(profile_id) + if filesystem_profile is not None: + return filesystem_profile + sync_profile = self._sync_backend.get_profile(profile_id) + if sync_profile is not None: + return sync_profile + if self._readonly_filesystem_backend is None: + return None + master_profile = self._readonly_filesystem_backend.get_profile(profile_id) + if master_profile is not None and self._should_expose_master_profile(master_profile): + return master_profile + return None + + def _ensure_profile_persistable(self, profile: profile_module.Profile) -> None: + if profile.get_storage_source() == enums.ProfileSource.EPHEMERAL: + raise errors.ProfileDataError("Ephemeral profiles cannot be persisted") + + def _get_backend_for_profile( + self, profile: profile_module.Profile + ) -> profile_backends_module.AbstractProfileBackend: + self._ensure_profile_persistable(profile) + if profile.is_sync_backed(): + return self._sync_backend + if self.is_master_overlay_profile(profile) and self._readonly_filesystem_backend is not None: + return self._readonly_filesystem_backend + return self._filesystem_backend + + def _has_any_profiles(self) -> bool: + return bool(self._list_profiles()) diff --git a/packages/commons/octobot_commons/profiles/profile_sync.py b/packages/commons/octobot_commons/profiles/profile_synchronizer.py similarity index 99% rename from packages/commons/octobot_commons/profiles/profile_sync.py rename to packages/commons/octobot_commons/profiles/profile_synchronizer.py index a6d5b73029..e36b737ea8 100644 --- a/packages/commons/octobot_commons/profiles/profile_sync.py +++ b/packages/commons/octobot_commons/profiles/profile_synchronizer.py @@ -88,6 +88,7 @@ def stop(self): self.sync_job.stop() + async def start_profile_synchronizer(current_config, on_profile_change): """ Start the clock synchronization loop if possible on this system diff --git a/packages/commons/octobot_commons/profiles/profile_types/__init__.py b/packages/commons/octobot_commons/profiles/profile_types/__init__.py new file mode 100644 index 0000000000..3c1d933f88 --- /dev/null +++ b/packages/commons/octobot_commons/profiles/profile_types/__init__.py @@ -0,0 +1,29 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +from octobot_commons.profiles.profile_types.profile import Profile +from octobot_commons.profiles.profile_types.profile_data_backed_profile import ( + ProfileDataBackedProfile, +) +from octobot_commons.profiles.profile_types.sync_profile import SyncProfile +from octobot_commons.profiles.profile_types.ephemeral_profile import EphemeralProfile + +__all__ = [ + "Profile", + "ProfileDataBackedProfile", + "SyncProfile", + "EphemeralProfile", +] diff --git a/packages/commons/octobot_commons/profiles/profile_types/ephemeral_profile.py b/packages/commons/octobot_commons/profiles/profile_types/ephemeral_profile.py new file mode 100644 index 0000000000..e61c063045 --- /dev/null +++ b/packages/commons/octobot_commons/profiles/profile_types/ephemeral_profile.py @@ -0,0 +1,61 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +from __future__ import annotations + +import octobot_commons.enums as enums +import octobot_commons.errors as errors +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_types.profile_data_backed_profile as profile_data_backed_profile_module + + +class EphemeralProfile(profile_data_backed_profile_module.ProfileDataBackedProfile): + """ + Short-lived RAM-only profile backed by ProfileData. + """ + + @classmethod + def from_profile_data( + cls, + profile_data: profile_data_module.ProfileData, + schema_path: str = None, + ) -> EphemeralProfile: + return cls(profile_data, schema_path=schema_path) + + def __init__( + self, + profile_data: profile_data_module.ProfileData, + schema_path: str = None, + ): + super().__init__(profile_data, profile_path=None, schema_path=schema_path) + + def is_sync_backed(self) -> bool: + return False + + def get_storage_source(self) -> enums.ProfileSource: + return enums.ProfileSource.EPHEMERAL + + def validate_and_save_config(self) -> None: + raise errors.ProfileDataError("Ephemeral profiles cannot be saved") + + def save(self) -> None: + raise errors.ProfileDataError("Ephemeral profiles cannot be saved") + + def delete(self) -> None: + raise errors.ProfileDataError("Ephemeral profiles cannot be deleted") + + def duplicate(self, name: str = None, description: str = None): + raise errors.ProfileDataError("Ephemeral profiles cannot be duplicated") diff --git a/packages/commons/octobot_commons/profiles/profile.py b/packages/commons/octobot_commons/profiles/profile_types/profile.py similarity index 73% rename from packages/commons/octobot_commons/profiles/profile.py rename to packages/commons/octobot_commons/profiles/profile_types/profile.py index 98a0b693c1..019098fa8e 100644 --- a/packages/commons/octobot_commons/profiles/profile.py +++ b/packages/commons/octobot_commons/profiles/profile_types/profile.py @@ -14,9 +14,11 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +from __future__ import annotations + import copy import os -import shutil +import typing import uuid import octobot_commons.constants as constants import octobot_commons.enums as enums @@ -79,13 +81,39 @@ def __init__(self, profile_path: str, schema_path: str = None): self.extra_backtesting_time_frames = [] self.config: dict = {} + self.tentacles_setup_config = None + self._profile_storage = None - def read_config(self): - """ - Reads a profile from self.path - :return: self - """ - return self.from_dict(json_util.read_file(self.config_file())) + def bind_profile_storage(self, profile_storage) -> None: + self._profile_storage = profile_storage + + def get_profile_storage(self): + return self._profile_storage + + def _require_profile_storage(self): + profile_storage = self._profile_storage + if profile_storage is None: + raise errors.ProfileDataError("ProfileStorage is not bound to this profile") + return profile_storage + + def is_sync_backed(self) -> bool: + return False + + def is_profile_data_tentacle_backed(self) -> bool: + return False + + def get_storage_source(self): + return enums.ProfileSource.FILESYSTEM + + @classmethod + def from_profile_data( + cls, + profile_data, + to_create_profile_path: str, + ) -> Profile: + profile = cls(to_create_profile_path) + profile.from_dict(profile_data._to_profile_dict()) + return profile def from_dict(self, profile_dict: dict): """ @@ -118,10 +146,6 @@ def from_dict(self, profile_dict: dict): constants.CONFIG_EXTRA_BACKTESTING_TIME_FRAMES, [] ) self.config = self.apply_default_values(profile_dict[constants.PROFILE_CONFIG]) - if self.avatar and self.path: - avatar_path = os.path.join(self.path, self.avatar) - if os.path.isfile(avatar_path): - self.avatar_path = avatar_path return self def save_config(self, global_config: dict): @@ -134,7 +158,7 @@ def save_config(self, global_config: dict): if element in global_config: self.config[element] = global_config[element] self.sync_partially_managed_elements(global_config) - self.validate_and_save_config() + self._save_through_profile_storage(global_config) def remove_deleted_elements(self, global_config): """ @@ -177,42 +201,18 @@ def validate(self): def validate_and_save_config(self) -> None: """ - JSON validates this profile and then saves its configuration file + JSON validates this profile and then saves its configuration :return: None """ self.validate() - self.save() + self._save_through_profile_storage(self._global_config_from_profile()) def save(self) -> None: """ - Saves the current profile configuration file + Saves the current profile configuration :return: None """ - json_util.safe_dump(self.as_dict(), self.config_file()) - - def rename_folder(self, new_name, should_raise) -> str: - """ - rename the profile folder - :param new_name: name of the new folder - :param should_raise: raises ProfileConflictError if the profile can't be renamed - :return: the new profile path - """ - new_path = os.path.join(os.path.split(self.path)[0], new_name) - if os.path.exists(new_path): - if should_raise: - raise errors.ProfileConflictError( - "Skipping folder renaming: a profile already exists at this path" - ) - return self.path - try: - os.rename(self.path, new_path) - self.path = new_path - except Exception as err: - commons_logging.get_logger("ProfileRenamer").error( - f"Error when renaming profile: {err}" - ) - raise errors.ProfileConflictError from err - return self.path + self.validate_and_save_config() def duplicate(self, name: str = None, description: str = None): """ @@ -221,25 +221,22 @@ def duplicate(self, name: str = None, description: str = None): :param description: description of the profile to create, uses the original's one by default :return: the created profile """ - clone = copy.deepcopy(self) - clone.name = name or clone.name - clone.description = description or clone.description - clone.profile_id = str(uuid.uuid4()) - clone.read_only = False - clone.imported = False - clone.origin_url = None - clone.auto_update = False - try: - clone.path = os.path.join( - os.path.split(self.path)[0], f"{clone.name}_{clone.profile_id}" - ) - shutil.copytree(self.path, clone.path) - except OSError: - # invalid profile name for a filename - clone.path = os.path.join(os.path.split(self.path)[0], clone.profile_id) - shutil.copytree(self.path, clone.path) - clone.save() - return clone + return self._require_profile_storage().duplicate_profile(self, name=name, description=description) + + def delete(self) -> None: + self._require_profile_storage().delete_profile(self.profile_id, profile=self) + + def _save_through_profile_storage(self, global_config: dict) -> None: + self._require_profile_storage().save_active_profile(self, global_config) + + def _global_config_from_profile(self) -> dict: + global_config = {} + for element in self.FULLY_MANAGED_ELEMENTS: + global_config[element] = self.config.get(element, {}) + for element in self.PARTIALLY_MANAGED_ELEMENTS: + if element in self.config: + global_config[element] = self.config[element] + return global_config def get_tentacles_config_path(self) -> str: """ @@ -247,6 +244,51 @@ def get_tentacles_config_path(self) -> str: """ return os.path.join(self.path, constants.CONFIG_TENTACLES_FILE) + def init_tentacles_setup_config(self) -> None: + setup = self._build_tentacles_setup_config() + self.bind_tentacles_setup_config(setup) + + def bind_tentacles_setup_config(self, tentacles_setup_config): + """ + Link profile and setup config both ways (setup.profile + profile.tentacles_setup_config). + """ + tentacles_setup_config.profile = self + self.tentacles_setup_config = tentacles_setup_config + return tentacles_setup_config + + def get_tentacles_data(self) -> typing.Optional[list]: + tentacles_setup_config = self.tentacles_setup_config + if tentacles_setup_config is None: + return None + try: + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + except ImportError: + return self._get_tentacles_data_without_tentacles_manager() + return profile_tentacles_util.collect_tentacles_data_from_setup( + tentacles_setup_config + ) + + def _get_tentacles_data_without_tentacles_manager(self) -> typing.Optional[list]: + return None + + def _build_tentacles_setup_config(self): + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + os.makedirs(self.path, exist_ok=True) + tentacles_config_path = self.get_tentacles_config_path() + if os.path.isfile(tentacles_config_path): + return profile_tentacles_util.load_setup_config_from_profile_path( + tentacles_config_path + ) + try: + profile_data = profile_data_module.ProfileData.from_filesystem_profile(self) + except (KeyError, OSError, TypeError): + profile_data = profile_data_module.ProfileData() + return profile_tentacles_util.build_setup_config_from_profile_data( + profile_data, self.path, import_registered_tentacles=True + ) + def as_dict(self) -> dict: """ :return: A dict representation of this profile configuration @@ -275,12 +317,6 @@ def as_dict(self) -> dict: constants.PROFILE_CONFIG: self.config, } - def config_file(self): - """ - :return: the path to this profile config file - """ - return os.path.join(self.path, constants.PROFILE_CONFIG_FILE) - def merge_partially_managed_element_into_config(self, config: dict, element: str): """ Merge this profile configuration's partially managed element into the given config @@ -373,71 +409,6 @@ def _filter_fill_elements( if key in allowed_keys: profile_config[element][key] = value - @staticmethod - def load_profile(profiles_path, profile_id, schema_path: str = None): - """ - :param profiles_path: the path to look for the profile - :param profile_id: the required profile id - :return: the loaded profile - """ - for profile in Profile.get_all_profiles(profiles_path, schema_path=schema_path): - if profile.profile_id == profile_id: - return profile - raise errors.NoProfileError(f"No profile with id: {profile_id}") - - @staticmethod - def get_all_profiles(profiles_path, ignore: str = None, schema_path: str = None): - """ - Loads profiles found in the given directory - :param profiles_path: Path to a directory containing profiles - :param ignore: A profile path to ignore - :param schema_path: Path to the json schema to pass to the created profile instances - :return: the profile instances list - """ - profiles = [] - ignored_path = None if ignore is None else os.path.normpath(ignore) - for profile_entry in os.scandir(profiles_path): - if ( - ignored_path is None - or os.path.normpath(profile_entry.path) != ignored_path - ): - profile = Profile._load_profile(profile_entry.path, schema_path) - if profile is not None: - profiles.append(profile) - return profiles - - @staticmethod - def _load_profile(profile_path: str, schema_path: str): - logger = commons_logging.get_logger("ProfileExplorer") - profile = Profile(profile_path, schema_path) - try: - if os.path.isfile(profile.config_file()): - profile.read_config() - return profile - logger.debug( - f"Ignored {profile_path} as it does not contain a profile configuration" - ) - except Exception as err: - logger.exception( - err, - True, - f"Ignored profile due to an error upon reading '{profile_path}': {err}", - ) - return None - - @staticmethod - def get_all_profiles_ids(profiles_path, ignore: str = None): - """ - Get ids of profiles found in the given directory - :param profiles_path: Path to a directory containing profiles - :param ignore: A profile path to ignore in ids listing - :return: the profile ids list - """ - return [ - profile.profile_id - for profile in Profile.get_all_profiles(profiles_path, ignore) - ] - @staticmethod def apply_default_values(config: dict) -> dict: """ diff --git a/packages/commons/octobot_commons/profiles/profile_types/profile_data_backed_profile.py b/packages/commons/octobot_commons/profiles/profile_types/profile_data_backed_profile.py new file mode 100644 index 0000000000..b00e43428a --- /dev/null +++ b/packages/commons/octobot_commons/profiles/profile_types/profile_data_backed_profile.py @@ -0,0 +1,83 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +from __future__ import annotations + +import typing + +import octobot_commons.errors as errors +import octobot_commons.profiles.profile_types.profile as profile_module +import octobot_commons.profiles.profile_data as profile_data_module + + +class ProfileDataBackedProfile(profile_module.Profile): + """ + Profile facade with tentacle config stored in profile_data (RAM only). + """ + + def __init__( + self, + profile_data: profile_data_module.ProfileData, + profile_path: str = None, + schema_path: str = None, + ): + super().__init__(profile_path, schema_path=schema_path) + self._profile_data = profile_data + self.from_dict(profile_data._to_profile_dict()) + if profile_data.profile_details.id: + self.profile_id = profile_data.profile_details.id + if profile_data.profile_details.name: + self.name = profile_data.profile_details.name + + def is_profile_data_tentacle_backed(self) -> bool: + return True + + def get_profile_data(self) -> profile_data_module.ProfileData: + return self._profile_data + + def get_tentacles_config_path(self) -> str: + raise errors.ProfileDataError( + "Profile data backed profiles have no filesystem tentacles config path" + ) + + def _build_tentacles_setup_config(self): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + return profile_tentacles_util.build_setup_config_from_profile_data( + self.get_profile_data(), output_path=None, import_registered_tentacles=False + ) + + def _get_tentacles_data_without_tentacles_manager(self) -> list: + return list(self.get_profile_data().tentacles) + + def get_tentacles_data(self) -> typing.Optional[list]: + tentacles_setup_config = self.tentacles_setup_config + if tentacles_setup_config is None: + return None + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + specific_configs_by_tentacle_name = { + tentacle_data.name: tentacle_data.config or {} + for tentacle_data in self.get_profile_data().tentacles + } + collected_tentacles_data = profile_tentacles_util.collect_tentacles_data_from_setup( + tentacles_setup_config, + specific_configs_by_tentacle_name=specific_configs_by_tentacle_name, + ) + return profile_tentacles_util.merge_inactive_tentacles_data_from_profile( + collected_tentacles_data, + self.get_profile_data(), + ) diff --git a/packages/commons/octobot_commons/profiles/profile_types/sync_profile.py b/packages/commons/octobot_commons/profiles/profile_types/sync_profile.py new file mode 100644 index 0000000000..e8027f3c4f --- /dev/null +++ b/packages/commons/octobot_commons/profiles/profile_types/sync_profile.py @@ -0,0 +1,50 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import octobot_commons.enums as enums +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_types.profile_data_backed_profile as profile_data_backed_profile_module + + +class SyncProfile(profile_data_backed_profile_module.ProfileDataBackedProfile): + """ + Profile facade backed by StrategyProvider profile_data. + """ + + def __init__( + self, + profile_data: profile_data_module.ProfileData, + runtime_path: str, + schema_path: str = None, + strategy_version: str = "1", + ): + super().__init__(profile_data, profile_path=runtime_path, schema_path=schema_path) + self._strategy_version = strategy_version + + def is_sync_backed(self) -> bool: + return True + + def get_storage_source(self) -> enums.ProfileSource: + return enums.ProfileSource.SYNC + + def get_strategy_version(self) -> str: + return self._strategy_version + + def set_profile_data(self, profile_data: profile_data_module.ProfileData) -> None: + self._profile_data = profile_data + self.from_dict(profile_data._to_profile_dict()) + if profile_data.profile_details.id: + self.profile_id = profile_data.profile_details.id diff --git a/packages/commons/octobot_commons/tentacles_management/class_inspector.py b/packages/commons/octobot_commons/tentacles_management/class_inspector.py index 1108225dff..b97dab3555 100644 --- a/packages/commons/octobot_commons/tentacles_management/class_inspector.py +++ b/packages/commons/octobot_commons/tentacles_management/class_inspector.py @@ -99,6 +99,7 @@ def get_class_from_string( module, parent_inspection=default_parent_inspection, error_when_not_found: bool = False, + case_insensitive: bool = False, ): """ Search a class from a class string in a specified module for a specified parent @@ -107,16 +108,22 @@ def get_class_from_string( :param module: the class expected module :param parent_inspection: the parent inspection :param error_when_not_found: if errors should be raised + :param case_insensitive: if True, match class names without case sensitivity :return: the class if found else None """ + def _name_matches(member_name: str) -> bool: + if case_insensitive: + return member_name.lower() == class_string.lower() + return member_name == class_string + if tentacle_class_by_name := { - m[0]: m[1] - for m in inspect.getmembers(module) - if (m[0] == class_string) - and hasattr(m[1], "__bases__") - and parent_inspection(m[1], parent) + member_name: member_class + for member_name, member_class in inspect.getmembers(module) + if _name_matches(member_name) + and hasattr(member_class, "__bases__") + and parent_inspection(member_class, parent) }: - return tentacle_class_by_name[class_string] + return next(iter(tentacle_class_by_name.values())) if error_when_not_found: raise ModuleNotFoundError(f"Cant find {class_string} module") return None # no class found diff --git a/packages/commons/octobot_commons/user_root_folder_provider.py b/packages/commons/octobot_commons/user_root_folder_provider.py index fd659a1d1a..06b005c0b6 100644 --- a/packages/commons/octobot_commons/user_root_folder_provider.py +++ b/packages/commons/octobot_commons/user_root_folder_provider.py @@ -78,6 +78,17 @@ def get_user_root_folder() -> str: return UserRootFolderProvider.instance().get_root() +def get_sync_data_root() -> str: + """ + Master sync data root (StrategyProvider storage, sync profile runtime, master wallets). + Child automation processes set ENV_OCTOBOT_SYNC_DATA_ROOT to the master user/ path. + """ + sync_data_root = os.getenv(commons_constants.ENV_OCTOBOT_SYNC_DATA_ROOT) + if sync_data_root: + return sync_data_root + return get_user_root_folder() + + def get_user_profiles_folder() -> str: """Module-level helper: profiles folder under the user root.""" return UserRootFolderProvider.instance().get_user_profiles_folder() diff --git a/packages/commons/plop/tentacles_config.json b/packages/commons/plop/tentacles_config.json new file mode 100644 index 0000000000..5a1e42d78b --- /dev/null +++ b/packages/commons/plop/tentacles_config.json @@ -0,0 +1,7 @@ +{ + "installation_context": { + "octobot_version": "2.1.1" + }, + "registered_tentacles": {}, + "tentacle_activation": {} +} \ No newline at end of file diff --git a/packages/commons/tests/configuration/conftest.py b/packages/commons/tests/configuration/conftest.py new file mode 100644 index 0000000000..ca70ddde5f --- /dev/null +++ b/packages/commons/tests/configuration/conftest.py @@ -0,0 +1,11 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. + +import pytest + +import octobot_commons.constants as constants + + +@pytest.fixture(autouse=True) +def reset_sync_data_root_env(monkeypatch): + monkeypatch.delenv(constants.ENV_OCTOBOT_SYNC_DATA_ROOT, raising=False) diff --git a/packages/commons/tests/configuration/test_configuration.py b/packages/commons/tests/configuration/test_configuration.py index f18ff01395..0949da808e 100644 --- a/packages/commons/tests/configuration/test_configuration.py +++ b/packages/commons/tests/configuration/test_configuration.py @@ -23,6 +23,7 @@ import octobot_commons.json_util import octobot_commons.configuration as configuration import octobot_commons.profiles as profiles +import octobot_commons.profiles.backends as profile_backends_module import octobot_commons.constants as constants import octobot_commons.tests.test_config as test_config from ..profiles import get_profiles_path @@ -38,6 +39,15 @@ def get_profile_path(): return test_config.TEST_CONFIG_FOLDER +def _load_test_profile(config, profile_path=None): + resolved_profile_path = profile_path or get_profile_path() + loaded_profile = profile_backends_module.FilesystemProfileBackend().read_profile_from_path( + resolved_profile_path + ) + loaded_profile.bind_profile_storage(config.profile_storage) + return loaded_profile + + @pytest.fixture def config(): return configuration.Configuration(get_fake_config_path(), get_profile_path()) @@ -62,16 +72,23 @@ def test_validate(config): def test_read(default_config): - with mock.patch.object(default_config, "load_profiles", mock.Mock()) as load_profiles_mock, \ - mock.patch.object(default_config, "_get_selected_profile", mock.Mock()) as _select_mock, \ - mock.patch.object(default_config, "select_profile", - mock.Mock()) as select_profile_mock: + with mock.patch.object( + default_config, + "load_profiles_if_possible_and_necessary", + mock.Mock(), + ) as load_profiles_mock: default_config.read() assert isinstance(default_config._read_config, dict) assert isinstance(default_config.config, dict) load_profiles_mock.assert_called_once() - _select_mock.assert_called_once() - select_profile_mock.assert_called_once() + with mock.patch.object( + default_config, + "load_profiles_if_possible_and_necessary", + mock.Mock(), + ) as load_profiles_mock: + default_config.read(activate_profile=False) + load_profiles_mock.assert_not_called() + assert default_config.profile is None def test_select_profile(config): @@ -89,8 +106,7 @@ def test_select_profile(config): def test_remove_profile(config): - config.profile = profiles.Profile(get_profile_path(), config.profile_schema_path) - config.profile.read_config() + config.profile = _load_test_profile(config) config.profile.read_only = True config.profile_by_id[config.profile.profile_id] = config.profile # id not in loaded profiles @@ -113,8 +129,7 @@ def test_remove_profile(config): def test_generate_config_from_user_config_and_profile(config): with open(DEFAULT_CONFIG) as config_file: config._read_config = json.load(config_file) - config.profile = profiles.Profile(get_profile_path(), config.profile_schema_path) - config.profile.read_config() + config.profile = _load_test_profile(config) for key in config.profile.FULLY_MANAGED_ELEMENTS: assert key not in config._read_config for key in config.profile.PARTIALLY_MANAGED_ELEMENTS: @@ -128,6 +143,145 @@ def test_generate_config_from_user_config_and_profile(config): assert config.config is not config._read_config +def _align_config_with_profile(config): + config.config = copy.deepcopy(config._read_config) if config._read_config else {} + config._generate_config_from_user_config_and_profile() + + +class TestConfigurationProfileManagedElementsChanged: + def test_returns_false_when_profile_unset(self, config): + config.profile = None + config.config = {"community": {"token": "value"}} + assert config._profile_managed_elements_changed() is False + + def test_returns_false_when_only_community_changed(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + config.config["community"] = {"token": "updated"} + assert config._profile_managed_elements_changed() is False + + def test_returns_true_when_fully_managed_element_changed(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + config.config[constants.CONFIG_CRYPTO_CURRENCIES] = {"Updated": {"pairs": ["ETH/USDT"]}} + assert config._profile_managed_elements_changed() is True + + def test_returns_true_when_partial_exchange_allowed_key_changed(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + exchange_name = next(iter(config.config[constants.CONFIG_EXCHANGES])) + config.config[constants.CONFIG_EXCHANGES][exchange_name][ + constants.CONFIG_ENABLED_OPTION + ] = not config.profile.config[constants.CONFIG_EXCHANGES][exchange_name][ + constants.CONFIG_ENABLED_OPTION + ] + assert config._profile_managed_elements_changed() is True + + def test_returns_false_when_partial_exchange_non_allowed_key_differs(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + exchange_name = next(iter(config.config[constants.CONFIG_EXCHANGES])) + config.config[constants.CONFIG_EXCHANGES][exchange_name][ + constants.CONFIG_EXCHANGE_KEY + ] = "updated-api-key" + assert config._profile_managed_elements_changed() is False + + +class TestConfigurationSaveSkipUnchangedProfile: + def test_skips_profile_save_when_only_non_profile_config_changed(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + config.config["community"] = {"token": "updated"} + with mock.patch( + "octobot_commons.configuration.configuration.config_file_manager.dump", + mock.Mock(), + ), mock.patch.object( + config, + "_get_config_without_profile_elements", + mock.Mock(return_value={}), + ), mock.patch.object( + config.profile, + "save_config", + mock.Mock(), + ) as save_profile_config_mock: + config.save() + save_profile_config_mock.assert_not_called() + + def test_saves_profile_when_managed_elements_changed(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + config.config[constants.CONFIG_CRYPTO_CURRENCIES] = {"Updated": {"pairs": ["ETH/USDT"]}} + with mock.patch( + "octobot_commons.configuration.configuration.config_file_manager.dump", + mock.Mock(), + ), mock.patch.object( + config, + "_get_config_without_profile_elements", + mock.Mock(return_value={}), + ), mock.patch.object( + config.profile, + "save_config", + mock.Mock(), + ) as save_profile_config_mock: + config.save() + save_profile_config_mock.assert_called_once_with(config.config) + + def test_save_profile_true_forces_persist(self, config): + config.profile = _load_test_profile(config) + config.config = {"community": {"token": "updated"}} + with mock.patch( + "octobot_commons.configuration.configuration.config_file_manager.dump", + mock.Mock(), + ), mock.patch.object( + config, + "_get_config_without_profile_elements", + mock.Mock(return_value={}), + ), mock.patch.object( + config.profile, + "save_config", + mock.Mock(), + ) as save_profile_config_mock: + config.save(save_profile=True) + save_profile_config_mock.assert_called_once_with(config.config) + + def test_sync_all_profiles_still_runs_when_active_profile_unchanged(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + config.config["community"] = {"token": "updated"} + with mock.patch( + "octobot_commons.configuration.configuration.config_file_manager.dump", + mock.Mock(), + ), mock.patch.object( + config, + "_get_config_without_profile_elements", + mock.Mock(return_value={}), + ), mock.patch.object( + config.profile, + "save_config", + mock.Mock(), + ), mock.patch.object( + config, + "_sync_other_profiles", + mock.Mock(), + ) as sync_other_profiles_mock: + config.save(sync_all_profiles=True) + sync_other_profiles_mock.assert_called_once_with() + + def test_save(config): save_file = "saved_config.json" config.config_path = save_file @@ -139,13 +293,13 @@ def test_save(config): with open(DEFAULT_CONFIG) as config_file: config._read_config = json.load(config_file) # add profile data - config.profile = profiles.Profile(get_profile_path(), config.profile_schema_path) - config.profile.read_config() + config.profile = _load_test_profile(config) with mock.patch.object(config, "_get_config_without_profile_elements", mock.Mock(return_value=config._read_config)) as _filter_mock, \ - mock.patch.object(config.profile, "save_config", mock.Mock()) as _save_profile_mock: - config.save() + mock.patch.object(config.profile, "save_config", mock.Mock()) as save_profile_config_mock: + config.save(save_profile=True) assert os.path.isfile(save_file) + save_profile_config_mock.assert_called_once_with(config.config) with open(save_file) as config_file: saved_config = json.load(config_file) assert saved_config == config._read_config @@ -178,6 +332,48 @@ def test_get_tentacles_config_path(config): constants.CONFIG_TENTACLES_FILE) +class TestGetActiveTentaclesSetupConfigProfileDataBacked: + def test_returns_in_memory_setup_without_filesystem_path(self, config): + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_commons.profiles.profile_types.ephemeral_profile as ephemeral_profile_module + + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + profile.init_tentacles_setup_config() + config.profile = profile + setup_config = config.get_active_tentacles_setup_config() + assert setup_config is profile.tentacles_setup_config + assert setup_config.profile is profile + + def test_init_setup_when_missing(self, config): + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_commons.profiles.profile_types.ephemeral_profile as ephemeral_profile_module + + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + config.profile = profile + assert profile.tentacles_setup_config is None + setup_config = config.get_active_tentacles_setup_config() + assert setup_config is not None + assert setup_config.profile is profile + + +class TestGetActiveTentaclesSetupConfigFilesystemBacked: + def test_delegates_to_tentacles_manager_api(self, config): + config.profile = profiles.Profile(get_profile_path(), config.profile_schema_path) + expected_setup = mock.Mock() + with mock.patch( + "octobot_tentacles_manager.api.get_tentacles_setup_config", + mock.Mock(return_value=expected_setup), + ) as get_setup_mock: + setup_config = config.get_active_tentacles_setup_config() + get_setup_mock.assert_called_once_with( + config.get_tentacles_config_path(), + profile=config.profile, + ) + assert setup_config is expected_setup + + def test_get_metrics_enabled(config): config.config = {} assert config.get_metrics_enabled() is True @@ -300,7 +496,9 @@ def test_get_selected_profile(config): assert config._get_selected_profile() == "55" # missing profile config._read_config[constants.CONFIG_PROFILE] = "66" - assert config._get_selected_profile() == "default" + with mock.patch.object(config.logger, "warning", mock.Mock()) as warning_mock: + assert config._get_selected_profile() == "default" + warning_mock.assert_called_once() # no default config.profile_by_id.pop("default") config._read_config[constants.CONFIG_PROFILE] = "66" @@ -311,6 +509,57 @@ def test_get_selected_profile(config): assert config._get_selected_profile() == "default" +class TestConfigurationSave: + def test_calls_profile_save_config_with_live_config(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + config.config[constants.CONFIG_CRYPTO_CURRENCIES] = {"Updated": {"pairs": ["ETH/USDT"]}} + with mock.patch( + "octobot_commons.configuration.configuration.config_file_manager.dump", + mock.Mock(), + ), mock.patch.object( + config, + "_get_config_without_profile_elements", + mock.Mock(return_value={}), + ), mock.patch.object( + config.profile, + "save_config", + mock.Mock(), + ) as save_profile_config_mock: + config.save() + save_profile_config_mock.assert_called_once_with(config.config) + + +class TestConfigurationDeferredProfileActivation: + def test_activate_saved_profile_loads_and_selects(self, config): + sync_profile_id = "sync-only-profile-id" + config._read_config = {constants.CONFIG_PROFILE: sync_profile_id} + config.config = copy.deepcopy(config._read_config) + sync_profile = profiles.Profile("sync-path") + sync_profile.profile_id = sync_profile_id + sync_profile.name = "AAAA" + with mock.patch.object(config, "load_profiles", mock.Mock()) as load_profiles_mock, \ + mock.patch.object(config, "_get_selected_profile", mock.Mock(return_value=sync_profile_id)) as get_selected_mock, \ + mock.patch.object(config, "select_profile", mock.Mock()) as select_profile_mock: + config.activate_saved_profile() + load_profiles_mock.assert_called_once() + get_selected_mock.assert_called_once() + select_profile_mock.assert_called_once_with(sync_profile_id) + + def test_read_without_activate_profile_leaves_profile_unset(self, default_config): + with mock.patch.object( + default_config, + "load_profiles_if_possible_and_necessary", + mock.Mock(), + ) as load_mock: + default_config.read(activate_profile=False) + load_mock.assert_not_called() + assert default_config.profile is None + assert default_config.config is not None + + def test_load_profiles(config): config.profiles_path = get_profiles_path() nb_profiles = 1 @@ -322,6 +571,115 @@ def test_load_profiles(config): assert config.profile_by_id["default"] is loaded_profile +class TestConfigurationRefreshSyncProfiles: + def test_no_op_when_sync_unavailable(self, config): + config.profile_by_id = {"sync-profile-id": mock.Mock()} + with mock.patch.object( + config.profile_storage, + "is_sync_available", + mock.Mock(return_value=False), + ), mock.patch.object( + config.profile_storage, + "list_sync_profiles", + mock.Mock(), + ) as list_sync_profiles_mock: + config.refresh_sync_profiles() + list_sync_profiles_mock.assert_not_called() + assert "sync-profile-id" in config.profile_by_id + + def test_adds_new_sync_profile(self, config): + new_sync_profile = mock.Mock() + new_sync_profile.is_sync_backed.return_value = True + config.profile_by_id = {} + config.profile = None + with mock.patch.object( + config.profile_storage, + "is_sync_available", + mock.Mock(return_value=True), + ), mock.patch.object( + config.profile_storage, + "list_sync_profiles", + mock.Mock(return_value={"new-sync-profile-id": new_sync_profile}), + ): + config.refresh_sync_profiles() + assert config.profile_by_id["new-sync-profile-id"] is new_sync_profile + + def test_replaces_updated_sync_profile(self, config): + stale_sync_profile = mock.Mock() + stale_sync_profile.is_sync_backed.return_value = True + refreshed_sync_profile = mock.Mock() + refreshed_sync_profile.is_sync_backed.return_value = True + config.profile_by_id = {"sync-profile-id": stale_sync_profile} + config.profile = None + with mock.patch.object( + config.profile_storage, + "is_sync_available", + mock.Mock(return_value=True), + ), mock.patch.object( + config.profile_storage, + "list_sync_profiles", + mock.Mock(return_value={"sync-profile-id": refreshed_sync_profile}), + ): + config.refresh_sync_profiles() + assert config.profile_by_id["sync-profile-id"] is refreshed_sync_profile + + def test_removes_deleted_sync_profile(self, config): + removed_sync_profile = mock.Mock() + removed_sync_profile.is_sync_backed.return_value = True + config.profile_by_id = {"removed-sync-profile-id": removed_sync_profile} + config.profile = None + with mock.patch.object( + config.profile_storage, + "is_sync_available", + mock.Mock(return_value=True), + ), mock.patch.object( + config.profile_storage, + "list_sync_profiles", + mock.Mock(return_value={}), + ): + config.refresh_sync_profiles() + assert "removed-sync-profile-id" not in config.profile_by_id + + def test_leaves_filesystem_profiles_untouched(self, config): + filesystem_profile = mock.Mock() + filesystem_profile.is_sync_backed.return_value = False + config.profile_by_id = {"filesystem-profile-id": filesystem_profile} + config.profile = None + with mock.patch.object( + config.profile_storage, + "is_sync_available", + mock.Mock(return_value=True), + ), mock.patch.object( + config.profile_storage, + "list_sync_profiles", + mock.Mock(return_value={}), + ): + config.refresh_sync_profiles() + assert config.profile_by_id["filesystem-profile-id"] is filesystem_profile + + def test_updates_active_profile_when_sync_backed(self, config): + stale_active_profile = mock.Mock() + stale_active_profile.is_sync_backed.return_value = True + stale_active_profile.profile_id = "active-sync-profile-id" + refreshed_active_profile = mock.Mock() + refreshed_active_profile.is_sync_backed.return_value = True + config.profile_by_id = {"active-sync-profile-id": stale_active_profile} + config.profile = stale_active_profile + with mock.patch.object( + config.profile_storage, + "is_sync_available", + mock.Mock(return_value=True), + ), mock.patch.object( + config.profile_storage, + "list_sync_profiles", + mock.Mock( + return_value={"active-sync-profile-id": refreshed_active_profile} + ), + ): + config.refresh_sync_profiles() + assert config.profile is refreshed_active_profile + + def test_get_config_without_profile_elements(config): config.profile = profiles.Profile(config.profiles_path) config.config = { @@ -335,3 +693,89 @@ def test_get_config_without_profile_elements(config): "plip": True, next(iter(profiles.Profile.PARTIALLY_MANAGED_ELEMENTS)): "tt" } + + +class TestConfigurationReadonlyProfileOverlay: + def _write_readonly_master_profile( + self, + profile_folder_path: str, + profile_id: str, + ) -> None: + import octobot_commons.json_util as json_util_module + + os.makedirs(profile_folder_path, exist_ok=True) + profile_file = { + constants.CONFIG_PROFILE: { + constants.CONFIG_ID: profile_id, + constants.CONFIG_NAME: "Non-Trading", + constants.CONFIG_READ_ONLY: True, + }, + constants.PROFILE_CONFIG: { + constants.CONFIG_CRYPTO_CURRENCIES: {}, + constants.CONFIG_EXCHANGES: {}, + constants.CONFIG_TRADER: {constants.CONFIG_ENABLED_OPTION: False}, + constants.CONFIG_SIMULATOR: { + constants.CONFIG_ENABLED_OPTION: True, + constants.CONFIG_STARTING_PORTFOLIO: {}, + constants.CONFIG_SIMULATOR_FEES: {}, + }, + constants.CONFIG_TRADING: { + constants.CONFIG_TRADER_REFERENCE_MARKET: constants.DEFAULT_REFERENCE_MARKET, + constants.CONFIG_TRADER_RISK: 1, + }, + constants.CONFIG_DISTRIBUTION: constants.DEFAULT_DISTRIBUTION, + }, + } + json_util_module.safe_dump( + profile_file, + os.path.join(profile_folder_path, constants.PROFILE_CONFIG_FILE), + ) + + def _child_config_with_readonly_overlay( + self, + tmp_path, + ) -> tuple[configuration.Configuration, str]: + child_user_root = tmp_path / "child" / "user" + child_profiles_path = child_user_root / constants.PROFILES_FOLDER + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + self._write_readonly_master_profile( + str(master_profiles_path / "non-trading"), + constants.DEFAULT_PROFILE, + ) + child_user_root.mkdir(parents=True) + config_path = child_user_root / constants.CONFIG_FILE + config_data = { + constants.CONFIG_PROFILE: "non-trading", + constants.CONFIG_READONLY_PROFILES_PATH: str(master_profiles_path), + constants.CONFIG_ACCEPTED_TERMS: True, + } + with open(config_path, "w", encoding="utf-8") as config_file: + json.dump(config_data, config_file) + bot_config = configuration.Configuration( + str(config_path), + str(child_profiles_path), + ) + return bot_config, str(master_profiles_path) + + def test_are_profiles_empty_or_missing_false_with_readonly_overlay(self, tmp_path): + bot_config, _master_profiles_path = self._child_config_with_readonly_overlay(tmp_path) + bot_config.read(should_raise=False, fill_missing_fields=True) + assert bot_config.are_profiles_empty_or_missing() is False + + def test_read_loads_profile_from_readonly_overlay(self, tmp_path): + bot_config, _master_profiles_path = self._child_config_with_readonly_overlay(tmp_path) + bot_config.read(should_raise=False, fill_missing_fields=True) + assert bot_config.profile is not None + assert bot_config.profile.profile_id == constants.DEFAULT_PROFILE + assert bot_config.config[constants.CONFIG_PROFILE] == constants.DEFAULT_PROFILE + + def test_save_skips_master_overlay_profile_persist(self, tmp_path): + bot_config, _master_profiles_path = self._child_config_with_readonly_overlay(tmp_path) + bot_config.read(should_raise=False, fill_missing_fields=True) + with mock.patch.object( + bot_config.profile_storage, + "save_active_profile", + mock.Mock(), + ) as save_active_profile_mock: + bot_config.save() + save_active_profile_mock.assert_not_called() diff --git a/packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py b/packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py index 9eeab0bd12..394d01e904 100644 --- a/packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py +++ b/packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py @@ -19,6 +19,7 @@ import asyncio import sqlite3 import contextlib +import tempfile import octobot_commons.asyncio_tools as asyncio_tools @@ -58,7 +59,8 @@ async def get_temp_empty_database(): async def test_invalid_file(): - file_name = "plop" + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name db = databases.SQLiteDatabase(file_name) try: await db.initialize() diff --git a/packages/commons/tests/dsl_interpreter/operators/test_process_bound_operator_mixin.py b/packages/commons/tests/dsl_interpreter/operators/test_process_bound_operator_mixin.py index 6393161fca..53d13d004e 100644 --- a/packages/commons/tests/dsl_interpreter/operators/test_process_bound_operator_mixin.py +++ b/packages/commons/tests/dsl_interpreter/operators/test_process_bound_operator_mixin.py @@ -76,6 +76,17 @@ def test_delegates_to_process_util(self): assert result == expected stop_mock.assert_called_once_with(99, logger=mock.sentinel.log) + def test_wraps_process_error_as_dsl_interpreter_error(self): + bound = process_bound_operator_mixin.ProcessBoundOperatorMixin() + bound.pid = 99 + with mock.patch.object( + process_util, + "request_graceful_stop_via_sigterm", + side_effect=commons_errors.ProcessError("failed to signal"), + ): + with pytest.raises(commons_errors.DSLInterpreterError, match="failed to signal"): + bound.request_graceful_stop() + @pytest.mark.asyncio class TestWaitUntilPidStopped: @@ -100,6 +111,16 @@ async def test_timeout_raises_dsl_error(self): poll_interval=0.01, ) + async def test_wraps_process_error_as_dsl_interpreter_error(self): + bound = process_bound_operator_mixin.ProcessBoundOperatorMixin() + with mock.patch.object( + process_util, + "wait_until_pid_stopped_async", + side_effect=commons_errors.ProcessError("wait failed"), + ): + with pytest.raises(commons_errors.DSLInterpreterError, match="wait failed"): + await bound.wait_until_pid_stopped(99, timeout_seconds=1.0) + class TestSpawnSubprocess: def test_sets_self_pid_from_child_and_returns_popen(self): diff --git a/packages/commons/tests/profiles/__init__.py b/packages/commons/tests/profiles/__init__.py index 95b67fe133..05a6b50be4 100644 --- a/packages/commons/tests/profiles/__init__.py +++ b/packages/commons/tests/profiles/__init__.py @@ -1,11 +1,6 @@ # Drakkar-Software OctoBot-Commons # Copyright (c) Drakkar-Software, All rights reserved. # -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU @@ -13,27 +8,5 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. -import os - -import pytest -import pathlib -import octobot_commons.profiles as profiles -import octobot_commons.tests.test_config as test_config - - -def get_profile_path(): - return test_config.TEST_CONFIG_FOLDER - - -def get_profiles_path(): - return pathlib.Path(get_profile_path()).parent - - -@pytest.fixture -def profile(): - return profiles.Profile(get_profile_path()) - -@pytest.fixture -def invalid_profile(): - return profiles.Profile(os.path.join(get_profile_path(), "invalid_profile")) +from tests.profiles.conftest import get_profile_path, get_profiles_path diff --git a/packages/commons/tests/profiles/conftest.py b/packages/commons/tests/profiles/conftest.py index 223511bb1e..ecfc40e880 100644 --- a/packages/commons/tests/profiles/conftest.py +++ b/packages/commons/tests/profiles/conftest.py @@ -1,42 +1,49 @@ # Drakkar-Software OctoBot-Commons # Copyright (c) Drakkar-Software, All rights reserved. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. -import pathlib -import shutil + +import os import pytest +import pathlib -import tests.profiles as profiles_tests +import octobot_commons.constants as constants +import octobot_commons.profiles as profiles +import octobot_commons.profiles.backends as profile_backends_module +import octobot_commons.profiles.profile_storage as profile_storage_module +import octobot_commons.tests.test_config as test_config PROFILES_FS_XDIST_GROUP = "profiles_fs" -_EPHEMERAL_PROFILE_DIRECTORIES = ( - "second_profile", - "other_profile", -) - - -@pytest.fixture(autouse=True) -def _clean_ephemeral_test_profile_directories(): - profiles_path = pathlib.Path(profiles_tests.get_profiles_path()) - for directory_name in _EPHEMERAL_PROFILE_DIRECTORIES: - directory_path = profiles_path.joinpath(directory_name) - if directory_path.is_dir(): - shutil.rmtree(directory_path) - yield - for directory_name in _EPHEMERAL_PROFILE_DIRECTORIES: - directory_path = profiles_path.joinpath(directory_name) - if directory_path.is_dir(): - shutil.rmtree(directory_path) + +def get_profile_path(): + return test_config.TEST_CONFIG_FOLDER + + +def get_profiles_path(): + return pathlib.Path(get_profile_path()).parent + + +@pytest.fixture +def profile_storage(tmp_path): + profiles_path = tmp_path / constants.PROFILES_FOLDER + profiles_path.mkdir() + storage = profile_storage_module.ProfileStorage(str(profiles_path), None) + yield storage + + +@pytest.fixture +def profile_storage_for_tests(): + return profile_storage_module.ProfileStorage(str(get_profiles_path()), None) + + +@pytest.fixture +def profile(profile_storage_for_tests): + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + loaded_profile = filesystem_backend.read_profile_from_path(get_profile_path()) + loaded_profile.bind_profile_storage(profile_storage_for_tests) + return loaded_profile + + +@pytest.fixture +def invalid_profile(): + return profiles.Profile(os.path.join(get_profile_path(), "invalid_profile")) diff --git a/packages/commons/tests/profiles/test_ephemeral_profile.py b/packages/commons/tests/profiles/test_ephemeral_profile.py new file mode 100644 index 0000000000..d5e986bda3 --- /dev/null +++ b/packages/commons/tests/profiles/test_ephemeral_profile.py @@ -0,0 +1,56 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. + +import mock +import pytest + +import octobot_commons.enums as enums +import octobot_commons.errors as errors +import octobot_commons.profiles.profile_types.ephemeral_profile as ephemeral_profile_module +import octobot_commons.profiles.profile_data as profile_data_module + + +class TestEphemeralProfileFromProfileData: + def test_from_profile_data_returns_ephemeral_profile(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + assert isinstance(profile, ephemeral_profile_module.EphemeralProfile) + assert profile.path is None + assert profile.get_storage_source() == enums.ProfileSource.EPHEMERAL + assert profile.is_profile_data_tentacle_backed() is True + assert profile.is_sync_backed() is False + + +class TestEphemeralProfileInitTentaclesSetupConfig: + def test_init_tentacles_setup_config_does_not_require_filesystem_path(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + profile.init_tentacles_setup_config() + assert profile.tentacles_setup_config is not None + + +class TestEphemeralProfileBindTentaclesSetupConfig: + def test_sets_both_profile_and_setup_references(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + setup = mock.Mock() + returned_setup = profile.bind_tentacles_setup_config(setup) + assert returned_setup is setup + assert setup.profile is profile + assert profile.tentacles_setup_config is setup + + +class TestEphemeralProfileGetTentaclesConfigPath: + def test_raises_profile_data_error(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + with pytest.raises(errors.ProfileDataError): + profile.get_tentacles_config_path() + + +class TestEphemeralProfileSave: + def test_save_raises_profile_data_error(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + with pytest.raises(errors.ProfileDataError): + profile.save() diff --git a/packages/commons/tests/profiles/test_filesystem_profile_backend.py b/packages/commons/tests/profiles/test_filesystem_profile_backend.py new file mode 100644 index 0000000000..8577263003 --- /dev/null +++ b/packages/commons/tests/profiles/test_filesystem_profile_backend.py @@ -0,0 +1,122 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. + +import json +import os + +import mock +import pytest + +import octobot_commons.constants as constants +import octobot_commons.errors as errors +import octobot_commons.json_util as json_util +import octobot_commons.profiles.backends as profile_backends_module +import octobot_commons.tests.test_config as test_config + +import tests.profiles.conftest as profiles_conftest +from tests.profiles import get_profile_path, get_profiles_path + + +class TestFilesystemProfileBackendReadWrite: + def test_read_profile_from_path(self): + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + profile = filesystem_backend.read_profile_from_path(get_profile_path()) + assert profile.profile_id == "default" + assert profile.name == "default" + assert profile.description == "OctoBot default profile." + assert profile.avatar == "default_profile.png" + assert profile.avatar_path == os.path.join( + test_config.TEST_CONFIG_FOLDER, "default_profile.png" + ) + assert profile.origin_url == "https://default.url" + assert profile.config[constants.CONFIG_DISTRIBUTION] == constants.DEFAULT_DISTRIBUTION + assert len(profile.config) == 6 + + def test_read_profile_from_path_raises_when_missing(self): + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + with pytest.raises(errors.ProfileDataError): + filesystem_backend.read_profile_from_path("") + + def test_write_profile_config(self): + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + profile = filesystem_backend.read_profile_from_path(get_profile_path()) + save_file = "profile_config.json" + if os.path.isfile(save_file): + os.remove(save_file) + try: + profile.config = {"a": 1} + with mock.patch.object( + filesystem_backend, + "config_file_path", + mock.Mock(return_value=save_file), + ): + filesystem_backend.write_profile_config(profile) + with open(save_file) as config_file: + saved_profile = json.load(config_file) + assert saved_profile == profile.as_dict() + finally: + if os.path.isfile(save_file): + os.remove(save_file) + + def test_config_file_path(self): + assert profile_backends_module.FilesystemProfileBackend.config_file_path( + get_profile_path() + ) == os.path.join(get_profile_path(), constants.PROFILE_CONFIG_FILE) + + +class TestFilesystemProfileBackendDiscovery: + def test_scan_profiles(self): + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + with mock.patch.object( + filesystem_backend, + "_load_profile_from_folder", + mock.Mock(), + ) as load_profile_mock: + nb_files = len(os.listdir(get_profiles_path())) + assert nb_files > 1 + filesystem_backend._scan_profiles(str(get_profiles_path())) + assert load_profile_mock.call_count == nb_files + + def test_load_profile_from_folder(self): + schema_path = "schema_path" + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + with mock.patch.object( + json_util, + "read_file", + mock.Mock(return_value={ + constants.CONFIG_PROFILE: {}, + constants.PROFILE_CONFIG: {}, + }), + ) as read_file_mock: + profile = filesystem_backend._load_profile_from_folder( + test_config.TEST_CONFIG_FOLDER, schema_path + ) + assert profile.path == test_config.TEST_CONFIG_FOLDER + assert profile.schema_path == schema_path + read_file_mock.assert_called_once() + + def test_load_profile(self): + profiles_path = str(get_profiles_path()) + filesystem_backend = profile_backends_module.FilesystemProfileBackend( + profiles_path, None + ) + profile = filesystem_backend.load_profile("default") + assert profile.profile_id == "default" + + @pytest.mark.xdist_group(name=profiles_conftest.PROFILES_FS_XDIST_GROUP) + def test_list_profile_ids(self, profile): + profiles_path = str(get_profiles_path()) + filesystem_backend = profile_backends_module.FilesystemProfileBackend( + profiles_path, None + ) + assert filesystem_backend.list_profile_ids() == ["default"] + assert filesystem_backend.list_profile_ids(ignore=profile.path) == [] + + +class TestFilesystemProfileBackendMutations: + def test_duplicate_profile_raises_not_implemented(self, profile): + filesystem_backend = profile_backends_module.FilesystemProfileBackend( + str(get_profiles_path()), None + ) + with pytest.raises(NotImplementedError): + filesystem_backend.duplicate_profile(profile) diff --git a/packages/commons/tests/profiles/test_profile.py b/packages/commons/tests/profiles/test_profile.py index 9b88eddc05..5a1d12be1b 100644 --- a/packages/commons/tests/profiles/test_profile.py +++ b/packages/commons/tests/profiles/test_profile.py @@ -13,56 +13,32 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. -import os import copy -import json -import shutil import pytest import mock import octobot_commons.json_util import octobot_commons.profiles as profiles import octobot_commons.constants as constants import octobot_commons.enums as enums +import octobot_commons.errors as errors import octobot_commons.tests.test_config as test_config -import tests.profiles.conftest as profiles_conftest -from tests.profiles import profile, get_profile_path, get_profiles_path - - -def test_read_config(profile): - save_ref = profile - assert profile.read_config() is save_ref - assert profile.profile_id == "default" - assert profile.name == "default" - assert profile.description == "OctoBot default profile." - assert profile.avatar == "default_profile.png" - assert profile.avatar_path == os.path.join(test_config.TEST_CONFIG_FOLDER, "default_profile.png") - assert profile.origin_url == "https://default.url" - # default value: distribution is not in profile config - assert profile.config[constants.CONFIG_DISTRIBUTION] == constants.DEFAULT_DISTRIBUTION - assert len(profile.config) == 6 - assert isinstance(profile.config, dict) - - profile.path = "" - with pytest.raises(FileNotFoundError): - profile.read_config() +from tests.profiles import get_profile_path def test_save_config(profile): - with mock.patch.object(profile, "validate_and_save_config", mock.Mock()) as validate_and_save_config_mock, \ + with mock.patch.object(profile, "_save_through_profile_storage", mock.Mock()) as save_mock, \ mock.patch.object(profile, "_filter_fill_elements", mock.Mock()) as _filter_fill_elements_mock: profile.config = {} - # nothing to operate on global_config = {} profile.save_config(global_config) assert profile.config == {} - validate_and_save_config_mock.assert_called_once() + save_mock.assert_called_once_with(global_config) _filter_fill_elements_mock.assert_not_called() - validate_and_save_config_mock.reset_mock() + save_mock.reset_mock() _filter_fill_elements_mock.reset_mock() profile.config = {} - # things in config global_config = { profile.FULLY_MANAGED_ELEMENTS[0]: "plop", profile.FULLY_MANAGED_ELEMENTS[1]: "plip", @@ -73,13 +49,12 @@ def test_save_config(profile): profile.FULLY_MANAGED_ELEMENTS[0]: "plop", profile.FULLY_MANAGED_ELEMENTS[1]: "plip" } - validate_and_save_config_mock.assert_called_once() + save_mock.assert_called_once_with(global_config) _filter_fill_elements_mock.assert_not_called() - validate_and_save_config_mock.reset_mock() + save_mock.reset_mock() _filter_fill_elements_mock.reset_mock() profile.config = {} - # things in config global_config = { profile.FULLY_MANAGED_ELEMENTS[0]: "plop", profile.FULLY_MANAGED_ELEMENTS[1]: "plip", @@ -91,7 +66,7 @@ def test_save_config(profile): profile.FULLY_MANAGED_ELEMENTS[0]: "plop", profile.FULLY_MANAGED_ELEMENTS[1]: "plip", } - validate_and_save_config_mock.assert_called_once() + save_mock.assert_called_once_with(global_config) _filter_fill_elements_mock.assert_called_once_with(global_config, profile.config, next(iter(profile.PARTIALLY_MANAGED_ELEMENTS)), @@ -107,61 +82,45 @@ def test_validate(profile): def test_validate_and_save_config(profile): - save_file = "profile_config.json" with mock.patch.object(profile, "validate", mock.Mock()) as validate_mock, \ - mock.patch.object(profile, "config_file", mock.Mock(return_value=save_file)), \ - mock.patch.object(profile, "save", mock.Mock()) as save_mock: + mock.patch.object(profile, "_save_through_profile_storage", mock.Mock()) as save_mock: profile.validate_and_save_config() validate_mock.assert_called_once() save_mock.assert_called_once() def test_save(profile): - save_file = "profile_config.json" - if os.path.isfile(save_file): - os.remove(save_file) - try: - profile.read_config() - with mock.patch.object(profile, "config_file", mock.Mock(return_value=save_file)): - profile.save() - with open(save_file) as config_file: - saved_profile = json.load(config_file) - assert saved_profile == profile.as_dict() - finally: - if os.path.isfile(save_file): - os.remove(save_file) + with mock.patch.object(profile, "validate_and_save_config", mock.Mock()) as validate_and_save_mock: + profile.save() + validate_and_save_mock.assert_called_once() + + +def test_save_requires_profile_storage(): + unbound_profile = profiles.Profile(get_profile_path()) + with pytest.raises(errors.ProfileDataError): + unbound_profile.save() def test_duplicate(profile): - with mock.patch.object(shutil, "copytree", mock.Mock()) as copytree_mock, \ - mock.patch.object(profiles.Profile, "save", mock.Mock()) as save_mock: + clone = mock.Mock() + with mock.patch.object( + profile.get_profile_storage(), + "duplicate_profile", + mock.Mock(return_value=clone), + ) as duplicate_mock: profile.read_only = True profile.imported = True profile.origin_url = "hello" - clone = profile.duplicate() - assert clone.name == profile.name - assert clone.description == profile.description - assert clone.profile_id != profile.description - assert clone.path != profile.path - assert clone.profile_id in clone.path - assert clone.profile_id is not None - # duplicates are not read_only - assert clone.read_only is False - # duplicates are never imported nor have an origin url - assert clone.imported is False - assert clone.origin_url is None - copytree_mock.assert_called_with(profile.path, clone.path) - save_mock.assert_called_once() + assert profile.duplicate() is clone + duplicate_mock.assert_called_once_with(profile, name=None, description=None) - clone = profile.duplicate(name="123", description="456") - assert clone.name == "123" - assert clone.name != profile.name - assert clone.description == "456" - assert clone.description != profile.description + profile.duplicate(name="123", description="456") + duplicate_mock.assert_called_with(profile, name="123", description="456") def test_as_dict(profile): - assert profile.as_dict() == { + empty_profile = profiles.Profile(get_profile_path()) + assert empty_profile.as_dict() == { constants.CONFIG_PROFILE: { constants.CONFIG_ID: None, constants.CONFIG_NAME: None, @@ -180,8 +139,6 @@ def test_as_dict(profile): }, constants.PROFILE_CONFIG: {}, } - profile.read_config() - # do not test read config profile.config = {"a": 1} profile.imported = True profile.complexity = enums.ProfileComplexity.DIFFICULT @@ -213,10 +170,6 @@ def test_as_dict(profile): } -def test_config_file(profile): - assert profile.config_file() == os.path.join(get_profile_path(), constants.PROFILE_CONFIG_FILE) - - def test_merge_partially_managed_element_into_config(profile): with mock.patch.object(profiles.Profile, "_merge_partially_managed_element", mock.Mock()) as _merge_mock: config = {} @@ -228,7 +181,6 @@ def test_merge_partially_managed_element_into_config(profile): def test_merge_partially_managed_element(profile): - profile.read_config() element = next(iter(profile.PARTIALLY_MANAGED_ELEMENTS)) template = profile.PARTIALLY_MANAGED_ELEMENTS[element] config = { @@ -239,7 +191,6 @@ def test_merge_partially_managed_element(profile): } } } - # add constants.CONFIG_ENABLED_OPTION profile._merge_partially_managed_element(config, profile.config, element, template) assert config == { constants.CONFIG_EXCHANGES: { @@ -254,7 +205,6 @@ def test_merge_partially_managed_element(profile): constants.CONFIG_EXCHANGES: {} } profile.config[constants.CONFIG_EXCHANGES]["binance"][constants.CONFIG_ENABLED_OPTION] = False - # add whole exchange profile._merge_partially_managed_element(config, profile.config, element, template) assert config == { constants.CONFIG_EXCHANGES: { @@ -268,7 +218,6 @@ def test_merge_partially_managed_element(profile): } } config = {} - # add whole exchange and exchanges key with 2 exchanges in profile profile.config[constants.CONFIG_EXCHANGES]["kucoin"] = { constants.CONFIG_ENABLED_OPTION: True, constants.CONFIG_EXCHANGE_TYPE: constants.CONFIG_EXCHANGE_FUTURE @@ -301,7 +250,6 @@ def test_merge_partially_managed_element(profile): } } } - # add constants.CONFIG_ENABLED_OPTION with 2 exchanges in profile, update constants.CONFIG_ENABLED_OPTION profile._merge_partially_managed_element(config, profile.config, element, template) assert config == { constants.CONFIG_EXCHANGES: { @@ -322,7 +270,6 @@ def test_merge_partially_managed_element(profile): def test_remove_deleted_elements(profile): - profile.read_config() element = next(iter(profile.PARTIALLY_MANAGED_ELEMENTS)) config = { constants.CONFIG_EXCHANGES: { @@ -335,13 +282,11 @@ def test_remove_deleted_elements(profile): } before_sync_elements_count = len(profile.config[element]) profile.remove_deleted_elements(config) - # did not remove any element assert before_sync_elements_count == len(profile.config[element]) profile.config[element]["plop"] = config[constants.CONFIG_EXCHANGES]["binance"] assert len(profile.config[element]) == before_sync_elements_count + 1 profile.remove_deleted_elements(config) assert before_sync_elements_count == len(profile.config[element]) - # removed "plop" element assert list(profile.config[element]) == ["binance"] @@ -356,7 +301,6 @@ def test_get_element_from_template(profile): def test_filter_fill_elements(profile): - profile.read_config() config = { constants.CONFIG_EXCHANGES: { "binance": { @@ -375,26 +319,3 @@ def test_filter_fill_elements(profile): constants.CONFIG_ENABLED_OPTION: True } } - - -def test_get_all_profiles(): - with mock.patch.object(profiles.Profile, "_load_profile", mock.Mock()) as _load_profile_mock: - nb_files = len(os.listdir(get_profiles_path())) - assert nb_files > 1 - profiles.Profile.get_all_profiles(get_profiles_path()) - assert _load_profile_mock.call_count == nb_files - - -def test_load_profile(): - schema_path = "schema_path" - with mock.patch.object(profiles.Profile, "read_config", mock.Mock()) as read_config_mock: - profile = profiles.Profile._load_profile(test_config.TEST_CONFIG_FOLDER, schema_path) - assert profile.path == test_config.TEST_CONFIG_FOLDER - assert profile.schema_path == schema_path - read_config_mock.assert_called_once() - - -@pytest.mark.xdist_group(name=profiles_conftest.PROFILES_FS_XDIST_GROUP) -def test_get_existing_profiles_ids(profile): - assert profiles.Profile.get_all_profiles_ids(get_profiles_path()) == ["default"] - assert profiles.Profile.get_all_profiles_ids(get_profiles_path(), ignore=profile.path) == [] diff --git a/packages/commons/tests/profiles/test_profile_data.py b/packages/commons/tests/profiles/test_profile_data.py index c80e734edf..f76b0e327d 100644 --- a/packages/commons/tests/profiles/test_profile_data.py +++ b/packages/commons/tests/profiles/test_profile_data.py @@ -22,7 +22,7 @@ import octobot_commons.constants as constants import octobot_commons.enums as enums -from tests.profiles import get_profile_path, profile +from tests.profiles import get_profile_path @pytest.fixture @@ -104,6 +104,7 @@ def profile_data_dict(): { 'name': 'plopEvaluator', 'config': {}, + 'activated': True, }, { 'name': 'plopEvaluator', @@ -114,6 +115,7 @@ def profile_data_dict(): 'n': None, } }, + 'activated': True, }, ], 'options': { 'values': { @@ -176,12 +178,14 @@ def min_profile_data_dict(): def test_from_profile(profile): - profile_data = profiles.ProfileData.from_profile(profile.read_config()) + profile_data = profiles.ProfileData.from_profile(profile) # check one element per attribute to be sure it's all parsed assert profile_data.distribution == "default" assert profile_data.profile_details.name == "default" assert profile_data.crypto_currencies[0].trading_pairs == ['BTC/USDT'] - assert profile_data.exchanges == [] + assert len(profile_data.exchanges) == 1 + assert profile_data.exchanges[0].internal_name == "binance" + assert profile_data.exchanges[0].exchange_type == constants.DEFAULT_EXCHANGE_TYPE assert profile_data.trader.enabled is False assert profile_data.trader_simulator.enabled is True assert profile_data.trader_simulator.starting_portfolio == {'BTC': 10, 'USDT': 1000} @@ -189,14 +193,48 @@ def test_from_profile(profile): assert profile_data.tentacles == [] -def test_to_profile(profile): - profile_data = profiles.ProfileData.from_profile(profile.read_config()) - created_profile = profile_data.to_profile("plop_path") +class TestProfileDataExchangesFromProfileConfig: + def test_maps_only_enabled_exchanges(self): + profile_config = { + constants.CONFIG_EXCHANGES: { + "binance": { + constants.CONFIG_ENABLED_OPTION: True, + constants.CONFIG_EXCHANGE_TYPE: "spot", + }, + "kucoin": { + constants.CONFIG_ENABLED_OPTION: False, + constants.CONFIG_EXCHANGE_TYPE: "spot", + }, + } + } + exchanges = profiles.ProfileData.exchanges_from_profile_config(profile_config) + assert len(exchanges) == 1 + assert exchanges[0].internal_name == "binance" + assert exchanges[0].exchange_type == "spot" + + def test_uses_default_exchange_type_when_missing(self): + profile_config = { + constants.CONFIG_EXCHANGES: { + "binance": { + constants.CONFIG_ENABLED_OPTION: True, + }, + } + } + exchanges = profiles.ProfileData.exchanges_from_profile_config(profile_config) + assert exchanges[0].exchange_type == constants.DEFAULT_EXCHANGE_TYPE + + +def test_from_profile_data(profile): + profile_data = profiles.ProfileData.from_profile(profile) + created_profile = profiles.Profile.from_profile_data(profile_data, "plop_path") # force missing values for crypto_data in profile.config[constants.CONFIG_CRYPTO_CURRENCIES].values(): crypto_data[constants.CONFIG_ENABLED_OPTION] = crypto_data.get(constants.CONFIG_ENABLED_OPTION, True) + for exchange_data in profile.config[constants.CONFIG_EXCHANGES].values(): + exchange_data[constants.CONFIG_EXCHANGE_TYPE] = exchange_data.get( + constants.CONFIG_EXCHANGE_TYPE, constants.DEFAULT_EXCHANGE_TYPE + ) # remove not stored values - profile.config[constants.CONFIG_EXCHANGES] = {} profile.avatar = profile.description = "" profile.complexity = enums.ProfileComplexity.MEDIUM profile.risk = enums.ProfileRisk.MODERATE diff --git a/packages/commons/tests/profiles/test_profile_data_backed_profile.py b/packages/commons/tests/profiles/test_profile_data_backed_profile.py new file mode 100644 index 0000000000..2773ba5421 --- /dev/null +++ b/packages/commons/tests/profiles/test_profile_data_backed_profile.py @@ -0,0 +1,50 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. + +import mock + +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_types.profile_data_backed_profile as profile_data_backed_profile_module + + +class TestProfileDataBackedProfileGetTentaclesData: + def test_merges_inactive_tentacle_configs_on_save(self): + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData( + name="GridTradingMode", + config={"flat_spread": 2}, + activated=False, + ), + ] + profile = profile_data_backed_profile_module.ProfileDataBackedProfile( + profile_data, profile_path="/tmp/profile" + ) + tentacles_setup_config = mock.Mock() + profile.tentacles_setup_config = tentacles_setup_config + collected = [ + profile_data_module.TentaclesData( + name="IndexTradingMode", + config={"refresh_interval": 0}, + activated=True, + ), + ] + + with mock.patch( + "octobot_tentacles_manager.configuration.profile_tentacles_util.collect_tentacles_data_from_setup", + mock.Mock(return_value=collected), + ), mock.patch( + "octobot_tentacles_manager.configuration.profile_tentacles_util.merge_inactive_tentacles_data_from_profile", + mock.Mock( + return_value=[ + collected[0], + profile_data.tentacles[0], + ] + ), + ) as merge_mock: + result = profile.get_tentacles_data() + + merge_mock.assert_called_once_with(collected, profile_data) + assert len(result) == 2 + assert result[1].name == "GridTradingMode" + assert result[1].activated is False diff --git a/packages/commons/tests/profiles/test_profile_sharing.py b/packages/commons/tests/profiles/test_profile_sharing.py index bae651c36b..828f0c497b 100644 --- a/packages/commons/tests/profiles/test_profile_sharing.py +++ b/packages/commons/tests/profiles/test_profile_sharing.py @@ -26,17 +26,23 @@ import octobot_commons.user_root_folder_provider as user_root_folder_provider import octobot_commons.errors as commons_errors import octobot_commons.profiles as profiles +import octobot_commons.profiles.backends as profile_backends_module import octobot_commons.profiles.profile_sharing as profile_sharing +import octobot_commons.profiles.profile_storage as profile_storage_module from octobot_commons.profiles.profile_sharing import _get_unique_profile_folder, _ensure_unique_profile_id, \ _get_profile_name import octobot_commons.tests.test_config as test_config import tests.profiles.conftest as profiles_conftest -from tests.profiles import profile, get_profile_path, invalid_profile +from tests.profiles import get_profile_path pytestmark = pytest.mark.xdist_group(name=profiles_conftest.PROFILES_FS_XDIST_GROUP) +def _profile_config_file(profile): + return profile_backends_module.FilesystemProfileBackend.config_file_path(profile.path) + + def test_export_profile(profile): export_path = "exported" exported_file = f"{export_path}.zip" @@ -49,10 +55,10 @@ def test_export_profile(profile): dir1=spec_tentacles_config, dir2=other_profile): # create fake tentacles config - shutil.copy(profile.config_file(), tentacles_config) + shutil.copy(_profile_config_file(profile), tentacles_config) os.mkdir(spec_tentacles_config) - shutil.copy(profile.config_file(), os.path.join(spec_tentacles_config, "t1.json")) - shutil.copy(profile.config_file(), os.path.join(spec_tentacles_config, "t2.json")) + shutil.copy(_profile_config_file(profile), os.path.join(spec_tentacles_config, "t1.json")) + shutil.copy(_profile_config_file(profile), os.path.join(spec_tentacles_config, "t2.json")) with mock.patch.object(os, "remove", mock.Mock()) as remove_mock: profiles.export_profile(profile, export_path) remove_mock.assert_not_called() @@ -81,11 +87,11 @@ def test_export_profile_with_existing_file(profile): dir1=spec_tentacles_config, dir2=other_profile): # create fake tentacles config - shutil.copy(profile.config_file(), tentacles_config) + shutil.copy(_profile_config_file(profile), tentacles_config) os.mkdir(spec_tentacles_config) - shutil.copy(profile.config_file(), os.path.join(spec_tentacles_config, "t1.json")) - shutil.copy(profile.config_file(), os.path.join(spec_tentacles_config, "t2.json")) - shutil.copy(profile.config_file(), f"{export_path}.{constants.PROFILE_EXPORT_FORMAT}") + shutil.copy(_profile_config_file(profile), os.path.join(spec_tentacles_config, "t1.json")) + shutil.copy(_profile_config_file(profile), os.path.join(spec_tentacles_config, "t2.json")) + shutil.copy(_profile_config_file(profile), f"{export_path}.{constants.PROFILE_EXPORT_FORMAT}") with mock.patch.object(os, "remove", mock.Mock()) as remove_mock: profiles.export_profile(profile, export_path) remove_mock.assert_called_once_with(f"{export_path}.{constants.PROFILE_EXPORT_FORMAT}") @@ -115,35 +121,35 @@ def test_import_install_profile(profile, invalid_profile): dir2=user_root_folder_provider.get_user_root_folder(), dir3=spec_tentacles_config): # create fake tentacles config - shutil.copy(profile.config_file(), tentacles_config) + shutil.copy(_profile_config_file(profile), tentacles_config) os.mkdir(spec_tentacles_config) - shutil.copy(profile.config_file(), os.path.join(spec_tentacles_config, "t1.json")) - shutil.copy(profile.config_file(), os.path.join(spec_tentacles_config, "t2.json")) + shutil.copy(_profile_config_file(profile), os.path.join(spec_tentacles_config, "t1.json")) + shutil.copy(_profile_config_file(profile), os.path.join(spec_tentacles_config, "t2.json")) profiles.export_profile(profile, export_path) - imported_profile_path = os.path.join(user_root_folder_provider.get_user_profiles_folder(), "default") with mock.patch.object(profile_sharing, "_ensure_unique_profile_id", mock.Mock()) \ as _ensure_unique_profile_id_mock: imported_profile = profiles.import_profile(exported_file, profile_schema, origin_url="plop.wow") assert isinstance(imported_profile, profiles.Profile) - profile.read_config() assert profile.name == imported_profile.name assert profile.path != imported_profile.path assert profile.imported is False assert imported_profile.imported is True assert imported_profile.origin_url == "plop.wow" _ensure_unique_profile_id_mock.assert_called_once() + imported_profile_path = imported_profile.path assert os.path.isdir(imported_profile_path) # ensure all files got imported for root, dirs, files in os.walk(profile.path): - dir_path = os.path.join(other_profile, "specific_config") if "specific_config" in root else other_profile + dir_path = os.path.join(imported_profile_path, "specific_config") if "specific_config" in root else imported_profile_path assert all( os.path.isfile(os.path.join(dir_path, f)) for f in files ) - assert isinstance(profiles.import_profile(exported_file, profile_schema), profiles.Profile) - assert os.path.isdir(f"{imported_profile_path}_2") + second_imported_profile = profiles.import_profile(exported_file, profile_schema) + assert isinstance(second_imported_profile, profiles.Profile) + assert os.path.isdir(second_imported_profile.path) + assert second_imported_profile.path != imported_profile_path assert os.path.isdir(imported_profile_path) - assert not os.path.isdir(f"{imported_profile_path}_3") # now with invalid profile profiles.export_profile(invalid_profile, export_path) @@ -152,17 +158,17 @@ def test_import_install_profile(profile, invalid_profile): def test_get_unique_profile_folder(profile): - assert _get_unique_profile_folder(profile.config_file()) == f"{profile.config_file()}_2" - other_file = f"{profile.config_file()}_2" - other_file_2 = f"{profile.config_file()}_3" - other_file_3 = f"{profile.config_file()}_5" + assert _get_unique_profile_folder(_profile_config_file(profile)) == f"{_profile_config_file(profile)}_2" + other_file = f"{_profile_config_file(profile)}_2" + other_file_2 = f"{_profile_config_file(profile)}_3" + other_file_3 = f"{_profile_config_file(profile)}_5" with _cleaned_tentacles(other_file, other_file_2, other_file_3): - shutil.copy(profile.config_file(), other_file) - assert _get_unique_profile_folder(profile.config_file()) == f"{profile.config_file()}_3" - shutil.copy(profile.config_file(), other_file_2) - assert _get_unique_profile_folder(profile.config_file()) == f"{profile.config_file()}_4" - shutil.copy(profile.config_file(), other_file_3) - assert _get_unique_profile_folder(profile.config_file()) == f"{profile.config_file()}_4" + shutil.copy(_profile_config_file(profile), other_file) + assert _get_unique_profile_folder(_profile_config_file(profile)) == f"{_profile_config_file(profile)}_3" + shutil.copy(_profile_config_file(profile), other_file_2) + assert _get_unique_profile_folder(_profile_config_file(profile)) == f"{_profile_config_file(profile)}_4" + shutil.copy(_profile_config_file(profile), other_file_3) + assert _get_unique_profile_folder(_profile_config_file(profile)) == f"{_profile_config_file(profile)}_4" def test_ensure_unique_profile_id(profile): @@ -171,10 +177,13 @@ def test_ensure_unique_profile_id(profile): other_profile_path = profiles_path.joinpath(other_profile) with _cleaned_tentacles(dir1=other_profile_path): shutil.copytree(profile.path, other_profile_path) - other_profile = profiles.Profile(other_profile_path).read_config() + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + other_profile = filesystem_backend.read_profile_from_path(str(other_profile_path)) + other_profile.bind_profile_storage(profile.get_profile_storage()) _ensure_unique_profile_id(other_profile) - other_profile.save() - ids = profiles.Profile.get_all_profiles_ids(profiles_path) + filesystem_backend.write_profile_config(other_profile) + profile_storage = profile_storage_module.ProfileStorage(str(profiles_path)) + ids = profile_storage.list_profile_ids() assert len(ids) == 2 # changed new profile id assert ids[0] != ids[1] diff --git a/packages/commons/tests/profiles/test_profile_storage.py b/packages/commons/tests/profiles/test_profile_storage.py new file mode 100644 index 0000000000..a13e804839 --- /dev/null +++ b/packages/commons/tests/profiles/test_profile_storage.py @@ -0,0 +1,472 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. + +import os +import copy + +import mock +import pytest + +import octobot_commons.constants as constants +import octobot_commons.errors as errors_module +import octobot_commons.profiles.profile_types.profile as profile_module +import octobot_commons.profiles.backends as profile_backends_module +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_storage as profile_storage_module +import octobot_commons.profiles.profile_types.sync_profile as sync_profile_module + + +class TestProfileStorageListProfiles: + def test_filesystem_profile_wins_on_id_conflict(self, tmp_path): + profiles_path = os.path.join(tmp_path, constants.PROFILES_FOLDER) + os.makedirs(profiles_path, exist_ok=True) + default_profile_path = os.path.join(profiles_path, constants.DEFAULT_PROFILE) + os.makedirs(default_profile_path, exist_ok=True) + profile_file = { + constants.CONFIG_PROFILE: { + constants.CONFIG_ID: constants.DEFAULT_PROFILE, + constants.CONFIG_NAME: "filesystem-default", + }, + constants.PROFILE_CONFIG: { + constants.CONFIG_CRYPTO_CURRENCIES: {}, + constants.CONFIG_EXCHANGES: {}, + constants.CONFIG_TRADER: {constants.CONFIG_ENABLED_OPTION: False}, + constants.CONFIG_SIMULATOR: { + constants.CONFIG_ENABLED_OPTION: True, + constants.CONFIG_STARTING_PORTFOLIO: {}, + constants.CONFIG_SIMULATOR_FEES: {}, + }, + constants.CONFIG_TRADING: { + constants.CONFIG_TRADER_REFERENCE_MARKET: constants.DEFAULT_REFERENCE_MARKET, + constants.CONFIG_TRADER_RISK: 1, + }, + constants.CONFIG_DISTRIBUTION: constants.DEFAULT_DISTRIBUTION, + }, + } + import octobot_commons.json_util as json_util + + json_util.safe_dump(profile_file, os.path.join(default_profile_path, constants.PROFILE_CONFIG_FILE)) + + sync_profile_data = profile_data_module.ProfileData.from_dict( + { + "profile_details": {"id": constants.DEFAULT_PROFILE, "name": "sync-default"}, + "trading": {"reference_market": constants.DEFAULT_REFERENCE_MARKET}, + } + ) + sync_profile = sync_profile_module.SyncProfile( + sync_profile_data, + os.path.join(tmp_path, "runtime", constants.DEFAULT_PROFILE), + ) + filesystem_backend = mock.Mock() + filesystem_backend.list_profiles.return_value = { + constants.DEFAULT_PROFILE: profile_module.Profile(default_profile_path) + } + filesystem_backend.list_profiles.return_value[ + constants.DEFAULT_PROFILE + ] = profile_backends_module.FilesystemProfileBackend().read_profile_from_path( + default_profile_path + ) + sync_backend = mock.Mock() + sync_backend.list_profiles.return_value = {constants.DEFAULT_PROFILE: sync_profile} + profile_storage = profile_storage_module.ProfileStorage( + profiles_path, + None, + filesystem_backend=filesystem_backend, + sync_backend=sync_backend, + ) + profiles = profile_storage._list_profiles() + assert profiles[constants.DEFAULT_PROFILE].name == "filesystem-default" + + +class TestProfileStorageConfigureSyncUser: + def test_configure_sync_user_validates_wallet(self, profile_storage, monkeypatch): + authenticator = mock.Mock() + authenticator.get_wallet_by_user_id.return_value = mock.Mock() + monkeypatch.setattr( + "octobot_commons.authentication.Authenticator.instance", + mock.Mock(return_value=authenticator), + ) + profile_storage.configure_sync_user("wallet-user") + assert profile_storage.is_sync_available() + authenticator.get_wallet_by_user_id.assert_called_once_with("wallet-user") + + +class TestProfileStorageBindProcessChildSyncUserId: + def test_bind_process_child_sync_user_id_sets_backend_without_wallet(self, profile_storage): + profile_storage.bind_process_child_sync_user_id("process-child-user") + assert profile_storage.is_sync_available() + assert profile_storage._sync_user_id == "process-child-user" + + def test_bind_process_child_sync_user_id_rejects_empty(self, profile_storage): + with pytest.raises(errors_module.ProfileDataError, match="non-empty"): + profile_storage.bind_process_child_sync_user_id("") + + +class TestProfileStorageListSyncProfiles: + def test_returns_empty_when_sync_unavailable(self, profile_storage): + assert profile_storage.list_sync_profiles() == {} + + def test_returns_sync_profiles_with_storage_bound(self, profile_storage): + sync_profile_data = profile_data_module.ProfileData.from_dict( + { + "profile_details": {"id": "sync-profile-id", "name": "sync-profile"}, + "trading": {"reference_market": constants.DEFAULT_REFERENCE_MARKET}, + } + ) + sync_profile = sync_profile_module.SyncProfile( + sync_profile_data, + os.path.join(profile_storage.profiles_path, "runtime", "sync-profile-id"), + ) + sync_backend = mock.Mock() + sync_backend.list_profiles.return_value = {"sync-profile-id": sync_profile} + profile_storage._sync_backend = sync_backend + profile_storage.bind_process_child_sync_user_id("process-child-user") + + profiles_by_id = profile_storage.list_sync_profiles() + + sync_backend.list_profiles.assert_called_once_with(None) + assert profiles_by_id == {"sync-profile-id": sync_profile} + assert sync_profile.get_profile_storage() is profile_storage + + +class TestSyncProfileBackendImportProfileData: + def test_import_profile_data_assigns_strategy_id(self, tmp_path): + sync_backend = profile_backends_module.SyncProfileBackend( + sync_user_id="wallet-user" + ) + profile_data = profile_data_module.ProfileData.from_dict( + { + "profile_details": {"name": "my-profile"}, + "trading": {"reference_market": constants.DEFAULT_REFERENCE_MARKET}, + } + ) + created_profile = None + + def _create_item(user_id, strategy): + nonlocal created_profile + assert user_id == "wallet-user" + assert strategy.id == profile_data.profile_details.id + created_profile = strategy + return strategy + + import octobot_sync.sync.collection_backend.errors as collection_errors + + strategy_provider = mock.Mock() + strategy_provider.update_item.side_effect = collection_errors.ItemNotFoundError( + "missing" + ) + strategy_provider.create_item.side_effect = _create_item + with mock.patch.object( + sync_backend, + "_get_strategy_provider", + mock.Mock(return_value=strategy_provider), + ): + profile = sync_backend.import_profile_data( + profile_data, + schema_path=None, + name="my-profile", + ) + assert profile.is_sync_backed() + assert profile.profile_id == profile_data.profile_details.id + assert created_profile is not None + + +class TestSyncProfileBackendListProfiles: + def test_list_profiles_logs_exception_on_failure(self): + sync_backend = profile_backends_module.SyncProfileBackend( + "/profiles", + sync_user_id="wallet-user", + ) + with mock.patch.object( + sync_backend, + "_list_profile_strategies", + mock.Mock(side_effect=RuntimeError("sync storage unavailable")), + ), mock.patch( + "octobot_commons.profiles.backends.sync_profile_backend._get_logger", + ) as get_logger_mock: + logger_mock = mock.Mock() + get_logger_mock.return_value = logger_mock + profiles_by_id = sync_backend.list_profiles() + assert profiles_by_id == {} + logger_mock.exception.assert_called_once() + + +class TestSyncProfileBackendDuplicateProfile: + def test_duplicate_profile_assigns_new_strategy_id(self, profile): + sync_backend = profile_backends_module.SyncProfileBackend( + sync_user_id="wallet-user" + ) + created_strategy_id = None + + def _create_item(user_id, strategy): + nonlocal created_strategy_id + assert user_id == "wallet-user" + created_strategy_id = strategy.id + return strategy + + import octobot_sync.sync.collection_backend.errors as collection_errors + + strategy_provider = mock.Mock() + strategy_provider.update_item.side_effect = collection_errors.ItemNotFoundError( + "missing" + ) + strategy_provider.create_item.side_effect = _create_item + with mock.patch.object( + sync_backend, + "_get_strategy_provider", + mock.Mock(return_value=strategy_provider), + ): + duplicate = sync_backend.duplicate_profile( + profile, + name="copy-name", + description="copy-desc", + ) + assert duplicate.is_sync_backed() + assert duplicate.profile_id == created_strategy_id + assert duplicate.profile_id != profile.profile_id + assert duplicate.name == "copy-name" + assert duplicate.description == "copy-desc" + assert duplicate.read_only is False + assert duplicate.imported is False + assert duplicate.origin_url is None + + +class TestProfileStorageListProfileIdsMerge: + def test_list_profile_ids_merges_filesystem_and_sync_ids(self): + filesystem_backend = mock.Mock() + filesystem_backend.list_profile_ids.return_value = ["fs-profile"] + sync_backend = mock.Mock() + sync_backend.list_profile_ids.return_value = ["sync-profile", "fs-profile"] + profile_storage = profile_storage_module.ProfileStorage( + "/profiles", + None, + filesystem_backend=filesystem_backend, + sync_backend=sync_backend, + ) + profile_ids = profile_storage.list_profile_ids() + assert profile_ids == ["fs-profile", "sync-profile"] + filesystem_backend.list_profile_ids.assert_called_once_with(ignore=None) + sync_backend.list_profile_ids.assert_called_once_with(ignore=None) + + +class TestProfileStorageDuplicateProfile: + def test_duplicate_profile_requires_sync(self, profile_storage, profile): + with pytest.raises( + errors_module.ProfileDataError, + match="configured wallet user id", + ): + profile_storage.duplicate_profile(profile) + + def test_duplicate_profile_delegates_to_sync_backend( + self, profile_storage, profile, monkeypatch + ): + authenticator = mock.Mock() + authenticator.get_wallet_by_user_id.return_value = mock.Mock() + monkeypatch.setattr( + "octobot_commons.authentication.Authenticator.instance", + mock.Mock(return_value=authenticator), + ) + profile_storage.configure_sync_user("wallet-user") + sync_duplicate = mock.Mock() + sync_duplicate.bind_profile_storage = mock.Mock() + sync_backend = mock.Mock() + sync_backend.duplicate_profile.return_value = sync_duplicate + profile_storage._sync_backend = sync_backend + result = profile_storage.duplicate_profile( + profile, name="copy", description="desc" + ) + sync_backend.duplicate_profile.assert_called_once_with( + profile, + name="copy", + description="desc", + ) + sync_duplicate.bind_profile_storage.assert_called_once_with(profile_storage) + assert result is sync_duplicate + + +class TestProfileStorageMasterOverlay: + def _write_profile_file(self, profile_path: str, profile_id: str, *, read_only: bool) -> None: + import octobot_commons.json_util as json_util + + os.makedirs(profile_path, exist_ok=True) + profile_file = { + constants.CONFIG_PROFILE: { + constants.CONFIG_ID: profile_id, + constants.CONFIG_NAME: profile_id, + constants.CONFIG_READ_ONLY: read_only, + }, + constants.PROFILE_CONFIG: { + constants.CONFIG_CRYPTO_CURRENCIES: {}, + constants.CONFIG_EXCHANGES: {}, + constants.CONFIG_TRADER: {constants.CONFIG_ENABLED_OPTION: False}, + constants.CONFIG_SIMULATOR: { + constants.CONFIG_ENABLED_OPTION: True, + constants.CONFIG_STARTING_PORTFOLIO: {}, + constants.CONFIG_SIMULATOR_FEES: {}, + }, + constants.CONFIG_TRADING: { + constants.CONFIG_TRADER_REFERENCE_MARKET: constants.DEFAULT_REFERENCE_MARKET, + constants.CONFIG_TRADER_RISK: 1, + }, + constants.CONFIG_DISTRIBUTION: constants.DEFAULT_DISTRIBUTION, + }, + } + json_util.safe_dump(profile_file, os.path.join(profile_path, constants.PROFILE_CONFIG_FILE)) + + def test_overlay_exposes_read_only_profiles(self, tmp_path): + child_profiles_path = tmp_path / "child" / constants.PROFILES_FOLDER + child_profiles_path.mkdir(parents=True) + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + readonly_profile_id = "readonly-strategy" + self._write_profile_file( + os.path.join(master_profiles_path, readonly_profile_id), + readonly_profile_id, + read_only=True, + ) + profile_storage = profile_storage_module.ProfileStorage(str(child_profiles_path), None) + profile_storage.configure_readonly_profiles_path(str(master_profiles_path)) + profiles = profile_storage.load_all_profiles() + assert readonly_profile_id in profiles + assert profiles[readonly_profile_id].read_only is True + + def test_overlay_ignores_editable_non_shared_profiles(self, tmp_path): + child_profiles_path = tmp_path / "child" / constants.PROFILES_FOLDER + child_profiles_path.mkdir(parents=True) + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + editable_profile_id = "editable-strategy" + self._write_profile_file( + os.path.join(master_profiles_path, editable_profile_id), + editable_profile_id, + read_only=False, + ) + profile_storage = profile_storage_module.ProfileStorage(str(child_profiles_path), None) + profile_storage.configure_readonly_profiles_path(str(master_profiles_path)) + profiles = profile_storage.load_all_profiles() + assert editable_profile_id not in profiles + + def test_local_filesystem_profile_wins_on_id_conflict(self, tmp_path): + child_profiles_path = tmp_path / "child" / constants.PROFILES_FOLDER + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + profile_id = "shared-id" + child_profiles_path.mkdir(parents=True, exist_ok=True) + self._write_profile_file( + os.path.join(master_profiles_path, profile_id), + profile_id, + read_only=True, + ) + self._write_profile_file( + os.path.join(child_profiles_path, profile_id), + profile_id, + read_only=False, + ) + profile_storage = profile_storage_module.ProfileStorage(str(child_profiles_path), None) + profile_storage.configure_readonly_profiles_path(str(master_profiles_path)) + profiles = profile_storage.load_all_profiles() + assert profiles[profile_id].name == profile_id + assert profiles[profile_id].path == os.path.join(child_profiles_path, profile_id) + + def test_save_active_profile_blocks_master_overlay_profile(self, tmp_path): + child_profiles_path = tmp_path / "child" / constants.PROFILES_FOLDER + child_profiles_path.mkdir(parents=True) + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + readonly_profile_id = "non-trading" + self._write_profile_file( + os.path.join(master_profiles_path, readonly_profile_id), + readonly_profile_id, + read_only=True, + ) + profile_storage = profile_storage_module.ProfileStorage(str(child_profiles_path), None) + profile_storage.configure_readonly_profiles_path(str(master_profiles_path)) + overlay_profile = profile_storage.get_profile(readonly_profile_id) + with pytest.raises( + errors_module.ProfileDataError, + match="shared from the master", + ): + profile_storage.save_active_profile(overlay_profile, {}) + + def test_delete_profile_blocks_master_overlay_profile(self, tmp_path): + child_profiles_path = tmp_path / "child" / constants.PROFILES_FOLDER + child_profiles_path.mkdir(parents=True) + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + readonly_profile_id = "readonly-strategy" + self._write_profile_file( + os.path.join(master_profiles_path, readonly_profile_id), + readonly_profile_id, + read_only=True, + ) + profile_storage = profile_storage_module.ProfileStorage(str(child_profiles_path), None) + profile_storage.configure_readonly_profiles_path(str(master_profiles_path)) + overlay_profile = profile_storage.get_profile(readonly_profile_id) + with pytest.raises( + errors_module.ProfileRemovalError, + match="shared from the master", + ): + profile_storage.delete_profile(readonly_profile_id, profile=overlay_profile) + + +class TestSyncProfileBackendSaveProfile: + def test_save_profile_persists_exchanges_and_portfolio(self, tmp_path): + sync_backend = profile_backends_module.SyncProfileBackend( + sync_user_id="wallet-user" + ) + initial_profile_data = profile_data_module.ProfileData.from_dict( + { + "profile_details": {"id": "aaaa", "name": "AAAA"}, + "trading": {"reference_market": constants.DEFAULT_REFERENCE_MARKET}, + "trader_simulator": { + "enabled": True, + "starting_portfolio": {"USDT": 100}, + }, + } + ) + profile = sync_profile_module.SyncProfile( + initial_profile_data, + str(tmp_path / "runtime"), + ) + profile.profile_id = "aaaa" + profile.name = "AAAA" + profile_storage = profile_storage_module.ProfileStorage( + str(tmp_path / constants.PROFILES_FOLDER), + None, + sync_backend=sync_backend, + ) + profile.bind_profile_storage(profile_storage) + global_config = { + constants.CONFIG_CRYPTO_CURRENCIES: copy.deepcopy( + profile.config[constants.CONFIG_CRYPTO_CURRENCIES] + ), + constants.CONFIG_DISTRIBUTION: profile.config[constants.CONFIG_DISTRIBUTION], + constants.CONFIG_TRADING: copy.deepcopy(profile.config[constants.CONFIG_TRADING]), + constants.CONFIG_TRADER: copy.deepcopy(profile.config[constants.CONFIG_TRADER]), + constants.CONFIG_SIMULATOR: { + constants.CONFIG_ENABLED_OPTION: True, + constants.CONFIG_STARTING_PORTFOLIO: {"BTC": 5, "USDT": 5000}, + constants.CONFIG_SIMULATOR_FEES: { + constants.CONFIG_SIMULATOR_FEES_MAKER: 0.1, + constants.CONFIG_SIMULATOR_FEES_TAKER: 0.1, + }, + }, + constants.CONFIG_EXCHANGES: { + "binance": { + constants.CONFIG_ENABLED_OPTION: True, + constants.CONFIG_EXCHANGE_TYPE: "spot", + }, + }, + } + strategy_provider = mock.Mock() + strategy_provider.update_item = mock.Mock() + with mock.patch.object( + sync_backend, + "_get_strategy_provider", + mock.Mock(return_value=strategy_provider), + ): + profile.save_config(global_config) + saved_profile_data = profile.get_profile_data() + assert saved_profile_data.trader_simulator.starting_portfolio == { + "BTC": 5, + "USDT": 5000, + } + assert len(saved_profile_data.exchanges) == 1 + assert saved_profile_data.exchanges[0].internal_name == "binance" + assert saved_profile_data.exchanges[0].exchange_type == "spot" + strategy_provider.update_item.assert_called_once() diff --git a/packages/commons/tests/static/profile.json b/packages/commons/tests/static/profile.json index caa51ba337..b5992503f7 100644 --- a/packages/commons/tests/static/profile.json +++ b/packages/commons/tests/static/profile.json @@ -1,43 +1,53 @@ { - "profile": { - "avatar": "default_profile.png", - "description": "OctoBot default profile.", - "id": "default", - "name": "default", - "origin_url": "https://default.url" - }, - "config": { - "crypto-currencies": { - "Bitcoin": { - "pairs": [ - "BTC/USDT" - ] - } + "config": { + "crypto-currencies": { + "Bitcoin": { + "pairs": [ + "BTC/USDT" + ] + } + }, + "distribution": "default", + "exchanges": { + "binance": { + "enabled": true + } + }, + "trader": { + "enabled": false, + "load-trade-history": true + }, + "trader-simulator": { + "enabled": true, + "fees": { + "maker": 0.1, + "taker": 0.1 + }, + "starting-portfolio": { + "BTC": 10, + "USDT": 1000 + } + }, + "trading": { + "paused": false, + "reference-market": "BTC", + "risk": 0.5 + } }, - "exchanges": { - "binance": { - "enabled": true - } - }, - "trading": { - "paused": false, - "reference-market": "BTC", - "risk": 0.5 - }, - "trader": { - "enabled": false, - "load-trade-history": true - }, - "trader-simulator": { - "enabled": true, - "fees": { - "maker": 0.1, - "taker": 0.1 - }, - "starting-portfolio": { - "BTC": 10, - "USDT": 1000 - } + "profile": { + "auto_update": false, + "avatar": "default_profile.png", + "complexity": 2, + "description": "OctoBot default profile.", + "extra_backtesting_time_frames": [], + "hidden": false, + "id": "default", + "imported": false, + "name": "default", + "origin_url": "https://default.url", + "read_only": false, + "risk": 2, + "slug": "", + "type": "live" } - } -} +} \ No newline at end of file diff --git a/packages/commons/tests/test_managed_child_process_registry.py b/packages/commons/tests/test_managed_child_process_registry.py new file mode 100644 index 0000000000..f6edd3faf6 --- /dev/null +++ b/packages/commons/tests/test_managed_child_process_registry.py @@ -0,0 +1,166 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import asyncio +import sys +import time + +import mock +import pytest + +import octobot_commons.managed_child_process_registry as managed_child_process_registry +import octobot_commons.process_util as process_util +import octobot_commons.singleton.singleton_class as singleton_class + + +@pytest.fixture(autouse=True) +def reset_managed_child_process_registry(): + yield + singleton_class.Singleton._instances.pop( + managed_child_process_registry.ManagedChildProcessRegistry, + None, + ) + + +class TestManagedChildProcessRegistryInstance: + def test_instance_returns_same_object(self): + first = managed_child_process_registry.ManagedChildProcessRegistry.instance() + second = managed_child_process_registry.ManagedChildProcessRegistry.instance() + assert first is second + + +class TestManagedChildProcessRegistryRegister: + def test_register_ignores_non_positive_pid(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + registry.register(0) + registry.register(-3) + assert registry.snapshot_running_pids() == frozenset() + + def test_register_adds_pid_to_snapshot(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object( + managed_child_process_registry.process_util, + "pid_is_running", + return_value=True, + ): + registry.register(101) + assert registry.snapshot_running_pids() == frozenset({101}) + + def test_register_is_idempotent(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object( + managed_child_process_registry.process_util, + "pid_is_running", + return_value=True, + ): + registry.register(202) + registry.register(202) + assert registry.snapshot_running_pids() == frozenset({202}) + + +class TestManagedChildProcessRegistryUnregister: + def test_unregister_removes_pid(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object(process_util, "pid_is_running", return_value=True): + registry.register(303) + registry.unregister(303) + assert registry.snapshot_running_pids() == frozenset() + + def test_unregister_unknown_pid_is_no_op(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + registry.unregister(99999) + assert registry.snapshot_running_pids() == frozenset() + + +class TestManagedChildProcessRegistrySnapshotRunningPids: + def test_lazy_prunes_dead_pids(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object(process_util, "pid_is_running", return_value=True): + registry.register(404) + with mock.patch.object(process_util, "pid_is_running", return_value=False): + assert registry.snapshot_running_pids() == frozenset() + assert registry.snapshot_running_pids() == frozenset() + + +class TestManagedChildProcessRegistryGracefulStopAll: + @pytest.mark.asyncio + async def test_stops_spawned_child_via_sigterm(self, tmp_path): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + child = process_util.spawn_managed_subprocess( + [ + sys.executable, + "-c", + "import time; time.sleep(30)", + ], + working_directory=str(tmp_path), + ) + try: + assert child.pid in registry.snapshot_running_pids() + outcomes = await registry.graceful_stop_all(timeout_seconds=15.0) + assert outcomes[child.pid] in {"stopped", "already_stopped"} + deadline = time.monotonic() + 15.0 + while child.poll() is None and time.monotonic() < deadline: + await asyncio.sleep(0.05) + assert child.poll() is not None + assert child.pid not in registry.snapshot_running_pids() + finally: + if child.poll() is None: + child.kill() + child.wait(timeout=10) + + @pytest.mark.asyncio + async def test_force_kills_child_when_graceful_stop_times_out(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + force_kill_requested = False + + def pid_is_running_side_effect(_pid): + return not force_kill_requested + + def request_force_kill_side_effect(_pid, *, logger=None): + nonlocal force_kill_requested + force_kill_requested = True + return {"status": "force_killed"} + + with ( + mock.patch.object( + managed_child_process_registry.process_util, + "pid_is_running", + side_effect=pid_is_running_side_effect, + ), + mock.patch.object( + managed_child_process_registry.process_util, + "request_graceful_stop_via_sigterm", + return_value={"status": "stopped", "signal": "sigterm"}, + ), + mock.patch.object( + managed_child_process_registry.process_util, + "request_force_kill", + side_effect=request_force_kill_side_effect, + ) as force_kill_mock, + ): + registry.register(777) + outcomes = await registry.graceful_stop_all( + timeout_seconds=0.1, + poll_interval=0.01, + ) + assert outcomes[777] == "force_killed" + force_kill_mock.assert_called_once_with(777, logger=registry._logger) + assert registry.snapshot_running_pids() == frozenset() + + @pytest.mark.asyncio + async def test_returns_empty_when_no_registered_children(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + outcomes = await registry.graceful_stop_all(timeout_seconds=1.0) + assert outcomes == {} diff --git a/packages/commons/tests/test_process_util.py b/packages/commons/tests/test_process_util.py index eba9fb9393..5453274eb6 100644 --- a/packages/commons/tests/test_process_util.py +++ b/packages/commons/tests/test_process_util.py @@ -23,12 +23,23 @@ import pytest import octobot_commons.errors as commons_errors +import octobot_commons.managed_child_process_registry as managed_child_process_registry import octobot_commons.process_util as process_util +import octobot_commons.singleton.singleton_class as singleton_class class TestSpawnManagedSubprocess: + @pytest.fixture(autouse=True) + def reset_managed_child_process_registry(self): + yield + singleton_class.Singleton._instances.pop( + managed_child_process_registry.ManagedChildProcessRegistry, + None, + ) + def test_popen_called_with_argv_cwd_env(self): fake_handle = mock.Mock(spec=subprocess.Popen) + fake_handle.pid = 1001 with mock.patch.object( process_util.subprocess, "Popen", @@ -54,12 +65,14 @@ def test_uses_os_environ_copy_when_environment_missing(self): {"EXISTING_ENV_KEY": "1"}, clear=False, ): + popen_mock.return_value.pid = 1002 process_util.spawn_managed_subprocess([], working_directory="/w") _args, keywords = popen_mock.call_args assert keywords["env"]["EXISTING_ENV_KEY"] == "1" def test_creationflags_hide_console_on_windows(self): fake_handle = mock.Mock(spec=subprocess.Popen) + fake_handle.pid = 1003 with mock.patch.object(process_util.sys, "platform", "win32"), mock.patch.object( process_util.subprocess, "Popen", @@ -73,6 +86,19 @@ def test_creationflags_hide_console_on_windows(self): 0, ) + def test_registers_child_pid_with_managed_child_process_registry(self): + fake_handle = mock.Mock(spec=subprocess.Popen) + fake_handle.pid = 4242 + with mock.patch.object( + process_util.subprocess, + "Popen", + return_value=fake_handle, + ): + process_util.spawn_managed_subprocess(["x"], working_directory="/work") + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object(process_util, "pid_is_running", return_value=True): + assert fake_handle.pid in registry.snapshot_running_pids() + class TestPidIsRunning: def test_non_positive_pid_is_false(self): @@ -126,14 +152,14 @@ def test_with_psutil_no_such_process_from_is_running(self): class TestRequestGracefulStopViaSigterm: def test_invalid_pid_raises(self): - with pytest.raises(commons_errors.DSLInterpreterError, match="Invalid pid"): + with pytest.raises(commons_errors.ProcessError, match="Invalid pid"): process_util.request_graceful_stop_via_sigterm(0) def test_raises_when_sigterm_unavailable(self): sentinel_signal_module = mock.Mock() sentinel_signal_module.SIGTERM = None with mock.patch.object(process_util, "signal", sentinel_signal_module): - with pytest.raises(commons_errors.DSLInterpreterError, match="SIGTERM is not available"): + with pytest.raises(commons_errors.ProcessError, match="SIGTERM is not available"): process_util.request_graceful_stop_via_sigterm(10) def test_returns_already_stopped_when_pid_not_running(self): @@ -155,12 +181,45 @@ def test_os_kill_failure_when_still_running_wraps_error(self): mock.patch.object(process_util.os, "kill", side_effect=OSError("perm denied")), ): with pytest.raises( - commons_errors.DSLInterpreterError, + commons_errors.ProcessError, match=r"Failed to send stop signal to pid=88", ): process_util.request_graceful_stop_via_sigterm(88) +class TestRequestForceKill: + def test_invalid_pid_raises(self): + with pytest.raises(commons_errors.ProcessError, match="Invalid pid"): + process_util.request_force_kill(0) + + def test_returns_already_stopped_when_pid_not_running(self): + with mock.patch.object(process_util, "pid_is_running", return_value=False): + result = process_util.request_force_kill(55) + assert result["status"] == "already_stopped" + + def test_kills_running_process(self): + fake_process = mock.Mock() + with ( + mock.patch.object(process_util, "pid_is_running", return_value=True), + mock.patch.object(process_util.psutil, "Process", return_value=fake_process), + ): + result = process_util.request_force_kill(66) + fake_process.kill.assert_called_once() + assert result["status"] == "force_killed" + + def test_no_such_process_returns_already_stopped(self): + with ( + mock.patch.object(process_util, "pid_is_running", return_value=True), + mock.patch.object( + process_util.psutil, + "Process", + side_effect=process_util.psutil.NoSuchProcess(77), + ), + ): + result = process_util.request_force_kill(77) + assert result["status"] == "already_stopped" + + class TestSpawnManagedSubprocessGracefulStopIntegration: def test_spawned_sleeping_child_can_be_stopped_by_request_graceful_stop_via_sigterm( self, diff --git a/packages/copy/octobot_copy/orders_mirroring/orders_synchronizer.py b/packages/copy/octobot_copy/orders_mirroring/orders_synchronizer.py index 7507b27e7c..c95e2258e8 100644 --- a/packages/copy/octobot_copy/orders_mirroring/orders_synchronizer.py +++ b/packages/copy/octobot_copy/orders_mirroring/orders_synchronizer.py @@ -992,6 +992,29 @@ async def _upsert_mirrored_reference_order( ) return out, replaced_cancelled, 0, None + def _get_locked_base_from_open_mirrored_sells( + self, + symbol: str, + exclude_order: typing.Optional[trading_personal_data.Order] = None, + ) -> decimal.Decimal: + parsed = symbol_util.parse_symbol(symbol) + base_currency = parsed.base + exclude_order_id = str(exclude_order.order_id) if exclude_order is not None else None + locked_base = trading_constants.ZERO + for order in self._exchange_interface.orders.get_open_orders(): + if order.symbol != symbol: + continue + if order.tag != copy_constants.MIRRORED_ORDER_TAG: + continue + if order.side is not trading_enums.TradeOrderSide.SELL: + continue + if exclude_order_id is not None and str(order.order_id) == exclude_order_id: + continue + if order.currency != base_currency: + continue + locked_base += self._exchange_interface.orders.get_order_locked_amount(order) + return locked_base + async def _compute_mirrored_quantity_type_and_price( self, symbol: str, @@ -1003,8 +1026,8 @@ async def _compute_mirrored_quantity_type_and_price( ) -> mirrored_quantity_compute_result.MirroredQuantityComputeResult: # Buys cap using free quote for new orders (sibling buys reserve quote). When re-checking an open # mirrored buy, add this order's locked quote back so ideal size matches portfolio semantics. - # New sells use total base (sibling sell locks still count). Open mirrored sells use available - # base plus this order's locked base for the same reason as buys. + # Sells cap using total base minus locked base from open mirrored sells (sibling locks count). + # When re-checking an open mirrored sell, exclude this order's lock from that sum. ( total_symbol_holding, _total_market_holding, @@ -1066,16 +1089,13 @@ async def _compute_mirrored_quantity_type_and_price( if effective_target_price else scaled_quantity, ) - elif ( - open_mirrored_order is not None - and open_mirrored_order.side is trading_enums.TradeOrderSide.SELL - ): - base_budget = available_symbol_holding + self._exchange_interface.orders.get_order_locked_amount( - open_mirrored_order + elif side is trading_enums.TradeOrderSide.SELL: + locked_by_siblings = self._get_locked_base_from_open_mirrored_sells( + symbol, + exclude_order=open_mirrored_order, ) + base_budget = total_symbol_holding - locked_by_siblings target_quantity = min(scaled_quantity, base_budget) - else: - target_quantity = min(scaled_quantity, total_symbol_holding) zero_short_reason: typing.Optional[str] = None if target_quantity <= trading_constants.ZERO: zero_short_reason = ( diff --git a/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer.py b/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer.py index c7b521ccb6..99a62398f8 100644 --- a/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer.py +++ b/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer.py @@ -1107,6 +1107,26 @@ async def run_grace(): class TestMirroredOrderSelfLockCreditCompute: """open_mirrored_order credits this line's locked funds so repeat sync does not false quantity mismatch.""" + @staticmethod + def _mirrored_sell_order( + *, + order_id: str, + symbol: str, + locked_quantity: decimal.Decimal, + price: decimal.Decimal, + ): + order = mock.Mock() + order.order_id = order_id + order.tag = copy_constants.MIRRORED_ORDER_TAG + order.side = trading_enums.TradeOrderSide.SELL + order.symbol = symbol + order.currency = symbol.split("/")[0] + order.origin_price = price + order.is_filled = mock.Mock(return_value=False) + order.get_locked_quantity = mock.Mock(return_value=locked_quantity) + order.get_computed_fee = mock.Mock(return_value=None) + return order + @staticmethod def _exchange_interface_for_compute( *, @@ -1115,6 +1135,7 @@ def _exchange_interface_for_compute( available_symbol: decimal.Decimal, available_market: decimal.Decimal, mark_price: decimal.Decimal, + open_mirrored_sell_orders: typing.Optional[list] = None, ): symbol_market = mock.Mock() market_quantity_total = total_market / mark_price if mark_price else trading_constants.ZERO @@ -1144,6 +1165,9 @@ def _exchange_interface_for_compute( side_effect=lambda symbol, quantity, limit_price: ([(quantity, limit_price)], symbol_market) ) exchange_if.orders.get_order_locked_amount = order_util.get_order_locked_amount + exchange_if.orders.get_open_orders = mock.Mock( + return_value=open_mirrored_sell_orders or [] + ) exchange_if.market.is_market_open_for_order_type = mock.Mock(return_value=True) return exchange_if @@ -1185,24 +1209,31 @@ async def run_compute(open_order): def test_sell_open_mirrored_order_adds_locked_base_to_cap(self): mark_price = decimal.Decimal("2000") + open_sell = self._mirrored_sell_order( + order_id="open-sell", + symbol="ETH/USDT", + locked_quantity=decimal.Decimal("1"), + price=mark_price, + ) + sibling_sell = self._mirrored_sell_order( + order_id="sibling-sell", + symbol="ETH/USDT", + locked_quantity=decimal.Decimal("8.95"), + price=mark_price, + ) exchange_if = self._exchange_interface_for_compute( total_symbol=decimal.Decimal("10"), total_market=decimal.Decimal("10000"), available_symbol=decimal.Decimal("0.05"), available_market=decimal.Decimal("500"), mark_price=mark_price, + open_mirrored_sell_orders=[open_sell, sibling_sell], ) synchronizer = orders_synchronizer_module.OrdersSynchronizer( _copied_account(), exchange_if, copy_entities.AccountCopySettings(), ) - open_sell = mock.Mock() - open_sell.side = trading_enums.TradeOrderSide.SELL - open_sell.symbol = "ETH/USDT" - open_sell.origin_price = mark_price - open_sell.get_locked_quantity = mock.Mock(return_value=decimal.Decimal("1")) - open_sell.get_computed_fee = mock.Mock(return_value=None) async def run_compute(open_order, scaled): return await synchronizer._compute_mirrored_quantity_type_and_price( @@ -1217,9 +1248,46 @@ async def run_compute(open_order, scaled): scaled = decimal.Decimal("2") ideal_without = asyncio.run(run_compute(None, scaled)).ideal_quantity ideal_with = asyncio.run(run_compute(open_sell, scaled)).ideal_quantity - assert ideal_without == decimal.Decimal("2") + assert ideal_without == decimal.Decimal("0.05") assert ideal_with == decimal.Decimal("1.05") + def test_new_sell_caps_to_total_minus_sibling_locked_base(self): + mark_price = decimal.Decimal("60300") + total_btc = decimal.Decimal("0.00753") + available_btc = decimal.Decimal("0.00068") + sibling_locked_btc = total_btc - available_btc + sibling_sell = self._mirrored_sell_order( + order_id="sibling-sell", + symbol="BTC/USDT", + locked_quantity=sibling_locked_btc, + price=decimal.Decimal("62188"), + ) + exchange_if = self._exchange_interface_for_compute( + total_symbol=total_btc, + total_market=decimal.Decimal("500"), + available_symbol=available_btc, + available_market=decimal.Decimal("15"), + mark_price=mark_price, + open_mirrored_sell_orders=[sibling_sell], + ) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + _copied_account(), + exchange_if, + copy_entities.AccountCopySettings(), + ) + + async def run_compute(): + return await synchronizer._compute_mirrored_quantity_type_and_price( + "BTC/USDT", + trading_enums.TradeOrderSide.SELL, + decimal.Decimal("0.00074"), + decimal.Decimal("61188"), + trading_enums.TraderOrderType.SELL_LIMIT, + open_mirrored_order=None, + ) + + assert asyncio.run(run_compute()).ideal_quantity == available_btc + class TestMirroredOrderSkipLogging: @staticmethod diff --git a/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_live_portfolio_sync.py b/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_live_portfolio_sync.py index c3536de837..7a1c66fe83 100644 --- a/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_live_portfolio_sync.py +++ b/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_live_portfolio_sync.py @@ -45,7 +45,16 @@ def _load_copy_tests_python_helpers(): pytestmark = pytest.mark.asyncio _BTC_USDT = "BTC/USDT" -_BTC_PRICE = decimal.Decimal("60000") +_BTC_PRICE = decimal.Decimal("60300") +_RECREATE_SELL_ORDER_ID = "4bd49d83-20d5-4258-8298-55da4bac60e7" +_SIBLING_SELL_ORDER_ID = "978f4fc0-d60a-42e5-8657-8973e7a53999" +_COUPLER_BTC_TOTAL = decimal.Decimal("0.00753") +_COUPLER_BTC_AVAILABLE = decimal.Decimal("0.00068") +_SIBLING_SELL_QUANTITY = _COUPLER_BTC_TOTAL - _COUPLER_BTC_AVAILABLE +_REFERENCE_BTC_TOTAL = decimal.Decimal("0.00753") +_REFERENCE_SELL_QUANTITY = decimal.Decimal("0.00074") +_REFERENCE_SELL_PRICE = decimal.Decimal("61188") +_SELL_SCENARIO_USDT_TOTAL = decimal.Decimal("500") _REPLACE_ORDER_ID = "dabfe054-a650-4a09-a296-8d22ceb6f664" _CREATE_ORDER_ID = "7cf7e7ad-b3e4-4f6a-a2f5-1ecff2cc176f" # Copier open buy before sync (oversized vs reference target); locks ~19.8 USDT at _BTC_PRICE. @@ -183,3 +192,121 @@ async def test_creates_buy_after_downsize_replace_frees_stale_quote(self, live_t assert replace_order.origin_quantity == decimal.Decimal("0.0002") assert create_order is not None assert create_order.origin_quantity == decimal.Decimal("0.00015") + + +def _replicable_btc_sell_limit_order( + *, + order_id: str, + quantity: decimal.Decimal, + price: decimal.Decimal, +) -> protocol_models.Order: + return protocol_models.Order( + id=order_id, + symbol=_BTC_USDT, + price=float(price), + quantity=float(quantity), + filled=0.0, + exchange_id="reference-exchange-id", + side=protocol_models.Side.SELL, + type=protocol_models.OrderType.LIMIT, + trigger_above=True, + reduce_only=False, + is_active=True, + status=protocol_models.OrderStatus.OPEN, + created_at=timestamp_util.utc_datetime_from_timestamp(time.time()), + ) + + +def _reference_account_with_one_sell() -> protocol_models.CopiedAccount: + return protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=time.time(), + copied_assets=[ + protocol_models.CopiedAsset( + name="BTC", + total=float(_REFERENCE_BTC_TOTAL), + available=float(_COUPLER_BTC_AVAILABLE), + ratio=0.5, + ), + protocol_models.CopiedAsset( + name="USDT", + total=float(_SELL_SCENARIO_USDT_TOTAL), + available=float(_SELL_SCENARIO_USDT_TOTAL), + ratio=0.5, + ), + ], + orders=[ + _replicable_btc_sell_limit_order( + order_id=_RECREATE_SELL_ORDER_ID, + quantity=_REFERENCE_SELL_QUANTITY, + price=_REFERENCE_SELL_PRICE, + ), + _replicable_btc_sell_limit_order( + order_id=_SIBLING_SELL_ORDER_ID, + quantity=_SIBLING_SELL_QUANTITY, + price=decimal.Decimal("62188"), + ), + ], + ) + + +class TestSynchronizeGraceElapsedSellRecreate: + @pytest.mark.parametrize("backtesting_config", ["USDT"], indirect=True) + async def test_creates_sell_after_grace_elapsed_respects_available_base(self, live_trading_trader): + # Jul 2 log: sibling sells lock most BTC; grace-elapsed resync recreates a missing mirrored + # sell. Sizing must cap to available base (total minus sibling locks), not raw total. + _config, exchange_manager, _trader = live_trading_trader + copy_tests_python_helpers.ensure_traded_symbol_pairs(exchange_manager, (_BTC_USDT,)) + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + + trading_api.force_set_mark_price(exchange_manager, _BTC_USDT, _BTC_PRICE) + portfolio_manager.portfolio.update_portfolio_from_balance( + { + "BTC": { + "available": _COUPLER_BTC_TOTAL, + "total": _COUPLER_BTC_TOTAL, + }, + "USDT": { + "available": _SELL_SCENARIO_USDT_TOTAL, + "total": _SELL_SCENARIO_USDT_TOTAL, + }, + }, + True, + ) + portfolio_manager.handle_balance_updated() + portfolio_manager.portfolio_value_holder.value_converter.missing_currency_data_in_exchange.discard("USDT") + portfolio_manager.handle_mark_price_update(_BTC_USDT, _BTC_PRICE) + + exchange_interface = copy_exchange.ExchangeInterface(exchange_manager) + await exchange_interface.orders.create_order( + trading_enums.TraderOrderType.SELL_LIMIT, + _BTC_USDT, + decimal.Decimal("62188"), + _SIBLING_SELL_QUANTITY, + decimal.Decimal("62188"), + tag=copy_constants.MIRRORED_ORDER_TAG, + order_id=_SIBLING_SELL_ORDER_ID, + wait_for_creation=True, + ) + + portfolio_manager.portfolio.get_currency_portfolio("BTC").available = _COUPLER_BTC_AVAILABLE + + reference_account = _reference_account_with_one_sell() + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference_account, + exchange_interface, + copy_entities.AccountCopySettings(), + ) + synchronizer.abort_mirrored_orphan_grace() + + refresh_portfolio_mock = mock.AsyncMock(return_value=True) + with mock.patch.object( + exchange_interface.portfolio, + "refresh_portfolio", + refresh_portfolio_mock, + ): + await synchronizer.synchronize() + + created_order = _open_order_by_id(exchange_manager, _RECREATE_SELL_ORDER_ID) + assert created_order is not None + assert created_order.origin_quantity == _COUPLER_BTC_AVAILABLE diff --git a/packages/flow/octobot_flow/entities/automations/fetched_dependencies.py b/packages/flow/octobot_flow/entities/automations/fetched_dependencies.py index 3559ee477d..6b96267559 100644 --- a/packages/flow/octobot_flow/entities/automations/fetched_dependencies.py +++ b/packages/flow/octobot_flow/entities/automations/fetched_dependencies.py @@ -10,3 +10,4 @@ class FetchedDependencies(octobot_commons.dataclasses.MinimizableDataclass): fetched_exchange_data: typing.Optional[fetched_exchange_data_import.FetchedExchangeData] = None fetched_copy_trading_data: typing.Optional[fetched_copy_trading_data_import.FetchedCopyTradingData] = None + skip_exchange: bool = False diff --git a/packages/flow/octobot_flow/jobs/automation_job.py b/packages/flow/octobot_flow/jobs/automation_job.py index 1fa3f381df..337c34abb1 100644 --- a/packages/flow/octobot_flow/jobs/automation_job.py +++ b/packages/flow/octobot_flow/jobs/automation_job.py @@ -232,6 +232,7 @@ async def _fetch_dependencies( return octobot_flow.entities.FetchedDependencies( fetched_exchange_data=None, fetched_copy_trading_data=None, + skip_exchange=True, ) if fetched_copy_trading_data := await self._init_all_required_copy_trading_data( maybe_community_repository, to_execute_actions, minimal_profile_data, diff --git a/packages/flow/octobot_flow/jobs/automation_runner_job.py b/packages/flow/octobot_flow/jobs/automation_runner_job.py index 8c7f48ddae..4eb378875b 100644 --- a/packages/flow/octobot_flow/jobs/automation_runner_job.py +++ b/packages/flow/octobot_flow/jobs/automation_runner_job.py @@ -180,7 +180,10 @@ async def actions_context( raise octobot_flow.errors.AutomationValidationError( f"A bot_id is required to run a bot. Found: {self.profile_data_provider.get_profile_data().profile_details.bot_id}" ) - async with self.exchange_manager_context(): + if self.fetched_dependencies.skip_exchange: yield self + else: + async with self.exchange_manager_context(): + yield self finally: self._to_execute_actions = None # type: ignore diff --git a/packages/flow/octobot_flow/logic/actions/actions_executor.py b/packages/flow/octobot_flow/logic/actions/actions_executor.py index fa4c15a056..f70e1a0dd9 100644 --- a/packages/flow/octobot_flow/logic/actions/actions_executor.py +++ b/packages/flow/octobot_flow/logic/actions/actions_executor.py @@ -349,7 +349,7 @@ def _sync_after_execution( ): if synchronized_exchange_account_elements: self._get_logger().info( - f"Exchange account elements are being updated from {len(synchronized_exchange_account_elements)}" + f"Exchange account elements are being updated from {len(synchronized_exchange_account_elements)} " f"synchronized exchange account elements on {[s.name for s in synchronized_exchange_account_elements]}" f"returned by actions; this iteration does not apply sync_from_exchange_manager from the " f"local exchange_manager.", diff --git a/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py b/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py index 28e3bff197..9082dbf4be 100644 --- a/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py +++ b/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py @@ -88,6 +88,8 @@ async def _action_execution_error_handler_wrapper( octobot_flow.enums.ActionErrorStatus.BLOCKCHAIN_WALLET_ERROR.value, str(err), ) + except octobot_trading.errors.PortfolioNegativeValueError: + raise except Exception as err: octobot_commons.logging.get_logger("action_execution").exception( err, diff --git a/packages/flow/octobot_flow/logic/dsl/dsl_executor.py b/packages/flow/octobot_flow/logic/dsl/dsl_executor.py index f874f0bfb8..c7f9223737 100644 --- a/packages/flow/octobot_flow/logic/dsl/dsl_executor.py +++ b/packages/flow/octobot_flow/logic/dsl/dsl_executor.py @@ -5,6 +5,7 @@ import octobot_commons.signals import octobot_commons.errors import octobot_commons.profiles +import octobot_commons.profiles.profile_types.ephemeral_profile as ephemeral_profile_module import octobot_commons.logging import octobot_trading.exchanges import octobot_trading.dsl @@ -37,7 +38,9 @@ def __init__( super().__init__() self._exchange_manager = exchange_manager self._dependencies = dependencies - self._dependencies_config: dict = profile_data.to_profile("").config + self._dependencies_config: dict = ephemeral_profile_module.EphemeralProfile.from_profile_data( + profile_data + ).config self._interpreter_signals: octobot_commons.dsl_interpreter.OperatorSignals = None # type: ignore (reset when interpreter is created) self._interpreter: octobot_commons.dsl_interpreter.Interpreter = self._create_interpreter( None, executor_id diff --git a/packages/flow/tests/functionnal_tests/conftest.py b/packages/flow/tests/functionnal_tests/conftest.py index 1264acfeb6..6d00cbc9a7 100644 --- a/packages/flow/tests/functionnal_tests/conftest.py +++ b/packages/flow/tests/functionnal_tests/conftest.py @@ -1,5 +1,10 @@ +import os +import pathlib + import pytest +import octobot_commons.constants as commons_constants + import tests.functionnal_tests as functionnal_tests @@ -7,3 +12,21 @@ def _mock_local_user_configuration(): with functionnal_tests.mocked_local_user_configuration(): yield + + +@pytest.fixture(autouse=True) +def _assert_master_user_config_unchanged(request): + if not os.path.isfile(os.path.join(os.getcwd(), "start.py")): + yield + return + master_config_path = pathlib.Path(commons_constants.USER_FOLDER) / commons_constants.CONFIG_FILE + if not master_config_path.is_file(): + yield + return + config_bytes_before = master_config_path.read_bytes() + yield + config_bytes_after = master_config_path.read_bytes() + assert config_bytes_before == config_bytes_after, ( + f"master user config must not be modified during functional test " + f"{request.node.nodeid!r}: {master_config_path}" + ) diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py index 40e1d6085d..8dd0b6a534 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py @@ -2,9 +2,11 @@ # Shared helpers/constants for octobot process functional tests (run_octobot_process, GridTradingMode). import asyncio +import contextlib import copy import decimal import json +import mock import os import pathlib import time @@ -19,6 +21,7 @@ import pytest import octobot_flow.jobs +import octobot_flow.jobs.automation_runner_job as automation_runner_job_module import octobot_flow.entities import octobot_flow.environment import octobot_flow.enums @@ -50,7 +53,54 @@ WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC = 2 RECALL_SCHEDULE_TOLERANCE_SEC = 1.5 + +class ExchangeManagerContextTracker: + def __init__(self) -> None: + self.entered_count = 0 + + +@contextlib.contextmanager +def track_exchange_manager_context() -> typing.Iterator[ExchangeManagerContextTracker]: + tracker = ExchangeManagerContextTracker() + + @contextlib.asynccontextmanager + async def counting_exchange_manager_context(self): + tracker.entered_count += 1 + self._exchange_manager = mock.Mock() + yield self._exchange_manager + + with mock.patch.object( + automation_runner_job_module.AutomationRunnerJob, + "exchange_manager_context", + counting_exchange_manager_context, + ): + yield tracker + + +def assert_exchange_manager_not_initialized(tracker: ExchangeManagerContextTracker) -> None: + assert tracker.entered_count == 0, ( + "expected exchange_manager_context to be skipped for process-bound actions, " + f"but it was entered {tracker.entered_count} time(s)" + ) + + +async def run_automation_job_without_exchange_manager( + automation_state: dict, + priority_actions: list, + updated_trading_signals: list, + auth_details: dict, +) -> octobot_flow.jobs.AutomationJob: + with track_exchange_manager_context() as tracker: + async with octobot_flow.jobs.AutomationJob( + automation_state, priority_actions, updated_trading_signals, auth_details + ) as automation_job: + await automation_job.run() + assert_exchange_manager_not_initialized(tracker) + return automation_job + + EXCHANGE_BINANCEUS = "binanceus" +FUNCTIONAL_TEST_USER_ID = "wallet-user" # --- DSL / DAG action ids (fixtures, dependencies, _get_action_by_id) --- ACTION_ID_INIT = "action_init" diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py index 22b3147366..1e066049a3 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py @@ -82,6 +82,7 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( run_dsl = ( "run_octobot_process(" f"{user_folder!r}, {repr(profile_2x2)}, " + f"user_id={octobot_process_functional_shared.FUNCTIONAL_TEST_USER_ID!r}, " f"waiting_time={octobot_process_functional_shared.WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC}, ping_timeout=30.0)" ) run_action = { @@ -146,8 +147,9 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC inner: typing.Optional[dict] = None # 2) First automation pass, then poll until the child reports init_state_ok (ready to query). - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as first_poll: - await first_poll.run() + first_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( first_poll.dump() ) @@ -160,8 +162,9 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( if not (inner and inner.get("init_state_ok") is True): while time.monotonic() < deadline: await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as poll_job: - await poll_job.run() + poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( poll_job.dump() ) @@ -196,9 +199,10 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( ] = None last_open_order_count = 0 while time.monotonic() < orders_deadline: - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as grid_poll_job: - await grid_poll_job.run() - job_dump_payload = grid_poll_job.dump() + grid_poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) + job_dump_payload = grid_poll_job.dump() octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( job_dump_payload ) @@ -235,6 +239,7 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( new_run_dsl = ( "run_octobot_process(" f"{user_folder!r}, {repr(profile_3x3)}, " + f"user_id={octobot_process_functional_shared.FUNCTIONAL_TEST_USER_ID!r}, " f"waiting_time={octobot_process_functional_shared.WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC}, ping_timeout=30.0)" ) update_config_priority_action = { @@ -263,9 +268,10 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( last_six_count = 0 inner_after: typing.Optional[dict] = None while time.monotonic() < six_orders_deadline: - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as six_poll: - await six_poll.run() - dump_payload = six_poll.dump() + six_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) + dump_payload = six_poll.dump() octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( dump_payload ) diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py index 87db26013c..b6d5920c02 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py @@ -45,6 +45,7 @@ async def test_run_octobot_process_lifecycle_grid_trading( run_dsl = ( "run_octobot_process(" f"{user_folder!r}, {repr(octobot_process_functional_shared.GRID_BINANCEUS_PROFILE_DATA)}, " + f"user_id={octobot_process_functional_shared.FUNCTIONAL_TEST_USER_ID!r}, " f"waiting_time={octobot_process_functional_shared.WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC}, ping_timeout=30.0)" ) run_action = { @@ -114,8 +115,9 @@ async def test_run_octobot_process_lifecycle_grid_trading( deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC inner: typing.Optional[dict] = None # Run DSL job once, then optionally poll until recall payload shows init_state_ok. - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as first_poll: - await first_poll.run() + first_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( first_poll.dump() ) @@ -128,8 +130,9 @@ async def test_run_octobot_process_lifecycle_grid_trading( if not (inner and inner.get("init_state_ok") is True): while time.monotonic() < deadline: await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as poll_job: - await poll_job.run() + poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( poll_job.dump() ) @@ -165,9 +168,10 @@ async def test_run_octobot_process_lifecycle_grid_trading( ] = None last_open_order_count = 0 while time.monotonic() < orders_deadline: - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as grid_poll_job: - await grid_poll_job.run() - job_dump_payload = grid_poll_job.dump() + grid_poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) + job_dump_payload = grid_poll_job.dump() octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( job_dump_payload ) @@ -245,8 +249,9 @@ async def test_run_octobot_process_lifecycle_grid_trading( # 3) Second automation run: re-call path only (no second Popen; same child pid). before = popen_calls["count"] - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as idem_job: - await idem_job.run() + idem_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( idem_job.dump() ) @@ -329,6 +334,7 @@ async def test_run_octobot_process_lifecycle_default_config_no_profile_data( ] run_dsl = ( f"run_octobot_process({user_folder!r}, " + f"user_id={octobot_process_functional_shared.FUNCTIONAL_TEST_USER_ID!r}, " f"exchange_auth_data={dsl_interpreter.format_parameter_value(exchange_auth)}, " f"waiting_time={octobot_process_functional_shared.WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC}, ping_timeout=30.0)" ) @@ -391,8 +397,9 @@ async def test_run_octobot_process_lifecycle_default_config_no_profile_data( deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC inner: typing.Optional[dict] = None - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as first_poll: - await first_poll.run() + first_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( first_poll.dump() ) @@ -405,8 +412,9 @@ async def test_run_octobot_process_lifecycle_default_config_no_profile_data( if not (inner and inner.get("init_state_ok") is True): while time.monotonic() < deadline: await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as poll_job: - await poll_job.run() + poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( poll_job.dump() ) @@ -441,13 +449,27 @@ async def test_run_octobot_process_lifecycle_default_config_no_profile_data( exchange_auth_entry["api_key"], exchange_auth_entry["api_secret"], ) - profile_json_path = ( + root_cfg = json.loads( + (user_root / common_constants.CONFIG_FILE).read_text(encoding="utf-8") + ) + expected_readonly_profiles_path = os.path.normpath( + os.path.join( + os.getcwd(), + common_constants.USER_FOLDER, + common_constants.PROFILES_FOLDER, + ) + ) + assert ( + root_cfg[common_constants.CONFIG_READONLY_PROFILES_PATH] + == expected_readonly_profiles_path + ) + local_non_trading_profile_json = ( user_root / common_constants.PROFILES_FOLDER / "non-trading" / common_constants.PROFILE_CONFIG_FILE ) - assert profile_json_path.is_file() + assert not local_non_trading_profile_json.exists() # First process_bot_state dump can lag init_state_ok (see shared wait helper). state_path = octobot_process_functional_shared._process_bot_state_path(inner) diff --git a/packages/flow/tests/jobs/test_automation_job.py b/packages/flow/tests/jobs/test_automation_job.py index 4e50da9cc6..a6644c44c0 100644 --- a/packages/flow/tests/jobs/test_automation_job.py +++ b/packages/flow/tests/jobs/test_automation_job.py @@ -7,6 +7,7 @@ import octobot_protocol.models as protocol_models import octobot_flow.entities +import octobot_flow.entities.actions.action_details as action_details import octobot_flow.errors import octobot_flow.jobs.automation_job as automation_job_module import octobot_flow.logic.actions @@ -17,6 +18,9 @@ STRATEGY_ID = "test-strategy-id" +_PROCESS_BOUND_DSL_SCRIPT = ( + "run_octobot_process('bots/b1', user_id='user_1', waiting_time=1.0, ping_timeout=30.0)" +) def _minimal_automation_job() -> automation_job_module.AutomationJob: @@ -30,6 +34,41 @@ def _minimal_automation_job() -> automation_job_module.AutomationJob: return automation_job_module.AutomationJob(automation_state, [], [], auth_details) +def _dsl_action(dsl_script: str, *, action_id: str = "action_dsl") -> action_details.DSLScriptActionDetails: + return action_details.DSLScriptActionDetails( + id=action_id, + dsl_script=dsl_script, + dependencies=[], + resolved_dsl_script=dsl_script, + ) + + +def _automation_job_with_exchange_dag( + *dag_actions: action_details.AbstractActionDetails, +) -> automation_job_module.AutomationJob: + automation_state = octobot_flow.entities.AutomationState.from_dict( + { + "exchange_account_details": { + "exchange_details": {"internal_name": "binanceus"}, + "auth_details": {}, + "portfolio": {}, + }, + "automation": { + "metadata": {"automation_id": "automation_1"}, + "actions_dag": {"actions": []}, + }, + } + ) + automation_state.automation.actions_dag.actions = list(dag_actions) + user_auth_details = octobot_flow.entities.UserAuthentication(wallet_address="0xtest") + return automation_job_module.AutomationJob( + automation_state.to_dict(include_default_values=False), + [], + [], + user_auth_details, + ) + + def _minimal_copied_account() -> protocol_models.CopiedAccount: return protocol_models.CopiedAccount( version=copy_constants.COPIED_ACCOUNT_VERSION, @@ -91,3 +130,33 @@ async def test_skips_emission_and_logs_when_wallet_not_found(self): assert emitted_signal.strategy_id == STRATEGY_ID assert emitted_signal.account is copied_account error_log_mock.assert_called_once_with(f"Skipping trading signal emission: {wallet_error}") + + +class TestFetchDependencies: + @pytest.mark.asyncio + async def test_sets_skip_exchange_when_executable_dag_is_process_bound_only(self): + process_bound_action = _dsl_action(_PROCESS_BOUND_DSL_SCRIPT, action_id="action_run") + automation_job = _automation_job_with_exchange_dag(process_bound_action) + + fetched_dependencies = await automation_job._fetch_dependencies( + None, + [process_bound_action], + ) + + assert fetched_dependencies.skip_exchange is True + assert fetched_dependencies.fetched_exchange_data is None + assert fetched_dependencies.fetched_copy_trading_data is None + + @pytest.mark.asyncio + async def test_does_not_set_skip_exchange_when_no_executable_dag_actions(self): + completed_process_action = _dsl_action(_PROCESS_BOUND_DSL_SCRIPT, action_id="action_run") + completed_process_action.executed_at = time.time() + stop_automation_action = _dsl_action("stop_automation()", action_id="action_stop") + automation_job = _automation_job_with_exchange_dag(completed_process_action) + + fetched_dependencies = await automation_job._fetch_dependencies( + None, + [stop_automation_action], + ) + + assert fetched_dependencies.skip_exchange is False diff --git a/packages/flow/tests/jobs/test_automation_runner_job_actions_context.py b/packages/flow/tests/jobs/test_automation_runner_job_actions_context.py new file mode 100644 index 0000000000..3420f496db --- /dev/null +++ b/packages/flow/tests/jobs/test_automation_runner_job_actions_context.py @@ -0,0 +1,171 @@ +# Drakkar-Software OctoBot-Flow + +import contextlib +import time +import typing + +import mock +import pytest + +import octobot_flow.entities +import octobot_flow.entities.actions.action_details as action_details +import octobot_flow.jobs.automation_runner_job as automation_runner_job_module + + +def _automation_state() -> octobot_flow.entities.AutomationState: + return octobot_flow.entities.AutomationState.from_dict( + { + "exchange_account_details": { + "exchange_details": {"internal_name": "binanceus"}, + "auth_details": {}, + "portfolio": {}, + }, + "automation": { + "metadata": {"automation_id": "automation_1"}, + "actions_dag": {"actions": []}, + }, + } + ) + + +def _fetched_dependencies(*, skip_exchange: bool = False) -> octobot_flow.entities.FetchedDependencies: + return octobot_flow.entities.FetchedDependencies(skip_exchange=skip_exchange) + + +def _runner_job( + *, + skip_exchange: bool = False, + automation_state: octobot_flow.entities.AutomationState | None = None, +) -> automation_runner_job_module.AutomationRunnerJob: + return automation_runner_job_module.AutomationRunnerJob( + automation_state or _automation_state(), + _fetched_dependencies(skip_exchange=skip_exchange), + None, + 0.0, + ) + + +def _dsl_action(dsl_script: str) -> action_details.DSLScriptActionDetails: + return action_details.DSLScriptActionDetails( + id="action_dsl", + dsl_script=dsl_script, + dependencies=[], + resolved_dsl_script=dsl_script, + ) + + +@contextlib.asynccontextmanager +async def _track_exchange_manager_context( + entered_calls: list[bool], +) -> typing.AsyncGenerator[None, None]: + @contextlib.asynccontextmanager + async def counting_exchange_manager_context(self): + entered_calls.append(True) + self._exchange_manager = mock.Mock() + yield self._exchange_manager + + with mock.patch.object( + automation_runner_job_module.AutomationRunnerJob, + "exchange_manager_context", + counting_exchange_manager_context, + ): + yield + + +def _automation_state_with_dag( + *dag_actions: action_details.AbstractActionDetails, +) -> octobot_flow.entities.AutomationState: + automation_state = _automation_state() + automation_state.automation.actions_dag.actions = list(dag_actions) + return automation_state + + +class TestAutomationRunnerJobActionsContext: + @pytest.mark.asyncio + async def test_skips_exchange_manager_for_process_bound_actions(self): + runner_job = _runner_job(skip_exchange=True) + entered_calls: list[bool] = [] + process_bound_action = _dsl_action( + "run_octobot_process('bots/b1', user_id='user_1', waiting_time=1.0, ping_timeout=30.0)" + ) + + async with _track_exchange_manager_context(entered_calls): + async with runner_job.actions_context([process_bound_action], True): + pass + + assert entered_calls == [] + assert runner_job._exchange_manager is None + + @pytest.mark.asyncio + async def test_enters_exchange_manager_for_non_process_bound_dsl_action(self): + runner_job = _runner_job() + entered_calls: list[bool] = [] + wait_action = _dsl_action("wait(1.0, 1.0)") + + async with _track_exchange_manager_context(entered_calls): + async with runner_job.actions_context([wait_action], True): + pass + + assert len(entered_calls) == 1 + + @pytest.mark.asyncio + async def test_enters_exchange_manager_for_configured_action(self): + runner_job = _runner_job() + entered_calls: list[bool] = [] + configured_action = action_details.ConfiguredActionDetails(id="action_init") + + async with _track_exchange_manager_context(entered_calls): + async with runner_job.actions_context([configured_action], True): + pass + + assert len(entered_calls) == 1 + + @pytest.mark.asyncio + async def test_enters_exchange_manager_for_empty_actions(self): + runner_job = _runner_job() + entered_calls: list[bool] = [] + + async with _track_exchange_manager_context(entered_calls): + async with runner_job.actions_context([], True): + pass + + assert len(entered_calls) == 1 + + @pytest.mark.asyncio + async def test_skips_exchange_manager_for_stop_automation_on_process_bound_dag(self): + process_bound_dag_action = _dsl_action( + "run_octobot_process('bots/b1', user_id='user_1', waiting_time=1.0, ping_timeout=30.0)" + ) + runner_job = _runner_job( + skip_exchange=True, + automation_state=_automation_state_with_dag(process_bound_dag_action), + ) + entered_calls: list[bool] = [] + stop_automation_action = _dsl_action("stop_automation()") + + async with _track_exchange_manager_context(entered_calls): + async with runner_job.actions_context([stop_automation_action], True): + pass + + assert entered_calls == [] + assert runner_job._exchange_manager is None + + @pytest.mark.asyncio + async def test_skips_exchange_manager_for_stop_automation_when_dag_action_between_recalls(self): + process_bound_dag_action = _dsl_action( + "run_octobot_process('bots/b1', user_id='user_1', waiting_time=1.0, ping_timeout=30.0)" + ) + process_bound_dag_action.executed_at = time.time() + runner_job = _runner_job( + skip_exchange=True, + automation_state=_automation_state_with_dag(process_bound_dag_action), + ) + entered_calls: list[bool] = [] + stop_automation_action = _dsl_action("stop_automation()") + + async with _track_exchange_manager_context(entered_calls): + async with runner_job.actions_context([stop_automation_action], True): + pass + + assert entered_calls == [] + assert runner_job._exchange_manager is None diff --git a/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py b/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py new file mode 100644 index 0000000000..b578bec604 --- /dev/null +++ b/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py @@ -0,0 +1,138 @@ +import pytest + +import octobot_commons.errors +import octobot_trading.enums +import octobot_trading.errors + +import octobot_flow.entities +import octobot_flow.enums +import octobot_flow.logic.dsl.dsl_action_execution_context + + +_PORTFOLIO_NEGATIVE_VALUE_ERROR_MESSAGE = ( + "Trying to update BTC with -0.00074 but quantity was 0.00068" +) +_DISABLED_FUNDS_TRANSFER_ERROR_MESSAGE = "Funds transfer is disabled" +_MISSING_MINIMAL_EXCHANGE_TRADE_VOLUME_MESSAGE = "Order volume below exchange minimum" +_UNSUPPORTED_HEDGE_CONTRACT_MESSAGE = "Hedge mode is not supported for this contract" +_INVALID_POSITION_SIDE_MESSAGE = "Invalid position side for this order" +_EXCHANGE_ACCOUNT_SYMBOL_PERMISSION_MESSAGE = "Symbol is not allowed on this account" +_INVALID_PARAMETER_FORMAT_MESSAGE = "Invalid signal parameter format" +_NOT_SUPPORTED_STOP_LOSS_ORDER_MESSAGE = "STOP_LOSS orders are not supported on binance" +_NOT_SUPPORTED_BUY_MARKET_ORDER_MESSAGE = "BUY_MARKET orders are not supported on binance" +_BLOCKCHAIN_WALLET_ERROR_MESSAGE = "Blockchain wallet connection failed" +_GENERIC_EXCEPTION_MESSAGE = "Unexpected DSL execution failure" + + +class TestDslActionExecutionReraisesPortfolioNegativeValueError: + @pytest.mark.asyncio + async def test_reraises_portfolio_negative_value_error(self): + class StubExecutor: + @octobot_flow.logic.dsl.dsl_action_execution_context.dsl_action_execution + async def execute_action(self, action, **_kwargs): + raise octobot_trading.errors.PortfolioNegativeValueError( + _PORTFOLIO_NEGATIVE_VALUE_ERROR_MESSAGE + ) + + action = octobot_flow.entities.DSLScriptActionDetails( + id="copy_1", + dsl_script="copy_exchange_account()", + ) + stub_executor = StubExecutor() + + with pytest.raises(octobot_trading.errors.PortfolioNegativeValueError) as raised_error: + await stub_executor.execute_action(action) + + assert str(raised_error.value) == _PORTFOLIO_NEGATIVE_VALUE_ERROR_MESSAGE + assert action.error_status is None + assert action.error_message is None + + +class TestDslActionExecutionMapsCaughtException: + @pytest.mark.asyncio + @pytest.mark.parametrize( + "raised_exception,expected_error_status", + [ + pytest.param( + octobot_trading.errors.DisabledFundsTransferError(_DISABLED_FUNDS_TRANSFER_ERROR_MESSAGE), + octobot_flow.enums.ActionErrorStatus.DISABLED_FUNDS_TRANSFER_ERROR, + id="disabled_funds_transfer", + ), + pytest.param( + octobot_trading.errors.MissingMinimalExchangeTradeVolume( + _MISSING_MINIMAL_EXCHANGE_TRADE_VOLUME_MESSAGE + ), + octobot_flow.enums.ActionErrorStatus.INVALID_ORDER, + id="missing_minimal_exchange_trade_volume", + ), + pytest.param( + octobot_trading.errors.UnsupportedHedgeContractError(_UNSUPPORTED_HEDGE_CONTRACT_MESSAGE), + octobot_flow.enums.ActionErrorStatus.UNSUPPORTED_HEDGE_POSITION, + id="unsupported_hedge_contract", + ), + pytest.param( + octobot_trading.errors.InvalidPositionSide(_INVALID_POSITION_SIDE_MESSAGE), + octobot_flow.enums.ActionErrorStatus.UNSUPPORTED_HEDGE_POSITION, + id="invalid_position_side", + ), + pytest.param( + octobot_trading.errors.ExchangeAccountSymbolPermissionError( + _EXCHANGE_ACCOUNT_SYMBOL_PERMISSION_MESSAGE + ), + octobot_flow.enums.ActionErrorStatus.SYMBOL_INCOMPATIBLE_WITH_ACCOUNT, + id="exchange_account_symbol_permission", + ), + pytest.param( + octobot_commons.errors.InvalidParameterFormatError(_INVALID_PARAMETER_FORMAT_MESSAGE), + octobot_flow.enums.ActionErrorStatus.INVALID_SIGNAL_FORMAT, + id="invalid_parameter_format", + ), + pytest.param( + octobot_trading.errors.NotSupportedOrderTypeError( + _NOT_SUPPORTED_STOP_LOSS_ORDER_MESSAGE, + octobot_trading.enums.TraderOrderType.STOP_LOSS, + ), + octobot_flow.enums.ActionErrorStatus.UNSUPPORTED_STOP_ORDER, + id="not_supported_order_type_stop_loss", + ), + pytest.param( + octobot_trading.errors.NotSupportedOrderTypeError( + _NOT_SUPPORTED_BUY_MARKET_ORDER_MESSAGE, + octobot_trading.enums.TraderOrderType.BUY_MARKET, + ), + octobot_flow.enums.ActionErrorStatus.INVALID_ORDER, + id="not_supported_order_type_buy_market", + ), + pytest.param( + octobot_trading.errors.BlockchainWalletError(_BLOCKCHAIN_WALLET_ERROR_MESSAGE), + octobot_flow.enums.ActionErrorStatus.BLOCKCHAIN_WALLET_ERROR, + id="blockchain_wallet", + ), + pytest.param( + RuntimeError(_GENERIC_EXCEPTION_MESSAGE), + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR, + id="generic_exception", + ), + ], + ) + async def test_maps_caught_exception_to_action_error_status( + self, + raised_exception, + expected_error_status, + ): + class StubExecutor: + @octobot_flow.logic.dsl.dsl_action_execution_context.dsl_action_execution + async def execute_action(self, action, **_kwargs): + raise raised_exception + + action = octobot_flow.entities.DSLScriptActionDetails( + id="action_1", + dsl_script="True", + resolved_dsl_script="True", + ) + stub_executor = StubExecutor() + + await stub_executor.execute_action(action) + + assert action.error_status == expected_error_status.value + assert action.error_message == str(raised_exception) diff --git a/packages/node/octobot_node/constants.py b/packages/node/octobot_node/constants.py index 793cfaa23c..65a1e7b45c 100644 --- a/packages/node/octobot_node/constants.py +++ b/packages/node/octobot_node/constants.py @@ -30,6 +30,10 @@ AUTOMATION_WORKFLOW_MAX_ITERATION_RETRIES = int(os.getenv("AUTOMATION_WORKFLOW_MAX_ITERATION_RETRIES", 19)) AUTOMATION_WORKFLOW_BACKOFF_RATE = float(os.getenv("AUTOMATION_WORKFLOW_BACKOFF_RATE", 1.5)) +DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS = 60.0 * 60.0 # 1 hour + +NODE_API_STOP_TIMEOUT_SECONDS = float(os.getenv("NODE_API_STOP_TIMEOUT_SECONDS", "5.0")) + # delay between authentication errors retry attempts of the current iteration (30 minutes) INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS = float(os.getenv("INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS", 1800)) diff --git a/packages/node/octobot_node/errors.py b/packages/node/octobot_node/errors.py index 72ab30bad2..d247a58664 100644 --- a/packages/node/octobot_node/errors.py +++ b/packages/node/octobot_node/errors.py @@ -15,6 +15,7 @@ # License along with this library. import octobot_flow.enums +import octobot_node.constants as constants class WorkflowError(Exception): """Base class for all workflow errors""" @@ -104,3 +105,7 @@ class UnknownTradingTypeError(UserActionError): class AmbiguousTradingTypeError(UserActionError): """Raised when multiple trading types are found for an account.""" + + +class UnrestartableAutomationError(UserActionError): + """Raised when an automation cannot be restarted.""" diff --git a/packages/node/octobot_node/scheduler/__init__.py b/packages/node/octobot_node/scheduler/__init__.py index fa819b5d7a..d0cf983b24 100644 --- a/packages/node/octobot_node/scheduler/__init__.py +++ b/packages/node/octobot_node/scheduler/__init__.py @@ -23,6 +23,8 @@ SCHEDULER: scheduler_lib.Scheduler = scheduler_lib.Scheduler() +_shutdown_done = False + def is_enabled() -> bool: return SCHEDULER.is_enabled() @@ -33,6 +35,8 @@ def is_initialized() -> bool: def initialize_scheduler(): + global _shutdown_done + _shutdown_done = False scheduler_logger.info("Initializing scheduler") SCHEDULER.create() octobot_node.scheduler.workflows.register_workflows() @@ -40,9 +44,13 @@ def initialize_scheduler(): async def shutdown_scheduler_and_trading_signal_channel() -> None: + global _shutdown_done + if _shutdown_done or not is_initialized(): + return try: import octobot_flow.repositories.community.trading_signals_channel as trading_signals_channel await trading_signals_channel.shutdown_internal_trading_signal_channel() except ImportError: pass SCHEDULER.stop() + _shutdown_done = True diff --git a/packages/node/octobot_node/scheduler/scheduler.py b/packages/node/octobot_node/scheduler/scheduler.py index e5359c3291..e0530bc161 100644 --- a/packages/node/octobot_node/scheduler/scheduler.py +++ b/packages/node/octobot_node/scheduler/scheduler.py @@ -122,7 +122,7 @@ def _get_dbos_workflow_id() -> typing.Optional[str]: if workflow_id := getattr(dbos.DBOS, "workflow_id", None): # group children workflows and parent workflows together # (a child workflow has the parent's workflow ID as a prefix) - return workflow_id[:octobot_node.constants.PARENT_WORKFLOW_ID_LENGTH] + return workflows_util.normalize_parent_automation_id(workflow_id) return None def is_enabled(self) -> bool: @@ -145,11 +145,13 @@ def start(self): self.logger.warning("Scheduler not initialized") def stop(self) -> None: - if self.INSTANCE: - self.INSTANCE.destroy() - self.logger.info("Scheduler stopped") - else: - self.logger.warning("Scheduler not initialized") + if not self.INSTANCE: + return + self.INSTANCE.destroy() + self.logger.info("Scheduler stopped") + Scheduler.INSTANCE = None + Scheduler.AUTOMATION_WORKFLOW_QUEUE = None + Scheduler.USER_ACTION_QUEUE = None def create_queues(self): self.AUTOMATION_WORKFLOW_QUEUE = dbos.Queue(name=octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value) @@ -222,13 +224,13 @@ async def _get_parent_and_children_automation_workflows( user_id, statuses, [octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value], load_output ) parent_workflow_ids = set( - workflow_id[:octobot_node.constants.PARENT_WORKFLOW_ID_LENGTH] + workflows_util.normalize_parent_automation_id(workflow_id) for workflow_id in workflow_ids ) return [ workflow for workflow in all_workflows - if workflow.workflow_id[:octobot_node.constants.PARENT_WORKFLOW_ID_LENGTH] in parent_workflow_ids + if workflows_util.normalize_parent_automation_id(workflow.workflow_id) in parent_workflow_ids ] async def _get_parent_and_children_automation_workflow_ids( @@ -290,6 +292,37 @@ async def resolve_active_automation_workflow_ids_for_parent_id( latest_workflow = workflows_util.get_latest_child_workflow(matching_workflows) return [latest_workflow.workflow_id] + async def resolve_latest_terminal_automation_workflow_for_parent_id( + self, + user_id: typing.Optional[str], + parent_id: str, + ) -> typing.Optional[dbos.WorkflowStatus]: + """ + Return the latest terminal (SUCCESS/ERROR) child workflow for ``parent_id`` that has + parseable automation output state, or None when no prior execution exists. + """ + matching_workflows = await self._get_parent_and_children_automation_workflows( + user_id, + [parent_id], + [ + dbos.WorkflowStatusString.SUCCESS, + dbos.WorkflowStatusString.ERROR, + ], + load_output=True, + ) + if not matching_workflows: + return None + sorted_workflows = sorted( + matching_workflows, + key=workflows_util._automation_child_workflow_sort_key, + reverse=True, + ) + for workflow_status in sorted_workflows: + workflow_output = workflows_util.parse_automation_workflow_output(workflow_status) + if workflow_output is not None and workflow_output.state: + return workflow_status + return None + async def _get_latest_workflow_for_each_automation( self, user_id: typing.Optional[str], @@ -562,7 +595,7 @@ async def get_automation_states(self, user_id: typing.Optional[str]) -> list[pro workflow_output = workflows_util.parse_automation_workflow_output(workflow) task = workflows_util.get_resolved_automation_task(workflow) if task: - task.id = workflow.workflow_id[:octobot_node.constants.PARENT_WORKFLOW_ID_LENGTH] + task.id = workflows_util.normalize_parent_automation_id(workflow.workflow_id) sources.append(automations_protocol.AutomationStateSource( task=task, workflow_status=workflow.status, diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py index 52216602d0..a3b01750ec 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py @@ -36,6 +36,7 @@ import octobot_node.scheduler.user_actions.user_actions_executor.account.refresh_accounts as user_actions_executor_refresh_accounts import octobot_node.scheduler.user_actions.user_actions_executor.automation.signal_automation as user_actions_executor_signal_automation import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_user_action_executor as user_actions_executor_strategy_base +import octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation as user_actions_executor_restart_automation import octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation as user_actions_executor_stop_automation from octobot_node.scheduler.user_actions.user_action_post_actions import UserActionPostActions @@ -53,6 +54,7 @@ EditAutomationActionExecutor = user_actions_executor_edit_automation.EditAutomationActionExecutor SignalAutomationActionExecutor = user_actions_executor_signal_automation.SignalAutomationActionExecutor StopAutomationActionExecutor = user_actions_executor_stop_automation.StopAutomationActionExecutor +RestartAutomationActionExecutor = user_actions_executor_restart_automation.RestartAutomationActionExecutor CreateAccountActionExecutor = user_actions_executor_create_account.CreateAccountActionExecutor EditAccountActionExecutor = user_actions_executor_edit_account.EditAccountActionExecutor DeleteAccountActionExecutor = user_actions_executor_delete_account.DeleteAccountActionExecutor @@ -76,6 +78,7 @@ "EditAutomationActionExecutor", "SignalAutomationActionExecutor", "StopAutomationActionExecutor", + "RestartAutomationActionExecutor", "CreateAccountActionExecutor", "EditAccountActionExecutor", "DeleteAccountActionExecutor", diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/automation_user_action_executor.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/automation_user_action_executor.py index d6a119a5fe..0ed2ec08cf 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/automation_user_action_executor.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/automation_user_action_executor.py @@ -71,6 +71,7 @@ def _get_error_message(self, exc: BaseException) -> protocol_models.AutomationAc node_errors.InvalidAutomationConfigurationError, node_errors.UnsupportedAutomationConfigurationTypeError, node_errors.UnsupportedUserActionConfigurationTypeError, + node_errors.UnrestartableAutomationError, ), ): return protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py index 300b77a1ce..c62b5ac172 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py @@ -202,6 +202,7 @@ def _create_automation_actions(self, user_action: protocol_models.UserAction) -> protocol_account, self._user_id, automation_id=automation_id, + strategy_id=stored_strategy.id, ), ] case protocol_models.CopyConfiguration() as copy_configuration: diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py new file mode 100644 index 0000000000..6847fcfab5 --- /dev/null +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py @@ -0,0 +1,172 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot Node is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or (at +# your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with OctoBot. If not, see . + +import json + +import octobot_flow.entities as flow_entities +import octobot_flow.enums as flow_enums +import octobot_protocol.models as protocol_models + +import octobot_node.errors as node_errors +import octobot_node.models as models +import octobot_node.scheduler as scheduler_module +import octobot_node.scheduler.task_context as task_context +import octobot_node.scheduler.user_actions.user_actions_executor.automation.automation_user_action_executor as automation_user_action_executor +import octobot_node.scheduler.user_actions.user_actions_executor.util.action_details_factory as action_details_factory +import octobot_node.scheduler.workflows_util as workflows_util + + +def _get_restart_automation_payload( + user_action: protocol_models.UserAction, +) -> protocol_models.RestartAutomationConfiguration: + wrapper = user_action.configuration + if wrapper is None or wrapper.actual_instance is None: + raise node_errors.InvalidUserActionPayloadError( + "UserAction.configuration must wrap a concrete restart-automation configuration." + ) + payload = wrapper.actual_instance + if not isinstance(payload, protocol_models.RestartAutomationConfiguration): + raise node_errors.InvalidUserActionPayloadError( + f"RestartAutomationActionExecutor expected RestartAutomationConfiguration, " + f"got {type(payload).__name__}" + ) + return payload + + +def _resolve_restart_reset_target_action_id( + automation_state: flow_entities.AutomationState, +) -> str: + actions_dag = automation_state.automation.actions_dag + for action in reversed(actions_dag.actions): + if action.id == action_details_factory._ACTION_ID_INIT: + continue + if isinstance(action, flow_entities.ConfiguredActionDetails): + if action.action == flow_enums.ActionType.APPLY_CONFIGURATION.value: + continue + if not action.can_be_reset(): + continue + elif not action.can_be_reset(): + continue + return action.id + raise node_errors.UnrestartableAutomationError( + "No resettable automation action found in the latest execution state." + ) + + +def prepare_automation_state_for_restart( + automation_state: flow_entities.AutomationState, +) -> flow_entities.AutomationState: + automation_state.automation.post_actions.stop_automation = False + automation_state.automation.execution.execution_error = None + reset_target_action_id = _resolve_restart_reset_target_action_id(automation_state) + automation_state.automation.actions_dag.reset_to(reset_target_action_id) + return automation_state + + +def _task_content_json_from_prepared_state( + automation_state: flow_entities.AutomationState, +) -> str: + return json.dumps({"state": automation_state.to_dict(include_default_values=False)}) + + +class RestartAutomationActionExecutor(automation_user_action_executor.AutomationUserActionExecutor): + async def _id_binds_to_user_action(self, restart_id: str) -> bool: + listed_user_actions = await scheduler_module.SCHEDULER.list_user_actions( + self._user_id, + active_only=False, + ) + user_action_ids = {user_action.id for user_action in listed_user_actions} + return restart_id in user_action_ids + + async def _assert_automation_not_running(self, parent_automation_id: str) -> None: + active_workflow_ids = await scheduler_module.SCHEDULER.resolve_active_automation_workflow_ids_for_parent_id( + self._user_id, + parent_automation_id, + ) + if active_workflow_ids: + raise node_errors.UnrestartableAutomationError( + f"Automation {parent_automation_id!r} is still running " + f"(active workflows: {active_workflow_ids!r})." + ) + + async def _build_restart_task(self, parent_automation_id: str) -> models.Task: + latest_workflow = ( + await scheduler_module.SCHEDULER.resolve_latest_terminal_automation_workflow_for_parent_id( + self._user_id, + parent_automation_id, + ) + ) + if latest_workflow is None: + raise node_errors.UnrestartableAutomationError( + f"No prior terminal execution found for automation {parent_automation_id!r}." + ) + workflow_output = workflows_util.parse_automation_workflow_output(latest_workflow) + if workflow_output is None or not workflow_output.state: + raise node_errors.UnrestartableAutomationError( + f"Latest execution for automation {parent_automation_id!r} has no usable output state." + ) + input_task = workflows_util.get_automation_input_task(latest_workflow) + task_name = input_task.name if input_task is not None else None + with task_context.encrypted_task( + models.Task( + content=workflow_output.state, + content_metadata=workflow_output.state_metadata, + ) + ): + automation_state_dict = workflows_util.get_automation_dict(workflow_output.state)[ + workflows_util.STATE_KEY + ] + automation_state = flow_entities.AutomationState.from_dict(automation_state_dict) + prepared_state = prepare_automation_state_for_restart(automation_state) + task_content = _task_content_json_from_prepared_state(prepared_state) + try: + next_workflow_id = workflows_util.build_next_child_automation_workflow_id( + latest_workflow.workflow_id + ) + except ValueError as error: + raise node_errors.UnrestartableAutomationError( + f"Cannot derive restart workflow id from latest execution " + f"{latest_workflow.workflow_id!r}: {error}" + ) from error + return models.Task( + id=next_workflow_id, + name=task_name, + content=task_content, + content_metadata=workflow_output.state_metadata, + type=models.TaskType.EXECUTE_ACTIONS.value, + user_id=self._user_id, + ) + + async def _do_execute( + self, + user_action: protocol_models.UserAction, + ) -> None: + if not scheduler_module.is_initialized(): + raise RuntimeError("Scheduler is not initialized") + + restart_payload = _get_restart_automation_payload(user_action) + parent_automation_id = workflows_util.normalize_parent_automation_id(restart_payload.id) + if await self._id_binds_to_user_action(restart_payload.id): + raise node_errors.UnrestartableAutomationError( + f"Restart id {restart_payload.id!r} binds to a user action and cannot be restarted." + ) + await self._assert_automation_not_running(parent_automation_id) + restart_task = await self._build_restart_task(parent_automation_id) + self.post_actions.to_create_automation_task = restart_task + self._mark_user_action_completed( + user_action, + created_automation_id=parent_automation_id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py index 0002d02b17..792fda9cc6 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py @@ -19,6 +19,7 @@ import octobot_node.errors as node_errors import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_user_action_executor as strategy_user_action_executor +import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_profile_validation as strategy_profile_validation def _get_create_strategy_payload( @@ -46,6 +47,9 @@ async def _do_execute( user_action: protocol_models.UserAction, ) -> None: create_payload = _get_create_strategy_payload(user_action) + strategy_profile_validation.validate_profile_strategy_configuration( + create_payload.configuration + ) collection_providers.StrategyProvider.instance().create_item( self._user_id, create_payload.configuration, diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py index 4255d277c0..1e0e0765b2 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py @@ -19,6 +19,7 @@ import octobot_node.errors as node_errors import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_user_action_executor as strategy_user_action_executor +import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_profile_validation as strategy_profile_validation def _get_edit_strategy_payload( @@ -54,6 +55,9 @@ async def _do_execute( raise node_errors.InvalidUserActionPayloadError( "EditStrategyConfiguration.id must match configuration.id." ) + strategy_profile_validation.validate_profile_strategy_configuration( + edit_payload.configuration + ) collection_providers.StrategyProvider.instance().update_item( self._user_id, edit_payload.configuration, diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/strategy_profile_validation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/strategy_profile_validation.py new file mode 100644 index 0000000000..33cf6cb9d2 --- /dev/null +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/strategy_profile_validation.py @@ -0,0 +1,50 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot Node is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or (at +# your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models +import octobot_protocol.models.generic_process_configuration as generic_process_configuration + +import octobot_node.errors as node_errors + + +def validate_profile_strategy_configuration( + strategy: protocol_models.Strategy, +) -> None: + configuration = strategy.configuration + if configuration is None or configuration.actual_instance is None: + return + if not isinstance( + configuration.actual_instance, + generic_process_configuration.GenericProcessConfiguration, + ): + return + generic_configuration = configuration.actual_instance + if generic_configuration.profile_data is None: + raise node_errors.InvalidUserActionPayloadError( + "GenericProcessConfiguration.profile_data is required for profile strategies." + ) + profile_details = dict( + generic_configuration.profile_data.get("profile_details") or {} + ) + profile_id = profile_details.get("id") + if profile_id is None: + profile_details["id"] = strategy.id + generic_configuration.profile_data["profile_details"] = profile_details + return + if profile_id != strategy.id: + raise node_errors.InvalidUserActionPayloadError( + "profile_data.profile_details.id must match strategy.id." + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py index e340354337..29a32a3ff1 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py @@ -43,6 +43,8 @@ def user_action_executor_factory( return user_actions_executor_package.EditAutomationActionExecutor case protocol_models.StopAutomationConfiguration: return user_actions_executor_package.StopAutomationActionExecutor + case protocol_models.RestartAutomationConfiguration: + return user_actions_executor_package.RestartAutomationActionExecutor case protocol_models.SignalAutomationConfiguration: return user_actions_executor_package.SignalAutomationActionExecutor case protocol_models.CreateAccountConfiguration: diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/action_details_factory.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/action_details_factory.py index 4f8099b94c..8a4051eafa 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/action_details_factory.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/action_details_factory.py @@ -450,13 +450,16 @@ def generic_process_action_factory( user_id: str, *, automation_id: str, + strategy_id: str | None = None, ) -> flow_entities.AbstractActionDetails: exchange_auth_data = _exchange_auth_data_list_from_protocol_account( protocol_account, user_id, ) - dsl_arguments = [f"{automation_id!r}"] - if generic_process_configuration.profile_data is not None: + dsl_arguments = [f"{automation_id!r}", f"user_id={user_id!r}"] + if strategy_id is not None: + dsl_arguments.append(f"sync_profile_id={strategy_id!r}") + elif generic_process_configuration.profile_data is not None: dsl_arguments.append( dsl_interpreter.format_parameter_value(generic_process_configuration.profile_data) ) diff --git a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py index ca2c04a230..8f29ebaa4e 100644 --- a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py @@ -188,14 +188,20 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) except octobot_flow.errors.CommunityTradingSignalError as err: execution_error = octobot_flow.enums.ActionErrorStatus.NO_TRADING_SIGNAL.value execution_error_message = str(err) - except octobot_trading.errors.AuthenticationError as err: + except ( + octobot_trading.errors.AuthenticationError, + octobot_trading.errors.PortfolioNegativeValueError, + ) as err: AutomationWorkflow.get_logger(parsed_inputs).error( - f"Authentication error: {err} ({err.__class__.__name__})" + f"{err.__class__.__name__} error (postponed iteration): {err}" ) - execution_error = octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR.value + execution_error_status, postpone_delay_seconds = ( + AutomationWorkflow._get_postponed_iteration_error_status_and_delay(err) + ) + next_step_at = time.time() + postpone_delay_seconds + execution_error = execution_error_status.value execution_error_message = str(err) postponed_iteration = True - next_step_at = time.time() + constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS has_next_actions_override = True next_iteration_description_override = parsed_inputs.task.content next_iteration_description_metadata_override = parsed_inputs.task.content_metadata @@ -235,9 +241,10 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) result, ) else: + retry_delay_seconds = max(0.0, (next_step_at or time.time()) - time.time()) AutomationWorkflow.get_logger(parsed_inputs).info( - f"Iteration postponed after authentication error, retry scheduled in " - f"{constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS:.0f} seconds" + f"Iteration postponed ({execution_error}: {execution_error_message}), " + f"retry scheduled in {retry_delay_seconds:.0f} seconds" ) #### End of decryped task context - no clear data after this point in encrypted context #### @@ -375,14 +382,14 @@ async def _process_pending_priority_actions_and_reschedule( AutomationWorkflow.get_logger(parsed_inputs).info( f"Stopping workflow, should stop: {latest_iteration_result.progress_status.should_stop}" ) - else: - # successful iteration and a new iteration is required, schedule next iteration, don't return anything - await AutomationWorkflow._schedule_next_iteration( - parsed_inputs, - latest_iteration_result.next_iteration_description, # type: ignore - latest_iteration_result.progress_status, - latest_iteration_result.next_iteration_description_metadata, - ) + return False, latest_iteration_result + # successful iteration and a new iteration is required, schedule next iteration, don't return anything + await AutomationWorkflow._schedule_next_iteration( + parsed_inputs, + latest_iteration_result.next_iteration_description, # type: ignore + latest_iteration_result.progress_status, + latest_iteration_result.next_iteration_description_metadata, + ) return True, latest_iteration_result @staticmethod @@ -413,14 +420,13 @@ async def _schedule_next_iteration( def _get_next_child_workflow_id() -> str: workflow_id = dbos.DBOS.workflow_id if workflow_id is None: - raise errors.WorkflowInputError("Missing current workflow ID while scheduling next iteration.") - parent_workflow_id = workflow_id[:constants.PARENT_WORKFLOW_ID_LENGTH] + raise errors.WorkflowInputError( + "Missing current workflow ID while scheduling next iteration." + ) try: - current_child_id = workflows_util.parse_automation_child_workflow_index(workflow_id) + return workflows_util.build_next_child_automation_workflow_id(workflow_id) except ValueError as error: raise errors.WorkflowInputError(str(error)) from error - next_child_id = current_child_id + 1 - return f"{parent_workflow_id}_{next_child_id}" @staticmethod def _create_next_iteration_inputs( @@ -477,3 +483,12 @@ def get_logger(parsed_inputs: params.AutomationWorkflowInputs) -> octobot_common return octobot_commons.logging.get_logger( parsed_inputs.task.name or AutomationWorkflow.__name__ ) + + @staticmethod + def _get_postponed_iteration_error_status_and_delay(error: Exception) -> tuple[ + octobot_flow.enums.ActionErrorStatus, float + ]: + if isinstance(error, octobot_trading.errors.AuthenticationError): + return octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR, constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS + # other errors, like PortfolioNegativeValueError + return octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR, constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS diff --git a/packages/node/octobot_node/scheduler/workflows_util.py b/packages/node/octobot_node/scheduler/workflows_util.py index 79b2b5b620..c618f81b6e 100644 --- a/packages/node/octobot_node/scheduler/workflows_util.py +++ b/packages/node/octobot_node/scheduler/workflows_util.py @@ -201,6 +201,16 @@ def filter_by_wallet( raise ValueError(f"Unsupported scheduler queue for wallet filter: {queue!r}") +def normalize_parent_automation_id(workflow_id: str) -> str: + return workflow_id[:octobot_node.constants.PARENT_WORKFLOW_ID_LENGTH] + + +def build_next_child_automation_workflow_id(current_workflow_id: str) -> str: + parent_id = normalize_parent_automation_id(current_workflow_id) + child_index = parse_automation_child_workflow_index(current_workflow_id) + return f"{parent_id}_{child_index + 1}" + + def parse_automation_child_workflow_index(workflow_id: str) -> int: """ Return the child iteration index encoded in a workflow ID. @@ -282,7 +292,12 @@ def parse_automation_workflow_output( if not workflow_status.output: return None try: - return params.AutomationWorkflowOutput.from_dict(json.loads(workflow_status.output)) + raw_output = workflow_status.output + if isinstance(raw_output, str): + raw_output = json.loads(raw_output) + if not isinstance(raw_output, dict): + raise TypeError(f"Unexpected workflow output type: {type(raw_output).__name__}") + return params.AutomationWorkflowOutput.from_dict(raw_output) except (json.JSONDecodeError, TypeError, ValueError) as error: logger.warning( "Failed to parse automation workflow output for %s: %s", diff --git a/packages/node/tests/conftest.py b/packages/node/tests/conftest.py index a39916a6a0..baf13c11f3 100644 --- a/packages/node/tests/conftest.py +++ b/packages/node/tests/conftest.py @@ -5,11 +5,16 @@ import pytest _TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS = 2 +_TESTS_RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS = 30.0 os.environ.setdefault( "RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS", str(_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS), ) +os.environ.setdefault( + "RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS", + str(_TESTS_RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS), +) import octobot.community.local_authenticator as local_community_auth @@ -39,3 +44,8 @@ def _fast_run_octobot_process_recall(monkeypatch): "RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS", _TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS, ) + monkeypatch.setattr( + node_constants, + "RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS", + _TESTS_RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS, + ) diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py b/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py index 1f5bfb3e64..2cc1ba4d9b 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py @@ -9,6 +9,7 @@ import mock import pytest +import dbos import octobot.constants as octobot_constants_module import octobot_protocol.models as octobot_protocol_models @@ -23,6 +24,7 @@ import octobot_node.config import octobot_node.scheduler import octobot_node.scheduler.workflows_util as workflows_util_module +import tentacles.Meta.DSL_operators.octobot_process_operators.octobot_process_ops as octobot_process_ops from tests.scheduler import temp_dbos_scheduler @@ -30,6 +32,8 @@ _T_INIT_SECONDS = 60.0 _T_STOP_SEND_SECONDS = 30.0 _T_STOP_COMPLETE_SECONDS = 45.0 +_T_RESTART_ENQUEUE_SECONDS = 30.0 +_T_RESTART_INIT_SECONDS = 60.0 _GENERIC_PROCESS_ACCOUNT_ID = "functional_generic_process_account" _GENERIC_PROCESS_AUTOMATION_NAME = "test_generic_process_default_config_automation" @@ -54,6 +58,13 @@ async def test_generic_process_default_config_lifecycle(self, temp_dbos_schedule monkeypatch.setenv(octobot_constants_module.ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS, "5") user_id = workflow_common_module.SIMULATOR_GRID_TEST_COMMUNITY_USER_ID + monkeypatch.setattr( + octobot_process_ops, + "_assert_sync_strategy_exists", + lambda sync_user_id, sync_profile_id: mock.Mock( + configuration=mock.Mock(actual_instance=mock.Mock(profile_data=None)) + ), + ) protocol_account = workflow_common_module.protocol_account_for_functional( account_id=_GENERIC_PROCESS_ACCOUNT_ID, usdc_total=1000.0, @@ -204,6 +215,60 @@ async def test_generic_process_default_config_lifecycle(self, temp_dbos_schedule else: pytest.fail(f"expected child pid {child_pid} to exit after AUTOMATION_STOP") + restart_user_action = workflow_common_module.build_restart_user_action( + automation_id=parent_automation_id, + user_action_id=f"ua-restart-{create_user_action.id}", + ) + try: + await asyncio.wait_for( + workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( + temp_dbos_scheduler, + restart_user_action, + user_id, + ), + timeout=_T_RESTART_ENQUEUE_SECONDS, + ) + except TimeoutError as exc: + raise AssertionError("execute_user_action timed out enqueueing automation restart") from exc + + await user_action_assertions_module.assert_user_action_selector_completed_automation_restart( + user_id=user_id, + user_action_id=restart_user_action.id, + ) + + restarted_inner_state = await octobot_process_workflow_module.wait_for_init_state_ok( + temp_dbos_scheduler, + metadata_automation_id, + timeout_sec=_T_RESTART_INIT_SECONDS, + active_workflows_only=True, + ) + assert restarted_inner_state.get("pid") + restarted_child_pid = int(restarted_inner_state["pid"]) + assert process_util_module.pid_is_running(restarted_child_pid) + child_pid = restarted_child_pid + child_user_root = restarted_inner_state.get("user_root") or child_user_root + child_log_folder = restarted_inner_state.get("log_folder") or child_log_folder + + workflow_rows_after_restart = await temp_dbos_scheduler.INSTANCE.list_workflows_async() + workflow_row_after_restart: typing.Any = None + for workflow_row in workflow_rows_after_restart: + if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + continue + if workflow_row.status not in ( + dbos.WorkflowStatusString.PENDING.value, + dbos.WorkflowStatusString.ENQUEUED.value, + ): + continue + workflow_row_after_restart = workflow_row + break + assert workflow_row_after_restart is not None + + protocol_automation_after_restart = await workflow_common_module.load_protocol_automation_state_for_workflow( + user_id, + workflow_row_after_restart, + ) + assert protocol_automation_after_restart.status == octobot_protocol_models.WorkflowStatus.RUNNING + if child_user_root and os.path.isdir(child_user_root): shutil.rmtree(child_user_root, ignore_errors=True) if child_log_folder and os.path.isdir(child_log_folder): diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py b/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py index 2fd231c6c3..184a51dcea 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py @@ -45,6 +45,8 @@ _T_SIGNAL_SECONDS = 5.0 _T_STOP_SEND_SECONDS = 5.0 _T_STOP_COMPLETE_SECONDS = 10.0 +_T_RESTART_ENQUEUE_SECONDS = 10.0 +_T_RESTART_RUNNING_SECONDS = 30.0 # Fast poll after stop/signal send; protocol status may flip RUNNING→COMPLETED quickly on CI. _POST_STOP_PROTOCOL_POLL_SECONDS = 0.05 @@ -429,3 +431,64 @@ async def test_trigger_task_grid_simulator_two_iterations_then_stop(self, temp_d protocol_state_final, _GRID_AUTOMATION_DISPLAY_NAME, ) + + restart_user_action = workflow_common_module.build_restart_user_action( + automation_id=parent_automation_id, + user_action_id=f"ua-restart-{create_user_action.id}", + ) + try: + await asyncio.wait_for( + workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( + temp_dbos_scheduler, + restart_user_action, + user_id, + ), + timeout=_T_RESTART_ENQUEUE_SECONDS, + ) + except TimeoutError as exc: + raise AssertionError("execute_user_action timed out enqueueing automation restart") from exc + + await user_action_assertions_module.assert_user_action_selector_completed_automation_restart( + user_id=user_id, + user_action_id=restart_user_action.id, + ) + + restart_running_deadline = time.monotonic() + _T_RESTART_RUNNING_SECONDS + workflow_row_after_restart = None + protocol_state_after_restart = None + while time.monotonic() < restart_running_deadline: + for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async(): + if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + continue + if workflow_row.status not in ( + dbos.WorkflowStatusString.PENDING.value, + dbos.WorkflowStatusString.ENQUEUED.value, + ): + continue + workflow_row_after_restart = workflow_row + protocol_state_after_restart = ( + await workflow_common_module.load_protocol_automation_state_for_workflow( + user_id, + workflow_row_after_restart, + ) + ) + if protocol_state_after_restart.status == octobot_protocol_models.WorkflowStatus.RUNNING: + break + if ( + workflow_row_after_restart is not None + and protocol_state_after_restart is not None + and protocol_state_after_restart.status == octobot_protocol_models.WorkflowStatus.RUNNING + ): + break + await asyncio.sleep(workflow_common_module.DEFAULT_WORKFLOW_POLL_INTERVAL_SECONDS) + else: + pytest.fail( + f"Timed out waiting for restarted grid automation {metadata_automation_id!r} to reach RUNNING" + ) + + assert workflow_row_after_restart is not None + assert protocol_state_after_restart.status == octobot_protocol_models.WorkflowStatus.RUNNING + protocol_assertions_module.assert_protocol_automation_metadata_name( + protocol_state_after_restart, + _GRID_AUTOMATION_DISPLAY_NAME, + ) diff --git a/packages/node/tests/functional_tests/util/octobot_process_workflow.py b/packages/node/tests/functional_tests/util/octobot_process_workflow.py index 3e71ba4166..6f856f792b 100644 --- a/packages/node/tests/functional_tests/util/octobot_process_workflow.py +++ b/packages/node/tests/functional_tests/util/octobot_process_workflow.py @@ -10,6 +10,7 @@ import typing import uuid +import dbos import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_flow.entities as flow_entities import octobot_protocol.models as protocol_models_module @@ -41,13 +42,16 @@ def seeded_generic_process_strategy_for_functional_wallet( stored_strategy_id: str = GENERIC_PROCESS_DEFAULT_STRATEGY_ID, profile_data: dict[str, typing.Any] | None = None, ) -> protocol_models_module.Strategy: + resolved_profile_data = profile_data + if resolved_profile_data is None: + resolved_profile_data = {"profile_details": {"id": stored_strategy_id}} return protocol_models_module.Strategy( id=stored_strategy_id, version=workflow_common_module.SIMULATOR_FUNCTIONAL_STRATEGY_VERSION, name="Generic process automation strategy", reference_market="USDC", configuration=protocol_models_module.StrategyConfiguration( - build_generic_process_configuration(profile_data=profile_data), + build_generic_process_configuration(profile_data=resolved_profile_data), ), ) @@ -198,6 +202,7 @@ async def wait_for_init_state_ok( *, timeout_sec: float = GLOBAL_INIT_TIMEOUT_SEC, poll_interval_sec: float = INIT_POLL_INTERVAL_SEC, + active_workflows_only: bool = False, ) -> dict: deadline = time.monotonic() + timeout_sec while time.monotonic() < deadline: @@ -205,6 +210,11 @@ async def wait_for_init_state_ok( for workflow_row in workflow_rows: import octobot_node.scheduler.workflows_util as workflows_util_module + if active_workflows_only and workflow_row.status not in ( + dbos.WorkflowStatusString.PENDING.value, + dbos.WorkflowStatusString.ENQUEUED.value, + ): + continue if workflows_util_module.get_automation_id(workflow_row) != automation_id: continue state_reader = workflows_util_module.get_automation_state_reader(workflow_row) diff --git a/packages/node/tests/functional_tests/util/user_action_assertions.py b/packages/node/tests/functional_tests/util/user_action_assertions.py index c23aa18ffb..ee02d6f261 100644 --- a/packages/node/tests/functional_tests/util/user_action_assertions.py +++ b/packages/node/tests/functional_tests/util/user_action_assertions.py @@ -125,6 +125,25 @@ async def assert_user_action_selector_completed_automation_stop( assert inner.error_message is None +async def assert_user_action_selector_completed_automation_restart( + *, + user_action_id: str, + user_id: str, +) -> None: + listed = await octobot_node.scheduler.SCHEDULER.list_user_actions(user_id) + by_id = merge_user_actions_latest_per_id(listed) + assert user_action_id in by_id, f"expected {user_action_id!r} in user action workflows, got {sorted(by_id)!r}" + stored = by_id[user_action_id] + assert stored.status == protocol_models_module.UserActionStatus.COMPLETED + assert stored.result is not None + inner = stored.result.actual_instance + assert isinstance(inner, protocol_models_module.AutomationActionResult) + assert inner.result_type == protocol_models_module.UserActionResultType.AUTOMATION + assert inner.error_details is None + assert inner.error_message is None + assert inner.created_automation_id + + async def assert_user_action_selector_completed_automation_signal( *, user_action_id: str, diff --git a/packages/node/tests/functional_tests/util/workflow_common.py b/packages/node/tests/functional_tests/util/workflow_common.py index 9ba1534677..a7b76e345f 100644 --- a/packages/node/tests/functional_tests/util/workflow_common.py +++ b/packages/node/tests/functional_tests/util/workflow_common.py @@ -123,6 +123,21 @@ def build_stop_user_action( ) +def build_restart_user_action( + *, + automation_id: str, + user_action_id: str, +) -> protocol_models_module.UserAction: + payload = protocol_models_module.RestartAutomationConfiguration( + action_type=protocol_models_module.UserActionType.AUTOMATION_RESTART, + id=automation_id, + ) + return protocol_models_module.UserAction( + id=user_action_id, + configuration=wrap_user_action_configuration(payload), + ) + + def build_forced_trigger_signal_user_action( *, automation_id: str, diff --git a/packages/node/tests/scheduler/test_shutdown.py b/packages/node/tests/scheduler/test_shutdown.py new file mode 100644 index 0000000000..7b993ab7b2 --- /dev/null +++ b/packages/node/tests/scheduler/test_shutdown.py @@ -0,0 +1,45 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import pytest +import mock + +import octobot_node.scheduler as scheduler_module + + +pytestmark = pytest.mark.asyncio + + +class TestShutdownSchedulerAndTradingSignalChannel: + async def test_second_call_is_no_op_after_shutdown(self): + with mock.patch.object(scheduler_module.SCHEDULER, "is_initialized", return_value=True): + with mock.patch.object(scheduler_module.SCHEDULER, "stop") as stop_mock: + await scheduler_module.shutdown_scheduler_and_trading_signal_channel() + await scheduler_module.shutdown_scheduler_and_trading_signal_channel() + stop_mock.assert_called_once() + + async def test_skips_when_scheduler_not_initialized(self): + with mock.patch.object(scheduler_module.SCHEDULER, "is_initialized", return_value=False): + with mock.patch.object(scheduler_module.SCHEDULER, "stop") as stop_mock: + await scheduler_module.shutdown_scheduler_and_trading_signal_channel() + stop_mock.assert_not_called() + + async def test_initialize_scheduler_resets_shutdown_guard(self): + with mock.patch.object(scheduler_module.SCHEDULER, "create"): + with mock.patch.object(scheduler_module.SCHEDULER, "start"): + with mock.patch("octobot_node.scheduler.workflows.register_workflows"): + scheduler_module._shutdown_done = True + scheduler_module.initialize_scheduler() + assert scheduler_module._shutdown_done is False diff --git a/packages/node/tests/scheduler/test_user_action_executor_factory.py b/packages/node/tests/scheduler/test_user_action_executor_factory.py index dbbae514e4..302c762ab0 100644 --- a/packages/node/tests/scheduler/test_user_action_executor_factory.py +++ b/packages/node/tests/scheduler/test_user_action_executor_factory.py @@ -98,6 +98,15 @@ def test_returns_stop_automation_executor_class(self): resolved_executor_cls = executor_factory_module.user_action_executor_factory(user_action_model) assert resolved_executor_cls is user_actions_executor_package.StopAutomationActionExecutor + def test_returns_restart_automation_executor_class(self): + configuration_inner = protocol_models.RestartAutomationConfiguration( + id="auto-restart", + action_type=protocol_models.UserActionType.AUTOMATION_RESTART, + ) + user_action_model = self._user_action(action_identifier="ua-restart", configuration_inner=configuration_inner) + resolved_executor_cls = executor_factory_module.user_action_executor_factory(user_action_model) + assert resolved_executor_cls is user_actions_executor_package.RestartAutomationActionExecutor + def test_returns_signal_automation_executor_class(self): configuration_inner = protocol_models.SignalAutomationConfiguration( action_type=protocol_models.UserActionType.AUTOMATION_SIGNAL, diff --git a/packages/node/tests/scheduler/test_workflows_util_automation_state.py b/packages/node/tests/scheduler/test_workflows_util_automation_state.py index 69e16f6c47..a30295063a 100644 --- a/packages/node/tests/scheduler/test_workflows_util_automation_state.py +++ b/packages/node/tests/scheduler/test_workflows_util_automation_state.py @@ -78,6 +78,34 @@ def _workflow_status_with_automation_task( return workflow_status +class TestNormalizeParentAutomationId: + def test_parent_workflow_id_unchanged(self): + assert workflows_util.normalize_parent_automation_id(_PARENT_WORKFLOW_ID) == _PARENT_WORKFLOW_ID + + def test_child_workflow_id_truncated_to_parent(self): + child_id = _child_workflow_id(5) + assert workflows_util.normalize_parent_automation_id(child_id) == _PARENT_WORKFLOW_ID + + +class TestBuildNextChildAutomationWorkflowId: + def test_parent_workflow_id_maps_to_first_child(self): + assert ( + workflows_util.build_next_child_automation_workflow_id(_PARENT_WORKFLOW_ID) + == _child_workflow_id(1) + ) + + def test_child_workflow_id_increments_suffix(self): + assert ( + workflows_util.build_next_child_automation_workflow_id(_child_workflow_id(2)) + == _child_workflow_id(3) + ) + + def test_invalid_suffix_raises_value_error(self): + invalid_child_id = f"{_PARENT_WORKFLOW_ID}-4-4" + with pytest.raises(ValueError, match="Invalid child workflow suffix format"): + workflows_util.build_next_child_automation_workflow_id(invalid_child_id) + + class TestParseAutomationChildWorkflowIndex: def test_parent_workflow_id_maps_to_zero(self): assert workflows_util.parse_automation_child_workflow_index(_PARENT_WORKFLOW_ID) == 0 diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py index fdf5c6efc8..e9cf3dd961 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py @@ -857,7 +857,8 @@ def test_generic_process_returns_init_and_run_octobot_process(self): ) expected_dsl = ( "run_octobot_process(" - f"{'ua-generic-process'!r}, exchange_auth_data={dsl_interpreter.format_parameter_value(expected_exchange_auth)}, " + f"{'ua-generic-process'!r}, user_id={_TEST_WALLET_ADDRESS!r}, sync_profile_id={strat_ref.id!r}, " + f"exchange_auth_data={dsl_interpreter.format_parameter_value(expected_exchange_auth)}, " f"{', '.join(action_details_factory._run_octobot_process_recall_kwarg_segments())})" ) _assert_init_action_matches_minimal_account( diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py new file mode 100644 index 0000000000..6dc2a13ed6 --- /dev/null +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py @@ -0,0 +1,325 @@ +import json +import datetime +import mock +import pytest +import dbos + +import octobot_flow.entities as flow_entities +import octobot_flow.enums as flow_enums +import octobot_protocol.models as protocol_models + +import octobot_node.errors as node_errors +import octobot_node.models as models_module +import octobot_node.scheduler as scheduler_module +import octobot_node.scheduler.workflows.params as workflow_params_module +import octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation as restart_automation_executor + +from .. import provider_assertions + + +_TEST_WALLET_ADDRESS = "0xaaabbbcccddd" +_PARENT_AUTOMATION_ID = "00000000-0000-4000-8000-000000000001" + + +def _wrap(configuration_payload) -> protocol_models.UserActionConfiguration: + return protocol_models.UserActionConfiguration.from_json(configuration_payload.to_json()) + + +def _user_action_restart(*, user_action_id: str, automation_parent_id: str) -> protocol_models.UserAction: + restart_payload = protocol_models.RestartAutomationConfiguration( + id=automation_parent_id, + action_type=protocol_models.UserActionType.AUTOMATION_RESTART, + ) + return protocol_models.UserAction(id=user_action_id, configuration=_wrap(restart_payload)) + + +def _stopped_automation_state_dict(*, stop_automation: bool = True) -> dict: + return { + "automation": { + "metadata": {"automation_id": _PARENT_AUTOMATION_ID}, + "actions_dag": { + "actions": [ + { + "id": "action_init", + "action": flow_enums.ActionType.APPLY_CONFIGURATION.value, + "executed_at": 1.0, + }, + { + "id": "action_run", + "dsl_script": "run_octobot_process('auto')", + "dependencies": [{"action_id": "action_init"}], + "executed_at": 2.0, + "result": {"pid": 42}, + }, + ] + }, + "post_actions": {"stop_automation": stop_automation}, + "execution": { + "previous_execution": {"triggered_at": 1.0}, + "current_execution": {"triggered_at": 2.0}, + }, + }, + "exchange_account_details": { + "exchange_details": {"internal_name": "binanceus"}, + }, + } + + +def _terminal_workflow_with_output( + *, + parent_id: str = _PARENT_AUTOMATION_ID, + stop_automation: bool = True, +) -> mock.Mock: + state_dict = _stopped_automation_state_dict(stop_automation=stop_automation) + task_content = json.dumps({"state": state_dict}) + task = models_module.Task( + name="restart-test-automation", + content=task_content, + type=models_module.TaskType.EXECUTE_ACTIONS.value, + ) + encoded_inputs = workflow_params_module.AutomationWorkflowInputs(task=task).to_dict( + include_default_values=False + ) + workflow_status = mock.Mock(spec=dbos.WorkflowStatus) + workflow_status.workflow_id = parent_id + workflow_status.updated_at = 100 + workflow_status.input = {"args": [encoded_inputs], "kwargs": {}} + workflow_status.output = json.dumps( + workflow_params_module.AutomationWorkflowOutput(state=task_content).to_dict( + include_default_values=False + ) + ) + return workflow_status + + +class TestPrepareAutomationStateForRestart: + def test_clears_stop_automation_and_resets_main_action(self): + automation_state = flow_entities.AutomationState.from_dict( + _stopped_automation_state_dict(stop_automation=True) + ) + prepared_state = restart_automation_executor.prepare_automation_state_for_restart(automation_state) + assert prepared_state.automation.post_actions.stop_automation is False + run_action = prepared_state.automation.actions_dag.get_actions_by_id()["action_run"] + assert run_action.executed_at is None + assert run_action.previous_execution_result == {"pid": 42} + init_action = prepared_state.automation.actions_dag.get_actions_by_id()["action_init"] + assert init_action.executed_at == 1.0 + + +class TestRestartAutomationActionExecutor: + @pytest.mark.asyncio + async def test_execute_enqueues_task_from_latest_output(self): + user_action = _user_action_restart( + user_action_id="ua-restart-1", + automation_parent_id=_PARENT_AUTOMATION_ID, + ) + terminal_workflow = _terminal_workflow_with_output() + executor = restart_automation_executor.RestartAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "list_user_actions", + new_callable=mock.AsyncMock, + return_value=[], + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "resolve_active_automation_workflow_ids_for_parent_id", + new_callable=mock.AsyncMock, + return_value=[], + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "resolve_latest_terminal_automation_workflow_for_parent_id", + new_callable=mock.AsyncMock, + return_value=terminal_workflow, + ), + ): + await executor.execute(user_action) + + scheduled_task = executor.post_actions.to_create_automation_task + assert scheduled_task is not None + assert scheduled_task.id == f"{_PARENT_AUTOMATION_ID}_1" + assert scheduled_task.name == "restart-test-automation" + assert scheduled_task.user_id == _TEST_WALLET_ADDRESS + task_payload = json.loads(scheduled_task.content) + assert task_payload["state"]["automation"]["post_actions"]["stop_automation"] is False + run_action = task_payload["state"]["automation"]["actions_dag"]["actions"][1] + assert run_action["executed_at"] is None + assert run_action["previous_execution_result"] == {"pid": 42} + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.COMPLETED, + result_channel="automation", + expect_error_details=False, + ) + inner = user_action.result.actual_instance + assert inner.created_automation_id == _PARENT_AUTOMATION_ID + + @pytest.mark.asyncio + async def test_raises_unrestartable_when_id_binds_to_user_action(self): + user_action = _user_action_restart( + user_action_id="ua-restart-2", + automation_parent_id=_PARENT_AUTOMATION_ID, + ) + bound_user_action = protocol_models.UserAction(id=_PARENT_AUTOMATION_ID) + executor = restart_automation_executor.RestartAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "list_user_actions", + new_callable=mock.AsyncMock, + return_value=[bound_user_action], + ), + pytest.raises(node_errors.UnrestartableAutomationError), + ): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="automation", + expect_error_details=True, + expected_error_message=protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION, + ) + + @pytest.mark.asyncio + async def test_raises_unrestartable_when_automation_still_running(self): + user_action = _user_action_restart( + user_action_id="ua-restart-3", + automation_parent_id=_PARENT_AUTOMATION_ID, + ) + executor = restart_automation_executor.RestartAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "list_user_actions", + new_callable=mock.AsyncMock, + return_value=[], + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "resolve_active_automation_workflow_ids_for_parent_id", + new_callable=mock.AsyncMock, + return_value=[f"{_PARENT_AUTOMATION_ID}_1"], + ), + pytest.raises(node_errors.UnrestartableAutomationError), + ): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="automation", + expect_error_details=True, + expected_error_message=protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION, + ) + + @pytest.mark.asyncio + async def test_raises_unrestartable_when_no_prior_execution(self): + user_action = _user_action_restart( + user_action_id="ua-restart-4", + automation_parent_id=_PARENT_AUTOMATION_ID, + ) + executor = restart_automation_executor.RestartAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "list_user_actions", + new_callable=mock.AsyncMock, + return_value=[], + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "resolve_active_automation_workflow_ids_for_parent_id", + new_callable=mock.AsyncMock, + return_value=[], + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "resolve_latest_terminal_automation_workflow_for_parent_id", + new_callable=mock.AsyncMock, + return_value=None, + ), + pytest.raises(node_errors.UnrestartableAutomationError), + ): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="automation", + expect_error_details=True, + expected_error_message=protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION, + ) + + @pytest.mark.asyncio + async def test_raises_when_scheduler_not_initialized(self): + user_action = _user_action_restart( + user_action_id="ua-restart-5", + automation_parent_id=_PARENT_AUTOMATION_ID, + ) + executor = restart_automation_executor.RestartAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.scheduler_module.is_initialized", + return_value=False, + ): + with pytest.raises(RuntimeError, match="Scheduler is not initialized"): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="automation", + expect_error_details=True, + expected_error_message=protocol_models.AutomationActionResultErrorMessage.INTERNAL_ERROR, + ) + + @pytest.mark.asyncio + async def test_invalid_payload_raises_invalid_user_action_payload(self): + wrong = protocol_models.CreateAccountConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_CREATE, + configuration=protocol_models.Account( + id="a", + name="n", + is_simulated=True, + created_at=datetime.datetime(2026, 6, 1, 12, 0, 0, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 6, 1, 13, 0, 0, tzinfo=datetime.UTC), + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="r", + exchange_config_ids=["test-exchange-config-id"], + ) + ), + ), + ) + user_action = protocol_models.UserAction(id="ua-bad", configuration=_wrap(wrong)) + executor = restart_automation_executor.RestartAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.scheduler_module.is_initialized", + return_value=True, + ), + pytest.raises(node_errors.InvalidUserActionPayloadError), + ): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="automation", + expect_error_details=True, + expected_error_message=protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION, + ) diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/strategy_executor_test_utils.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/strategy_executor_test_utils.py index 6469b09126..f0c94980a6 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/strategy_executor_test_utils.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/strategy_executor_test_utils.py @@ -35,7 +35,7 @@ def minimal_strategy( ) -> protocol_models.Strategy: configuration = protocol_models.GenericProcessConfiguration( configuration_type=protocol_models.ActionConfigurationType.GENERIC_PROCESS, - profile_data={}, + profile_data={"profile_details": {"id": strategy_id}}, ) return protocol_models.Strategy( id=strategy_id, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/test_channel_user_action_executor_get_error_message.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/test_channel_user_action_executor_get_error_message.py index 73e4e25313..ef04ea7881 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/test_channel_user_action_executor_get_error_message.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/test_channel_user_action_executor_get_error_message.py @@ -23,6 +23,7 @@ import octobot_node.scheduler.user_actions.user_actions_executor.exchange_config.create_exchange_config as create_exchange_config_executor import octobot_node.scheduler.user_actions.user_actions_executor.strategy.create_strategy as create_strategy_executor import octobot_node.scheduler.user_actions.user_actions_executor.account_auth.create_account_auth as create_account_auth_executor +import octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation as restart_automation_executor import octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation as stop_automation_executor _WALLET = "0xwallet" @@ -39,6 +40,11 @@ def test_invalid_user_action_payload(self): resolved = executor._get_error_message(node_errors.InvalidUserActionPayloadError("bad")) assert resolved == protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION + def test_unrestartable_automation_error(self): + executor = restart_automation_executor.RestartAutomationActionExecutor(_WALLET) + resolved = executor._get_error_message(node_errors.UnrestartableAutomationError("cannot restart")) + assert resolved == protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION + def test_ambiguous_active_automation_workflow(self): executor = stop_automation_executor.StopAutomationActionExecutor(_WALLET) resolved = executor._get_error_message(node_errors.AmbiguousActiveAutomationWorkflowError("ambiguous")) diff --git a/packages/node/tests/scheduler/workflows/test_automation_workflow.py b/packages/node/tests/scheduler/workflows/test_automation_workflow.py index c1c0f67202..495fc131fb 100644 --- a/packages/node/tests/scheduler/workflows/test_automation_workflow.py +++ b/packages/node/tests/scheduler/workflows/test_automation_workflow.py @@ -637,17 +637,40 @@ async def test_execute_iteration_logs_and_reraises_when_octobot_actions_job_fail @pytest.mark.asyncio @required_imports - async def test_execute_iteration_authentication_error_sets_postponed_iteration( - self, import_automation_workflow, task + @pytest.mark.parametrize( + "run_side_effect,expected_error_status,expected_retry_delay_seconds", + [ + pytest.param( + octobot_trading_errors.AuthenticationError("Invalid API credentials"), + octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR.value, + octobot_node.constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS, + id="authentication_error", + ), + pytest.param( + octobot_trading_errors.PortfolioNegativeValueError( + "Trying to update BTC with -0.00074 but quantity was 0.00068" + ), + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR.value, + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS, + id="portfolio_negative_value_error", + ), + ], + ) + async def test_execute_iteration_postponed_error_sets_postponed_iteration( + self, + import_automation_workflow, + task, + run_side_effect, + expected_error_status, + expected_retry_delay_seconds, ): task_content = json.dumps({"params": {"ACTIONS": "trade", "EXCHANGE_FROM": "binance", "ORDER_SYMBOL": "ETH/BTC", "ORDER_AMOUNT": 1, "ORDER_TYPE": "market", "ORDER_SIDE": "BUY", "SIMULATED_PORTFOLIO": {"BTC": 1}}}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) - authentication_error_message = "Invalid API credentials" mock_octobot_actions_job_class, _ = _octobot_actions_job_mock_class( - run_side_effect=octobot_trading_errors.AuthenticationError(authentication_error_message), + run_side_effect=run_side_effect, ) fixed_now = 1000.0 @@ -668,11 +691,11 @@ async def test_execute_iteration_authentication_error_sets_postponed_iteration( update_account_trading_mock.assert_not_called() parsed_progress_status = params.ProgressStatus.model_validate(result["progress_status"]) - assert parsed_progress_status.error == octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR.value - assert parsed_progress_status.error_message == authentication_error_message + assert parsed_progress_status.error == expected_error_status + assert parsed_progress_status.error_message == str(run_side_effect) assert parsed_progress_status.postponed_iteration is True assert parsed_progress_status.next_step_at == ( - fixed_now + octobot_node.constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS + fixed_now + expected_retry_delay_seconds ) assert result["has_next_actions"] is True assert result["next_iteration_description"] == task_content @@ -855,15 +878,38 @@ async def test_execute_iteration_continues_when_trading_persistence_wallet_missi class TestExecuteAutomationPostponedIteration: @pytest.mark.asyncio @required_imports + @pytest.mark.parametrize( + "error_status,error_message,retry_delay_seconds", + [ + pytest.param( + octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR.value, + "Invalid API credentials", + octobot_node.constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS, + id="authentication_error", + ), + pytest.param( + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR.value, + "Trying to update BTC with -0.00074 but quantity was 0.00068", + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS, + id="portfolio_negative_value_error", + ), + ], + ) async def test_execute_automation_reschedules_on_postponed_iteration( - self, temp_dbos_scheduler, import_automation_workflow, parsed_inputs + self, + temp_dbos_scheduler, + import_automation_workflow, + parsed_inputs, + error_status, + error_message, + retry_delay_seconds, ): postponed_iteration_result = params.AutomationWorkflowIterationResult( progress_status=params.ProgressStatus( latest_step="no action executed", - next_step_at=time.time() + octobot_node.constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS, - error=octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR.value, - error_message="Invalid API credentials", + next_step_at=time.time() + retry_delay_seconds, + error=error_status, + error_message=error_message, postponed_iteration=True, should_stop=False, ), @@ -978,7 +1024,7 @@ async def test_process_pending_returns_false_when_should_stop(self, import_autom should_continue, _ = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._process_pending_priority_actions_and_reschedule( parsed_inputs, iteration_result ) - assert should_continue is True + assert should_continue is False @pytest.mark.asyncio async def test_process_pending_raises_when_no_next_iteration_after_priority_actions( @@ -1160,6 +1206,41 @@ def test_create_next_iteration_inputs_uses_zero_when_execution_time_none(self, i assert result.execution_time == 0 +class TestGetPostponedIterationErrorStatusAndDelay: + @pytest.mark.parametrize( + "error,expected_status,expected_delay", + [ + pytest.param( + octobot_trading_errors.AuthenticationError("Invalid API credentials"), + octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR, + octobot_node.constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS, + id="authentication_error", + ), + pytest.param( + octobot_trading_errors.PortfolioNegativeValueError( + "Trying to update BTC with -0.00074 but quantity was 0.00068" + ), + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR, + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS, + id="portfolio_negative_value_error", + ), + ], + ) + def test_returns_expected_status_and_delay( + self, + import_automation_workflow, + error, + expected_status, + expected_delay, + ): + resolved_status, resolved_delay = ( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + ._get_postponed_iteration_error_status_and_delay(error) + ) + assert resolved_status == expected_status + assert resolved_delay == expected_delay + + class TestShouldContinueWorkflow: def test_should_continue_returns_stop_on_error_when_error(self, import_automation_workflow, parsed_inputs): progress = params.ProgressStatus(error="some_error", should_stop=False) diff --git a/packages/protocol/.openapi-generator/FILES b/packages/protocol/.openapi-generator/FILES index f79681b486..f867b5b8d9 100644 --- a/packages/protocol/.openapi-generator/FILES +++ b/packages/protocol/.openapi-generator/FILES @@ -82,6 +82,7 @@ docs/Position.md docs/PositionStatus.md docs/PositionSummary.md docs/RefreshAccountsConfiguration.md +docs/RestartAutomationConfiguration.md docs/Side.md docs/SignalAutomationConfiguration.md docs/SignalAutomationConfigurationSignalPayload.md @@ -196,6 +197,7 @@ octobot_protocol/models/position.py octobot_protocol/models/position_status.py octobot_protocol/models/position_summary.py octobot_protocol/models/refresh_accounts_configuration.py +octobot_protocol/models/restart_automation_configuration.py octobot_protocol/models/side.py octobot_protocol/models/signal_automation_configuration.py octobot_protocol/models/signal_automation_configuration_signal_payload.py @@ -308,6 +310,7 @@ test/test_position.py test/test_position_status.py test/test_position_summary.py test/test_refresh_accounts_configuration.py +test/test_restart_automation_configuration.py test/test_side.py test/test_signal_automation_configuration.py test/test_signal_automation_configuration_signal_payload.py diff --git a/packages/protocol/docs/RestartAutomationConfiguration.md b/packages/protocol/docs/RestartAutomationConfiguration.md new file mode 100644 index 0000000000..a012eb4114 --- /dev/null +++ b/packages/protocol/docs/RestartAutomationConfiguration.md @@ -0,0 +1,31 @@ +# RestartAutomationConfiguration + +RestartAutomationConfiguration + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**id** | **str** | | +**action_type** | [**UserActionType**](UserActionType.md) | automation_restart | + +## Example + +```python +from octobot_protocol.models.restart_automation_configuration import RestartAutomationConfiguration + +# TODO update the JSON string below +json = "{}" +# create an instance of RestartAutomationConfiguration from a JSON string +restart_automation_configuration_instance = RestartAutomationConfiguration.from_json(json) +# print the JSON string representation of the object +print(RestartAutomationConfiguration.to_json()) + +# convert the object into a dict +restart_automation_configuration_dict = restart_automation_configuration_instance.to_dict() +# create an instance of RestartAutomationConfiguration from a dict +restart_automation_configuration_from_dict = RestartAutomationConfiguration.from_dict(restart_automation_configuration_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/UserActionType.md b/packages/protocol/docs/UserActionType.md index 2d57fb2d1f..8d10bdec95 100644 --- a/packages/protocol/docs/UserActionType.md +++ b/packages/protocol/docs/UserActionType.md @@ -10,6 +10,8 @@ UserActionType * `AUTOMATION_STOP` (value: `'automation_stop'`) +* `AUTOMATION_RESTART` (value: `'automation_restart'`) + * `AUTOMATION_SIGNAL` (value: `'automation_signal'`) * `ACCOUNT_CREATE` (value: `'account_create'`) diff --git a/packages/protocol/octobot_protocol/models/__init__.py b/packages/protocol/octobot_protocol/models/__init__.py index 225894506f..e93f519553 100644 --- a/packages/protocol/octobot_protocol/models/__init__.py +++ b/packages/protocol/octobot_protocol/models/__init__.py @@ -97,6 +97,7 @@ from octobot_protocol.models.position_status import PositionStatus from octobot_protocol.models.position_summary import PositionSummary from octobot_protocol.models.refresh_accounts_configuration import RefreshAccountsConfiguration +from octobot_protocol.models.restart_automation_configuration import RestartAutomationConfiguration from octobot_protocol.models.side import Side from octobot_protocol.models.signal_automation_configuration import SignalAutomationConfiguration from octobot_protocol.models.signal_automation_configuration_signal_payload import SignalAutomationConfigurationSignalPayload diff --git a/packages/protocol/octobot_protocol/models/restart_automation_configuration.py b/packages/protocol/octobot_protocol/models/restart_automation_configuration.py new file mode 100644 index 0000000000..344156f194 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/restart_automation_configuration.py @@ -0,0 +1,91 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field, StrictStr +from typing import Any, ClassVar, Dict, List +from octobot_protocol.models.user_action_type import UserActionType +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class RestartAutomationConfiguration(BaseModel): + """ + RestartAutomationConfiguration + """ # noqa: E501 + id: StrictStr + action_type: UserActionType = Field(description="automation_restart") + __properties: ClassVar[List[str]] = ["id", "action_type"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of RestartAutomationConfiguration from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of RestartAutomationConfiguration from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "id": obj.get("id"), + "action_type": obj.get("action_type") + }) + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/user_action_configuration.py b/packages/protocol/octobot_protocol/models/user_action_configuration.py index b5a431f73e..f4f6866acc 100644 --- a/packages/protocol/octobot_protocol/models/user_action_configuration.py +++ b/packages/protocol/octobot_protocol/models/user_action_configuration.py @@ -32,13 +32,14 @@ from octobot_protocol.models.edit_exchange_config_configuration import EditExchangeConfigConfiguration from octobot_protocol.models.edit_strategy_configuration import EditStrategyConfiguration from octobot_protocol.models.refresh_accounts_configuration import RefreshAccountsConfiguration +from octobot_protocol.models.restart_automation_configuration import RestartAutomationConfiguration from octobot_protocol.models.signal_automation_configuration import SignalAutomationConfiguration from octobot_protocol.models.stop_automation_configuration import StopAutomationConfiguration from pydantic import StrictStr, Field from typing import Union, List, Set, Optional, Dict from typing_extensions import Literal, Self -USERACTIONCONFIGURATION_ONE_OF_SCHEMAS = ["CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration"] +USERACTIONCONFIGURATION_ONE_OF_SCHEMAS = ["CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "RestartAutomationConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration"] class UserActionConfiguration(BaseModel): """ @@ -50,36 +51,38 @@ class UserActionConfiguration(BaseModel): oneof_schema_2_validator: Optional[EditAutomationConfiguration] = None # data type: StopAutomationConfiguration oneof_schema_3_validator: Optional[StopAutomationConfiguration] = None + # data type: RestartAutomationConfiguration + oneof_schema_4_validator: Optional[RestartAutomationConfiguration] = None # data type: SignalAutomationConfiguration - oneof_schema_4_validator: Optional[SignalAutomationConfiguration] = None + oneof_schema_5_validator: Optional[SignalAutomationConfiguration] = None # data type: CreateAccountConfiguration - oneof_schema_5_validator: Optional[CreateAccountConfiguration] = None + oneof_schema_6_validator: Optional[CreateAccountConfiguration] = None # data type: EditAccountConfiguration - oneof_schema_6_validator: Optional[EditAccountConfiguration] = None + oneof_schema_7_validator: Optional[EditAccountConfiguration] = None # data type: DeleteAccountConfiguration - oneof_schema_7_validator: Optional[DeleteAccountConfiguration] = None + oneof_schema_8_validator: Optional[DeleteAccountConfiguration] = None # data type: CreateExchangeConfigConfiguration - oneof_schema_8_validator: Optional[CreateExchangeConfigConfiguration] = None + oneof_schema_9_validator: Optional[CreateExchangeConfigConfiguration] = None # data type: EditExchangeConfigConfiguration - oneof_schema_9_validator: Optional[EditExchangeConfigConfiguration] = None + oneof_schema_10_validator: Optional[EditExchangeConfigConfiguration] = None # data type: DeleteExchangeConfigConfiguration - oneof_schema_10_validator: Optional[DeleteExchangeConfigConfiguration] = None + oneof_schema_11_validator: Optional[DeleteExchangeConfigConfiguration] = None # data type: RefreshAccountsConfiguration - oneof_schema_11_validator: Optional[RefreshAccountsConfiguration] = None + oneof_schema_12_validator: Optional[RefreshAccountsConfiguration] = None # data type: CreateStrategyConfiguration - oneof_schema_12_validator: Optional[CreateStrategyConfiguration] = None + oneof_schema_13_validator: Optional[CreateStrategyConfiguration] = None # data type: EditStrategyConfiguration - oneof_schema_13_validator: Optional[EditStrategyConfiguration] = None + oneof_schema_14_validator: Optional[EditStrategyConfiguration] = None # data type: DeleteStrategyConfiguration - oneof_schema_14_validator: Optional[DeleteStrategyConfiguration] = None + oneof_schema_15_validator: Optional[DeleteStrategyConfiguration] = None # data type: CreateAccountAuthConfiguration - oneof_schema_15_validator: Optional[CreateAccountAuthConfiguration] = None + oneof_schema_16_validator: Optional[CreateAccountAuthConfiguration] = None # data type: EditAccountAuthConfiguration - oneof_schema_16_validator: Optional[EditAccountAuthConfiguration] = None + oneof_schema_17_validator: Optional[EditAccountAuthConfiguration] = None # data type: DeleteAccountAuthConfiguration - oneof_schema_17_validator: Optional[DeleteAccountAuthConfiguration] = None - actual_instance: Optional[Union[CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration]] = None - one_of_schemas: Set[str] = { "CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration" } + oneof_schema_18_validator: Optional[DeleteAccountAuthConfiguration] = None + actual_instance: Optional[Union[CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration]] = None + one_of_schemas: Set[str] = { "CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "RestartAutomationConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration" } model_config = ConfigDict( validate_assignment=True, @@ -120,6 +123,11 @@ def actual_instance_must_validate_oneof(cls, v): error_messages.append(f"Error! Input type `{type(v)}` is not `StopAutomationConfiguration`") else: match += 1 + # validate data type: RestartAutomationConfiguration + if not isinstance(v, RestartAutomationConfiguration): + error_messages.append(f"Error! Input type `{type(v)}` is not `RestartAutomationConfiguration`") + else: + match += 1 # validate data type: SignalAutomationConfiguration if not isinstance(v, SignalAutomationConfiguration): error_messages.append(f"Error! Input type `{type(v)}` is not `SignalAutomationConfiguration`") @@ -192,10 +200,10 @@ def actual_instance_must_validate_oneof(cls, v): match += 1 if match > 1: # more than 1 match - raise ValueError("Multiple matches found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("Multiple matches found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) elif match == 0: # no match - raise ValueError("No match found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("No match found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) else: return v @@ -260,6 +268,11 @@ def from_json(cls, json_str: str) -> Self: instance.actual_instance = EditAutomationConfiguration.from_json(json_str) return instance + # check if data type is `RestartAutomationConfiguration` + if _data_type == "automation_restart": + instance.actual_instance = RestartAutomationConfiguration.from_json(json_str) + return instance + # check if data type is `SignalAutomationConfiguration` if _data_type == "automation_signal": instance.actual_instance = SignalAutomationConfiguration.from_json(json_str) @@ -318,6 +331,12 @@ def from_json(cls, json_str: str) -> Self: match += 1 except (ValidationError, ValueError) as e: error_messages.append(str(e)) + # deserialize data into RestartAutomationConfiguration + try: + instance.actual_instance = RestartAutomationConfiguration.from_json(json_str) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) # deserialize data into SignalAutomationConfiguration try: instance.actual_instance = SignalAutomationConfiguration.from_json(json_str) @@ -405,10 +424,10 @@ def from_json(cls, json_str: str) -> Self: if match > 1: # more than 1 match - raise ValueError("Multiple matches found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("Multiple matches found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) elif match == 0: # no match - raise ValueError("No match found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("No match found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) else: return instance @@ -422,7 +441,7 @@ def to_json(self) -> str: else: return json.dumps(self.actual_instance) - def to_dict(self) -> Optional[Union[Dict[str, Any], CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration]]: + def to_dict(self) -> Optional[Union[Dict[str, Any], CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration]]: """Returns the dict representation of the actual instance""" if self.actual_instance is None: return None diff --git a/packages/protocol/octobot_protocol/models/user_action_type.py b/packages/protocol/octobot_protocol/models/user_action_type.py index fbe31eec41..8836a794d0 100644 --- a/packages/protocol/octobot_protocol/models/user_action_type.py +++ b/packages/protocol/octobot_protocol/models/user_action_type.py @@ -29,6 +29,7 @@ class UserActionType(str, Enum): AUTOMATION_CREATE = 'automation_create' AUTOMATION_EDIT = 'automation_edit' AUTOMATION_STOP = 'automation_stop' + AUTOMATION_RESTART = 'automation_restart' AUTOMATION_SIGNAL = 'automation_signal' ACCOUNT_CREATE = 'account_create' ACCOUNT_EDIT = 'account_edit' diff --git a/packages/protocol/octobot_protocol_ts/models/RestartAutomationConfiguration.ts b/packages/protocol/octobot_protocol_ts/models/RestartAutomationConfiguration.ts new file mode 100644 index 0000000000..1e69d83d92 --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/RestartAutomationConfiguration.ts @@ -0,0 +1,51 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { UserActionType } from '../models/UserActionType'; + +/** +* RestartAutomationConfiguration +*/ +export class RestartAutomationConfiguration { + 'id': string; + /** + * automation_restart + */ + 'action_type': 'automation_restart'; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: {[index: string]: string} | undefined = undefined; + + static readonly attributeTypeMap: Array<{name: string, baseName: string, type: string, format: string}> = [ + { + "name": "id", + "baseName": "id", + "type": "string", + "format": "" + }, + { + "name": "action_type", + "baseName": "action_type", + "type": "UserActionType", + "format": "" + } ]; + + static getAttributeTypeMap() { + return RestartAutomationConfiguration.attributeTypeMap; + } + + public constructor() { + } +} + + diff --git a/packages/protocol/octobot_protocol_ts/models/UserActionConfiguration.ts b/packages/protocol/octobot_protocol_ts/models/UserActionConfiguration.ts index 0422574164..2bc1cdbf1e 100644 --- a/packages/protocol/octobot_protocol_ts/models/UserActionConfiguration.ts +++ b/packages/protocol/octobot_protocol_ts/models/UserActionConfiguration.ts @@ -25,6 +25,7 @@ import { EditAutomationConfiguration } from '../models/EditAutomationConfigurati import { EditExchangeConfigConfiguration } from '../models/EditExchangeConfigConfiguration'; import { EditStrategyConfiguration } from '../models/EditStrategyConfiguration'; import { RefreshAccountsConfiguration } from '../models/RefreshAccountsConfiguration'; +import { RestartAutomationConfiguration } from '../models/RestartAutomationConfiguration'; import { SignalAutomationConfiguration } from '../models/SignalAutomationConfiguration'; import { StopAutomationConfiguration } from '../models/StopAutomationConfiguration'; @@ -33,7 +34,7 @@ import { StopAutomationConfiguration } from '../models/StopAutomationConfigurati * Type * @export */ -export type UserActionConfiguration = CreateAccountAuthConfiguration | CreateAccountConfiguration | CreateAutomationConfiguration | CreateExchangeConfigConfiguration | CreateStrategyConfiguration | DeleteAccountAuthConfiguration | DeleteAccountConfiguration | DeleteExchangeConfigConfiguration | DeleteStrategyConfiguration | EditAccountAuthConfiguration | EditAccountConfiguration | EditAutomationConfiguration | EditExchangeConfigConfiguration | EditStrategyConfiguration | RefreshAccountsConfiguration | SignalAutomationConfiguration | StopAutomationConfiguration; +export type UserActionConfiguration = CreateAccountAuthConfiguration | CreateAccountConfiguration | CreateAutomationConfiguration | CreateExchangeConfigConfiguration | CreateStrategyConfiguration | DeleteAccountAuthConfiguration | DeleteAccountConfiguration | DeleteExchangeConfigConfiguration | DeleteStrategyConfiguration | EditAccountAuthConfiguration | EditAccountConfiguration | EditAutomationConfiguration | EditExchangeConfigConfiguration | EditStrategyConfiguration | RefreshAccountsConfiguration | RestartAutomationConfiguration | SignalAutomationConfiguration | StopAutomationConfiguration; /** * @type UserActionConfigurationClass @@ -52,6 +53,7 @@ export class UserActionConfigurationClass { "accounts_refresh": "RefreshAccountsConfiguration", "automation_create": "CreateAutomationConfiguration", "automation_edit": "EditAutomationConfiguration", + "automation_restart": "RestartAutomationConfiguration", "automation_signal": "SignalAutomationConfiguration", "automation_stop": "StopAutomationConfiguration", "exchange_config_create": "CreateExchangeConfigConfiguration", @@ -78,3 +80,4 @@ export class UserActionConfigurationClass { + diff --git a/packages/protocol/octobot_protocol_ts/models/UserActionType.ts b/packages/protocol/octobot_protocol_ts/models/UserActionType.ts index 910a785bfe..d87710ca2f 100644 --- a/packages/protocol/octobot_protocol_ts/models/UserActionType.ts +++ b/packages/protocol/octobot_protocol_ts/models/UserActionType.ts @@ -14,4 +14,4 @@ /** * UserActionType */ -export type UserActionType = 'automation_create' | 'automation_edit' | 'automation_stop' | 'automation_signal' | 'account_create' | 'account_edit' | 'account_delete' | 'accounts_refresh' | 'exchange_config_create' | 'exchange_config_edit' | 'exchange_config_delete' | 'strategy_create' | 'strategy_edit' | 'strategy_delete' | 'account_auth_create' | 'account_auth_edit' | 'account_auth_delete' +export type UserActionType = 'automation_create' | 'automation_edit' | 'automation_stop' | 'automation_restart' | 'automation_signal' | 'account_create' | 'account_edit' | 'account_delete' | 'accounts_refresh' | 'exchange_config_create' | 'exchange_config_edit' | 'exchange_config_delete' | 'strategy_create' | 'strategy_edit' | 'strategy_delete' | 'account_auth_create' | 'account_auth_edit' | 'account_auth_delete' diff --git a/packages/protocol/octobot_protocol_ts/models/index.ts b/packages/protocol/octobot_protocol_ts/models/index.ts index db1897ead0..0155f9774c 100644 --- a/packages/protocol/octobot_protocol_ts/models/index.ts +++ b/packages/protocol/octobot_protocol_ts/models/index.ts @@ -82,6 +82,7 @@ export * from "./Position"; export * from "./PositionStatus"; export * from "./PositionSummary"; export * from "./RefreshAccountsConfiguration"; +export * from "./RestartAutomationConfiguration"; export * from "./Side"; export * from "./SignalAutomationConfiguration"; export * from "./SignalAutomationConfigurationSignalPayload"; diff --git a/packages/protocol/openapi.json b/packages/protocol/openapi.json index 41b3451ab9..3838c15a7c 100644 --- a/packages/protocol/openapi.json +++ b/packages/protocol/openapi.json @@ -1693,6 +1693,23 @@ } } }, + "RestartAutomationConfiguration": { + "description": "RestartAutomationConfiguration", + "type": "object", + "required": [ + "id", + "action_type" + ], + "properties": { + "id": { + "type": "string" + }, + "action_type": { + "$ref": "#/components/schemas/UserActionType", + "description": "automation_restart" + } + } + }, "AutomationSignalType": { "description": "AutomationSignalType", "type": "string", @@ -2015,6 +2032,7 @@ "automation_create", "automation_edit", "automation_stop", + "automation_restart", "automation_signal", "account_create", "account_edit", @@ -2228,6 +2246,9 @@ { "$ref": "#/components/schemas/StopAutomationConfiguration" }, + { + "$ref": "#/components/schemas/RestartAutomationConfiguration" + }, { "$ref": "#/components/schemas/SignalAutomationConfiguration" }, @@ -2277,6 +2298,7 @@ "automation_create": "#/components/schemas/CreateAutomationConfiguration", "automation_edit": "#/components/schemas/EditAutomationConfiguration", "automation_stop": "#/components/schemas/StopAutomationConfiguration", + "automation_restart": "#/components/schemas/RestartAutomationConfiguration", "automation_signal": "#/components/schemas/SignalAutomationConfiguration", "account_create": "#/components/schemas/CreateAccountConfiguration", "account_edit": "#/components/schemas/EditAccountConfiguration", diff --git a/packages/protocol/test/test_restart_automation_configuration.py b/packages/protocol/test/test_restart_automation_configuration.py new file mode 100644 index 0000000000..f75cbbc985 --- /dev/null +++ b/packages/protocol/test/test_restart_automation_configuration.py @@ -0,0 +1,54 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.restart_automation_configuration import RestartAutomationConfiguration + +class TestRestartAutomationConfiguration(unittest.TestCase): + """RestartAutomationConfiguration unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> RestartAutomationConfiguration: + """Test RestartAutomationConfiguration + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `RestartAutomationConfiguration` + """ + model = RestartAutomationConfiguration() + if include_optional: + return RestartAutomationConfiguration( + id = '', + action_type = 'automation_create' + ) + else: + return RestartAutomationConfiguration( + id = '', + action_type = 'automation_create', + ) + """ + + def testRestartAutomationConfiguration(self): + """Test RestartAutomationConfiguration""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/services/octobot_services/constants.py b/packages/services/octobot_services/constants.py index e8c0ec9bee..f520b564eb 100644 --- a/packages/services/octobot_services/constants.py +++ b/packages/services/octobot_services/constants.py @@ -13,6 +13,7 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +import os # Config CONFIG_CATEGORY_SERVICES = "services" @@ -253,3 +254,5 @@ # external resources EXTERNAL_RESOURCE_CURRENT_USER_FORM = "current-user-feedback-form" EXTERNAL_RESOURCE_PUBLIC_ANNOUNCEMENTS = "public-announcements" + +SERVICE_STOP_TIMEOUT_SECONDS = float(os.getenv("OCTOBOT_SERVICE_STOP_TIMEOUT_SECONDS", "5.0")) diff --git a/packages/services/octobot_services/managers/service_manager.py b/packages/services/octobot_services/managers/service_manager.py index 73b5fafe69..4c32fb2d00 100644 --- a/packages/services/octobot_services/managers/service_manager.py +++ b/packages/services/octobot_services/managers/service_manager.py @@ -13,18 +13,33 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +import asyncio + import octobot_commons.logging as logging +import octobot_services.constants as constants import octobot_services.services as services - async def stop_services(): + logger = logging.get_logger(__name__) for service_instance in _get_service_instances(): + service_name = service_instance.get_name() try: - logging.get_logger(__name__).debug(f"Stopping {service_instance.get_name()} ...") - await service_instance.stop() - logging.get_logger(__name__).debug(f"Stopped {service_instance.get_name()}") - except Exception as e: - raise e + logger.debug(f"Stopping {service_name} ...") + await asyncio.wait_for( + service_instance.stop(), + timeout=constants.SERVICE_STOP_TIMEOUT_SECONDS, + ) + logger.debug(f"Stopped {service_name}") + except asyncio.TimeoutError: + logger.warning( + f"Timed out stopping {service_name} after {constants.SERVICE_STOP_TIMEOUT_SECONDS}s, continuing shutdown" + ) + except Exception as error: + logger.exception( + error, + True, + f"Error when stopping {service_name}: {error}", + ) def _get_service_instances(): diff --git a/packages/services/tests/test_service_manager.py b/packages/services/tests/test_service_manager.py new file mode 100644 index 0000000000..7f570c841c --- /dev/null +++ b/packages/services/tests/test_service_manager.py @@ -0,0 +1,56 @@ +# Drakkar-Software OctoBot-Services +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import asyncio + +import mock +import pytest + +import octobot_services.constants as services_constants +import octobot_services.managers.service_manager as service_manager_module + + +pytestmark = pytest.mark.asyncio + + +class _SlowService: + def get_name(self): + return "SlowService" + + async def stop(self): + await asyncio.sleep(60) + + +class _FastService: + stopped = False + + def get_name(self): + return "FastService" + + async def stop(self): + _FastService.stopped = True + + +class TestStopServices: + async def test_continues_after_slow_service_timeout(self): + _FastService.stopped = False + with mock.patch.object( + service_manager_module, + "_get_service_instances", + return_value=[_SlowService(), _FastService()], + ): + with mock.patch.object(services_constants, "SERVICE_STOP_TIMEOUT_SECONDS", 0.01): + await service_manager_module.stop_services() + assert _FastService.stopped is True diff --git a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_provider.py b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_provider.py index 1028659d47..6b61185367 100644 --- a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_provider.py +++ b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_provider.py @@ -18,11 +18,10 @@ import abc import typing -import cachetools - import octobot_sync.sync.collection_backend.abstract_local_collection_provider as abstract_provider import octobot_sync.sync.collection_backend.base_local_collection_storage as base_storage import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_backend.file_checksum_tracked_cache as file_checksum_tracked_cache import octobot_sync.sync.collection_backend.state_model as state_model import octobot_sync.sync.collection_backend.tolerant_state_loading as tolerant_state_loading @@ -64,9 +63,12 @@ def _create_storage( ) def _setup_caches(self) -> None: - self._cache: cachetools.TTLCache[str, S] = cachetools.TTLCache( - maxsize=self._CACHE_MAXSIZE, - ttl=self._CACHE_TTL_SECONDS, + self._cache: file_checksum_tracked_cache.FileChecksumTrackedCache[str, S] = ( + file_checksum_tracked_cache.FileChecksumTrackedCache( + self._storage, + maxsize=self._CACHE_MAXSIZE, + ttl=self._CACHE_TTL_SECONDS, + ) ) @abc.abstractmethod @@ -81,10 +83,10 @@ def _get_item_id_for_key(self, items_key: str, item: typing.Any) -> str: return self._get_item_id(item) def _get_cached_state(self, user_id: str) -> S | None: - return self._cache.get(user_id) + return self._cache.get_if_fresh(user_id, user_id) def _set_cached_state(self, user_id: str, state: S) -> None: - self._cache[user_id] = state + self._cache.set(user_id, user_id, state) def _empty_state(self) -> S: return typing.cast( diff --git a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py index aeaa8b6e3a..05370b1a2f 100644 --- a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py +++ b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py @@ -16,6 +16,7 @@ import datetime +import hashlib import json import os import pathlib @@ -32,6 +33,9 @@ import octobot_sync.sync.collection_backend.tolerant_state_loading as tolerant_state_loading +_MISSING_FILE_CHECKSUM = "" + + class BaseLocalCollectionStorage: """ Thread-safe, per-wallet-user_id encrypted collection storage. @@ -41,7 +45,7 @@ class BaseLocalCollectionStorage: """ def __init__(self, collection: str, base_folder: typing.Optional[str] = None) -> None: - root = base_folder or user_root_folder_provider.get_user_root_folder() + root = base_folder or user_root_folder_provider.get_sync_data_root() self.collection = collection self._root = pathlib.Path(root) / collection self._lock = threading.Lock() @@ -62,6 +66,15 @@ def _missing_data_error(self, storage_key: str) -> collection_errors.CollectionN f"{self.collection} file does not exist for user_id {storage_key}" ) + def get_file_checksum(self, storage_key: str) -> str: + """Return the SHA-256 hex digest of the raw on-disk collection file bytes.""" + path = self._file_path(storage_key) + if not path.exists(): + return _MISSING_FILE_CHECKSUM + with self._lock: + with open(path, "rb") as handle: + return hashlib.sha256(handle.read()).hexdigest() + def _payload_to_json_bytes(self, payload: state_model.StateModel) -> bytes: """Serialize a state dict to JSON bytes (handles datetime values from protocol models).""" diff --git a/packages/sync/octobot_sync/sync/collection_backend/file_checksum_tracked_cache.py b/packages/sync/octobot_sync/sync/collection_backend/file_checksum_tracked_cache.py new file mode 100644 index 0000000000..1819a9e434 --- /dev/null +++ b/packages/sync/octobot_sync/sync/collection_backend/file_checksum_tracked_cache.py @@ -0,0 +1,67 @@ +# Drakkar-Software OctoBot-Sync +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + + +import dataclasses +import typing + +import cachetools + +import octobot_sync.sync.collection_backend.base_local_collection_storage as base_storage +import octobot_sync.sync.collection_backend.state_model as state_model + + +CacheKeyT = typing.TypeVar("CacheKeyT") +S = typing.TypeVar("S", bound=state_model.StateModel) + + +@dataclasses.dataclass(frozen=True, slots=True) +class CachedStateEnvelope(typing.Generic[S]): + state: S + file_checksum: str + + +class FileChecksumTrackedCache(typing.Generic[CacheKeyT, S]): + """TTL cache that invalidates entries when the backing collection file changes.""" + + def __init__( + self, + storage: base_storage.BaseLocalCollectionStorage, + *, + maxsize: int, + ttl: float, + ) -> None: + self._storage = storage + self._cache: cachetools.TTLCache[CacheKeyT, CachedStateEnvelope[S]] = cachetools.TTLCache( + maxsize=maxsize, + ttl=ttl, + ) + + def get_if_fresh(self, cache_key: CacheKeyT, storage_key: str) -> S | None: + envelope = self._cache.get(cache_key) + if envelope is None: + return None + current_checksum = self._storage.get_file_checksum(storage_key) + if current_checksum != envelope.file_checksum: + self._cache.pop(cache_key, None) + return None + return envelope.state + + def set(self, cache_key: CacheKeyT, storage_key: str, state: S) -> None: + self._cache[cache_key] = CachedStateEnvelope( + state=state, + file_checksum=self._storage.get_file_checksum(storage_key), + ) diff --git a/packages/sync/octobot_sync/sync/collection_backend/single_item_local_collection_provider.py b/packages/sync/octobot_sync/sync/collection_backend/single_item_local_collection_provider.py index e69ebb80e3..f78820419c 100644 --- a/packages/sync/octobot_sync/sync/collection_backend/single_item_local_collection_provider.py +++ b/packages/sync/octobot_sync/sync/collection_backend/single_item_local_collection_provider.py @@ -17,10 +17,9 @@ import typing -import cachetools - import octobot_sync.sync.collection_backend.abstract_local_collection_provider as abstract_provider import octobot_sync.sync.collection_backend.base_local_collection_storage as base_storage +import octobot_sync.sync.collection_backend.file_checksum_tracked_cache as file_checksum_tracked_cache import octobot_sync.sync.collection_backend.single_item_local_collection_storage as single_item_storage import octobot_sync.sync.collection_backend.state_model as state_model @@ -47,9 +46,12 @@ def _create_storage( ) def _setup_caches(self) -> None: - self._state_cache: cachetools.TTLCache[tuple[str, str], S] = cachetools.TTLCache( - maxsize=self._CACHE_MAXSIZE, - ttl=self._CACHE_TTL_SECONDS, + self._state_cache: file_checksum_tracked_cache.FileChecksumTrackedCache[tuple[str, str], S] = ( + file_checksum_tracked_cache.FileChecksumTrackedCache( + self._storage, + maxsize=self._CACHE_MAXSIZE, + ttl=self._CACHE_TTL_SECONDS, + ) ) def _build_identifier(self, user_id: str, account_id: str) -> str: @@ -58,10 +60,12 @@ def _build_identifier(self, user_id: str, account_id: str) -> str: ) def _get_cached_state(self, user_id: str, account_id: str) -> S | None: - return self._state_cache.get((user_id, account_id)) + identifier = self._build_identifier(user_id, account_id) + return self._state_cache.get_if_fresh((user_id, account_id), identifier) def _set_cached_state(self, user_id: str, account_id: str, state: S) -> None: - self._state_cache[(user_id, account_id)] = state + identifier = self._build_identifier(user_id, account_id) + self._state_cache.set((user_id, account_id), identifier, state) def load_state(self, user_id: str, account_id: str) -> S: cached_state = self._get_cached_state(user_id, account_id) diff --git a/packages/sync/octobot_sync/sync/collections.py b/packages/sync/octobot_sync/sync/collections.py index cb27e0a500..a42b911b21 100644 --- a/packages/sync/octobot_sync/sync/collections.py +++ b/packages/sync/octobot_sync/sync/collections.py @@ -150,12 +150,10 @@ def load_sync_config( path = collections_path or os.path.join( commons_constants.USER_FOLDER, constants.COLLECTIONS_FILE ) - if not os.path.isfile(path): - logger.warning( - f"Collections file not found at {path}, using default config" - ) - return DEFAULT_SYNC_CONFIG - return load_config_file(path) + if os.path.isfile(path): + logger.warning(f"Using custom collections file at {path}") + return load_config_file(path) + return DEFAULT_SYNC_CONFIG def is_replicable_collection(col: CollectionConfig) -> bool: diff --git a/packages/sync/tests/sync/collection_backend/test_abstract_local_collection_provider.py b/packages/sync/tests/sync/collection_backend/test_abstract_local_collection_provider.py index ca991ba570..9383bafc22 100644 --- a/packages/sync/tests/sync/collection_backend/test_abstract_local_collection_provider.py +++ b/packages/sync/tests/sync/collection_backend/test_abstract_local_collection_provider.py @@ -16,13 +16,13 @@ import typing -import cachetools import mock import pydantic import octobot.community.authentication as community_authentication import octobot_sync.sync.collection_backend.abstract_local_collection_provider as abstract_provider_module import octobot_sync.sync.collection_backend.base_local_collection_storage as base_storage_module +import octobot_sync.sync.collection_backend.file_checksum_tracked_cache as file_checksum_tracked_cache_module import octobot_sync.sync.collection_backend.single_item_local_collection_storage as single_item_storage_module _TEST_ADDRESS = "0xaaabbbcccddd" @@ -49,7 +49,8 @@ def _create_storage( ) def _setup_caches(self) -> None: - self._state_cache: cachetools.TTLCache[tuple[str, str], _TestState] = cachetools.TTLCache( + self._state_cache = file_checksum_tracked_cache_module.FileChecksumTrackedCache( + self._storage, maxsize=self._CACHE_MAXSIZE, ttl=self._CACHE_TTL_SECONDS, ) @@ -83,7 +84,10 @@ def test_calls_setup_caches(self, tmp_path): provider = _make_provider(tmp_path) assert hasattr(provider, "_state_cache") - assert isinstance(provider._state_cache, cachetools.TTLCache) + assert isinstance( + provider._state_cache, + file_checksum_tracked_cache_module.FileChecksumTrackedCache, + ) class TestAbstractLocalCollectionProviderGetWalletPrivateKey: diff --git a/packages/sync/tests/sync/collection_backend/test_base_local_collection_provider.py b/packages/sync/tests/sync/collection_backend/test_base_local_collection_provider.py index da6fa26fbe..2098057418 100644 --- a/packages/sync/tests/sync/collection_backend/test_base_local_collection_provider.py +++ b/packages/sync/tests/sync/collection_backend/test_base_local_collection_provider.py @@ -385,3 +385,25 @@ def test_unknown_items_key_raises(self, tmp_path): with pytest.raises(collection_errors.UnsupportedItemsKeyError): provider._get_item_id_for_key("unknown_key", _item("item-1")) + +class TestBaseLocalCollectionProviderCacheInvalidation: + def test_reloads_from_disk_when_file_changed_externally(self, tmp_path): + provider = _make_provider(tmp_path) + with _patch_wallet(): + provider.create_item(_TEST_ADDRESS, _item("item-1", label="Cached")) + provider.list_items(_TEST_ADDRESS) + + external_state = _TestState( + version="1.0.0", + items=[_TestItem(id="external", label="From disk")], + ) + provider._storage.save_state(_TEST_ADDRESS, _TEST_PRIVATE_KEY, external_state) + + with _patch_wallet(): + listed = provider.list_items(_TEST_ADDRESS) + + assert len(listed) == 1 + assert listed[0].id == "external" + assert listed[0].label == "From disk" + + diff --git a/packages/sync/tests/sync/collection_backend/test_base_local_collection_storage.py b/packages/sync/tests/sync/collection_backend/test_base_local_collection_storage.py index fbb57bf612..ee15c1d556 100644 --- a/packages/sync/tests/sync/collection_backend/test_base_local_collection_storage.py +++ b/packages/sync/tests/sync/collection_backend/test_base_local_collection_storage.py @@ -248,3 +248,37 @@ def test_different_collections_are_isolated(self, tmp_path): assert storage_a.load_state(_TEST_ADDRESS, _TEST_PRIVATE_KEY, TestStateModel) == state assert storage_b.load_state(_TEST_ADDRESS, _TEST_PRIVATE_KEY, TestStateModel) == TestStateModel(version="1.0.0", items=[TestItemModel(id="b1")]) + + +class TestBaseLocalCollectionStorageGetFileChecksum: + def test_returns_empty_string_when_file_absent(self, tmp_path): + storage = _make_storage(tmp_path) + + checksum = storage.get_file_checksum(_TEST_ADDRESS) + + assert checksum == "" + + def test_returns_stable_checksum_for_same_file_bytes(self, tmp_path): + storage = _make_storage(tmp_path) + storage.save_state(_TEST_ADDRESS, _TEST_PRIVATE_KEY, _SAMPLE_STATE) + + first_checksum = storage.get_file_checksum(_TEST_ADDRESS) + second_checksum = storage.get_file_checksum(_TEST_ADDRESS) + + assert first_checksum == second_checksum + assert len(first_checksum) == 64 + + def test_returns_different_checksum_after_file_rewrite(self, tmp_path): + storage = _make_storage(tmp_path) + storage.save_state(_TEST_ADDRESS, _TEST_PRIVATE_KEY, _SAMPLE_STATE) + first_checksum = storage.get_file_checksum(_TEST_ADDRESS) + + storage.save_state( + _TEST_ADDRESS, + _TEST_PRIVATE_KEY, + TestStateModel(version="1.0.0", items=[TestItemModel(id="item-3")]), + ) + second_checksum = storage.get_file_checksum(_TEST_ADDRESS) + + assert first_checksum != second_checksum + diff --git a/packages/sync/tests/sync/collection_backend/test_single_item_local_collection_provider.py b/packages/sync/tests/sync/collection_backend/test_single_item_local_collection_provider.py index 44f469dcea..1d7ac91274 100644 --- a/packages/sync/tests/sync/collection_backend/test_single_item_local_collection_provider.py +++ b/packages/sync/tests/sync/collection_backend/test_single_item_local_collection_provider.py @@ -132,6 +132,29 @@ def test_persists_and_updates_cache(self, tmp_path): assert persisted_state == _SAMPLE_STATE +class TestSingleItemLocalCollectionProviderCacheInvalidation: + def test_reloads_from_disk_when_file_changed_externally(self, tmp_path): + provider = _make_provider(tmp_path) + with _patch_wallet(): + provider.save_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID, _SAMPLE_STATE) + provider.load_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID) + + external_state = _TestState( + version="1.0.0", + items=[_TestItem(id="external", label="From disk")], + ) + identifier = provider._build_identifier(_TEST_ADDRESS, _TEST_ACCOUNT_ID) + provider._storage.save_state(identifier, _TEST_PRIVATE_KEY, external_state) + + with _patch_wallet(): + loaded_state = provider.load_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID) + + assert loaded_state.items is not None + assert len(loaded_state.items) == 1 + assert loaded_state.items[0].id == "external" + assert loaded_state.items[0].label == "From disk" + + class TestSingleItemLocalCollectionProviderLoadStateEncrypted: def test_reads_encrypted_blob_for_account_id(self, tmp_path): provider = _make_provider(tmp_path) diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py index bf39a5dfe2..e7d0a6f477 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py @@ -16,8 +16,10 @@ import asyncio import json import os +import pathlib import shutil import sys +import threading import time import types import typing @@ -35,7 +37,7 @@ import octobot_commons.profiles.profile_data as profile_data_module import octobot_commons.profiles.profile_data_import as profile_data_import import octobot_commons.profiles.exchange_auth_data as exchange_auth_data_module -import octobot_commons.profiles.profile as profiles_profile_module +import octobot_commons.profiles.profile_types.profile as profiles_profile_module import octobot_commons.profiles.tentacles_profile_data_translator as tentacles_profile_data_translator import octobot_commons.enums as commons_enums import octobot_commons.configuration @@ -46,6 +48,9 @@ import octobot_flow.entities.accounts.process_bot_state as process_bot_state_import import octobot_node.constants as octobot_node_constants import octobot_services.constants as services_constants +import octobot_protocol.models.generic_process_configuration as generic_process_configuration +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers # Written only after a successful full init so re-runs can detect an existing per-bot tree. DSL_PREPARED_MARKER = ".octobot_dsl_prepared" @@ -328,6 +333,104 @@ async def _convert_profile_data_to_profile_directory( ) +def _path_segments(relative_path: str) -> tuple[str, ...]: + return tuple( + segment + for segment in str(relative_path).replace("\\", "/").split("/") + if segment + ) + + +def _assert_automation_child_config_path(config_path: str) -> None: + """ + Reject writes outside ``user/automations//config.json`` (master ``user/config.json`` is forbidden). + """ + normalized_config_path = os.path.normpath(config_path) + if os.path.basename(normalized_config_path) != commons_constants.CONFIG_FILE: + raise commons_errors.DSLInterpreterError( + f"Process child config must be named {commons_constants.CONFIG_FILE!r}, not {config_path!r}." + ) + path_segments = pathlib.PurePath(normalized_config_path).parts + automation_prefix = ( + commons_constants.USER_FOLDER, + commons_constants.AUTOMATIONS_FOLDER, + ) + prefix_length = len(automation_prefix) + for segment_index in range(len(path_segments) - prefix_length): + if path_segments[segment_index : segment_index + prefix_length] != automation_prefix: + continue + leaf_segments = path_segments[segment_index + prefix_length : -1] + if not leaf_segments: + raise commons_errors.DSLInterpreterError( + f"Process child config must live under " + f"{commons_constants.USER_AUTOMATIONS_FOLDER}//{commons_constants.CONFIG_FILE}, " + f"not {config_path!r}." + ) + if ".." in leaf_segments: + raise commons_errors.DSLInterpreterError( + f"Process child config path must not contain parent segments: {config_path!r}." + ) + return + raise commons_errors.DSLInterpreterError( + f"Process child config must be under {commons_constants.USER_AUTOMATIONS_FOLDER}//, " + f"not {config_path!r}." + ) + + +def _assert_automation_rel_folder( + rel_folder: str, + expected_prefix: tuple[str, ...], + *, + cli_flag_label: str, + expected_folder_path: str, +) -> None: + path_segments = _path_segments(rel_folder) + if len(path_segments) < len(expected_prefix) + 1: + raise commons_errors.DSLInterpreterError( + f"Process child {cli_flag_label} must be under {expected_folder_path}//, " + f"got {rel_folder!r}." + ) + if path_segments[: len(expected_prefix)] != expected_prefix: + raise commons_errors.DSLInterpreterError( + f"Process child {cli_flag_label} must start with {expected_folder_path}/, " + f"got {rel_folder!r}." + ) + if ".." in path_segments: + raise commons_errors.DSLInterpreterError( + f"Process child {cli_flag_label} must not contain parent segments: {rel_folder!r}." + ) + + +def _assert_automation_rel_user_folder(rel_user_folder: str) -> None: + _assert_automation_rel_folder( + rel_user_folder, + ( + commons_constants.USER_FOLDER, + commons_constants.AUTOMATIONS_FOLDER, + ), + cli_flag_label="--user-folder", + expected_folder_path=commons_constants.USER_AUTOMATIONS_FOLDER, + ) + + +def _assert_automation_rel_log_folder(rel_log_folder: str) -> None: + _assert_automation_rel_folder( + rel_log_folder, + tuple(octobot_node_constants.AUTOMATION_LOGS_FOLDER.split("/")), + cli_flag_label="--log-folder", + expected_folder_path=octobot_node_constants.AUTOMATION_LOGS_FOLDER, + ) + + +def _assert_spawn_cmd_isolation(cmd: list[str], rel_user: str, rel_log: str) -> None: + if "--user-folder" not in cmd or "--log-folder" not in cmd: + raise commons_errors.DSLInterpreterError( + "Process child spawn command must include --user-folder and --log-folder." + ) + _assert_automation_rel_user_folder(rel_user) + _assert_automation_rel_log_folder(rel_log) + + def _write_user_root_config_json( config_path: str, profile_id: str, @@ -335,16 +438,20 @@ def _write_user_root_config_json( exchange_auth_data: typing.Optional[ list[exchange_auth_data_module.ExchangeAuthData] ] = None, + readonly_profiles_path: str | None = None, ) -> None: """ Writes user-root ``config.json``: selected profile, disabled web auto-open for DSL-spawned processes, optional exchange stubs from ``profile_data``, then credentials from ``exchange_auth_data`` (merged into ``exchanges``). """ + _assert_automation_child_config_path(config_path) # Load packaged defaults; pin profile and disable browser auto-open for headless DSL children. default_cfg = json_util.read_file(octobot_constants.DEFAULT_CONFIG_FILE) default_cfg[commons_constants.CONFIG_PROFILE] = profile_id default_cfg[commons_constants.CONFIG_ACCEPTED_TERMS] = True + if readonly_profiles_path: + default_cfg[commons_constants.CONFIG_READONLY_PROFILES_PATH] = readonly_profiles_path services_cfg = default_cfg.setdefault(services_constants.CONFIG_CATEGORY_SERVICES, {}) web_cfg = services_cfg.setdefault(services_constants.CONFIG_WEB, {}) web_cfg[services_constants.CONFIG_AUTO_OPEN_IN_WEB_BROWSER] = False @@ -374,17 +481,6 @@ def _write_user_root_config_json( json_util.safe_dump(default_cfg, config_path) -def _executor_non_trading_profile_source(working_directory: str) -> str: - return os.path.normpath( - os.path.join( - working_directory, - commons_constants.USER_FOLDER, - commons_constants.PROFILES_FOLDER, - DEFAULT_DSL_PROFILE_ID, - ) - ) - - def _executor_profiles_directory(working_directory: str) -> str: return os.path.normpath( os.path.join( @@ -395,57 +491,40 @@ def _executor_profiles_directory(working_directory: str) -> str: ) -async def _copy_read_only_profiles_to_user_root( +def _child_master_profile_config_kwargs( working_directory: str, - user_root: str, - *, - active_profile_id: str, -) -> None: - """ - Copy read-only profiles from the master OctoBot into a generic process child layout. +) -> dict[str, typing.Any]: + return { + "readonly_profiles_path": _executor_profiles_directory(working_directory), + } - Generic process bots start on the default non-trading profile but should still see - the same read-only strategy profiles as the master (community/imported templates). - Editable profiles are intentionally omitted so each child keeps its own user edits. - """ - profiles_src = _executor_profiles_directory(working_directory) - if not os.path.isdir(profiles_src): - return - for profile in profiles_profile_module.Profile.get_all_profiles(profiles_src): - if not profile.read_only: - continue - # Active profile was already copied by _copy_non_trading_profile_to_user_root. - if profile.profile_id == active_profile_id: - continue - destination_profile_path = os.path.join( - user_root, - commons_constants.PROFILES_FOLDER, - profile.profile_id, - ) - if os.path.exists(destination_profile_path): - shutil.rmtree(destination_profile_path) - shutil.copytree(profile.path, destination_profile_path) +def _get_sync_strategy(sync_user_id: str, strategy_id: str) -> typing.Any: + return collection_providers.StrategyProvider.instance().get_item( + sync_user_id, + strategy_id, + ) -async def _copy_non_trading_profile_to_user_root( - working_directory: str, - user_root: str, -) -> str: - source_profile_path = _executor_non_trading_profile_source(working_directory) - if not os.path.isdir(source_profile_path): + +def _sync_strategy_has_profile_data(strategy: typing.Any) -> bool: + configuration = strategy.configuration + if configuration is None or configuration.actual_instance is None: + return False + if not isinstance( + configuration.actual_instance, + generic_process_configuration.GenericProcessConfiguration, + ): + return False + return configuration.actual_instance.profile_data is not None + + +def _assert_sync_strategy_exists(sync_user_id: str, sync_profile_id: str) -> typing.Any: + try: + return _get_sync_strategy(sync_user_id, sync_profile_id) + except collection_errors.ItemNotFoundError as err: raise commons_errors.DSLInterpreterError( - f"Default profile not found at {source_profile_path!r}; expected " - f"{DEFAULT_DSL_PROFILE_ID!r} under the OctoBot user profiles folder." - ) - destination_profile_path = os.path.join( - user_root, - commons_constants.PROFILES_FOLDER, - DEFAULT_DSL_PROFILE_ID, - ) - if os.path.exists(destination_profile_path): - shutil.rmtree(destination_profile_path) - shutil.copytree(source_profile_path, destination_profile_path) - return DEFAULT_DSL_PROFILE_ID + f"sync strategy {sync_profile_id!r} not found for sync user {sync_user_id!r}." + ) from err async def ensure_user_profile_and_layout( @@ -456,6 +535,9 @@ async def ensure_user_profile_and_layout( exchange_auth_data: typing.Optional[ list[exchange_auth_data_module.ExchangeAuthData] ] = None, + *, + sync_profile_id: str | None = None, + user_id: str | None = None, ) -> dict[str, typing.Any]: """ One-time layout under user_root (/user/automations//): @@ -463,9 +545,7 @@ async def ensure_user_profile_and_layout( Idempotent when config.json + marker both exist. """ dsl_interpreter.ProcessBoundOperatorMixin.reject_user_path_segment(user_folder) - user_folder_leaf_segments = [ - segment for segment in str(user_folder).replace("\\", "/").split("/") if segment - ] + user_folder_leaf_segments = _path_segments(user_folder) user_root = os.path.normpath( os.path.join( working_directory, @@ -486,24 +566,7 @@ async def ensure_user_profile_and_layout( os.makedirs(user_root, exist_ok=True) - if profile_data_dict is None: - # Generic process: default non-trading profile plus master's read-only profiles. - profile_id = await _copy_non_trading_profile_to_user_root( - working_directory, - user_root, - ) - await _copy_read_only_profiles_to_user_root( - working_directory, - user_root, - active_profile_id=profile_id, - ) - _write_user_root_config_json( - config_path, - profile_id, - None, - exchange_auth_data, - ) - else: + if profile_data_dict is not None: # Import writes to a throwaway folder first: the real profile id is assigned during import (see rename below). temp_profile_path = os.path.join( user_root, @@ -529,7 +592,40 @@ async def ensure_user_profile_and_layout( shutil.rmtree(final_profile_path) os.replace(temp_profile_path, final_profile_path) - _write_user_root_config_json(config_path, profile_id, profile_data, exchange_auth_data) + _write_user_root_config_json( + config_path, + profile_id, + profile_data, + exchange_auth_data, + **_child_master_profile_config_kwargs(working_directory), + ) + elif sync_profile_id is not None: + if not user_id or not str(user_id).strip(): + raise commons_errors.DSLInterpreterError( + f"sync_profile_id={sync_profile_id!r} requires user_id." + ) + strategy = _assert_sync_strategy_exists(str(user_id), sync_profile_id) + if _sync_strategy_has_profile_data(strategy): + profile_id = sync_profile_id + else: + profile_id = DEFAULT_DSL_PROFILE_ID + _write_user_root_config_json( + config_path, + profile_id, + None, + exchange_auth_data, + **_child_master_profile_config_kwargs(working_directory), + ) + else: + # Generic process: master non-trading + read-only profiles via overlay config. + profile_id = DEFAULT_DSL_PROFILE_ID + _write_user_root_config_json( + config_path, + profile_id, + None, + exchange_auth_data, + **_child_master_profile_config_kwargs(working_directory), + ) # Mirror default reference tentacles layout expected by the child. ref_src = source_reference_tentacles_config or os.path.join( @@ -568,7 +664,7 @@ def _read_top_level_profile_id(config_path: str) -> str | None: def _ensure_log_folder_path(working_directory: str, user_folder: str) -> str: """Absolute log directory for this `user_folder` (matches ensure_state.log_folder).""" - log_folder_param_segments = [segment for segment in str(user_folder).replace("\\", "/").split("/") if segment] + log_folder_param_segments = _path_segments(user_folder) return os.path.normpath( os.path.join( working_directory, @@ -578,14 +674,24 @@ def _ensure_log_folder_path(working_directory: str, user_folder: str) -> str: ) -def _ensure_child_environ(web_port: int, node_port: int, bind_host: str) -> dict: - """Environment passed to the OctoBot child (ports and bind addresses).""" +def _ensure_child_environ( + web_port: int, + node_port: int, + bind_host: str, + sync_user_id: str, + working_directory: str, +) -> dict: + """Environment passed to the OctoBot child (ports, bind addresses, sync user id).""" child_env = os.environ.copy() child_env[services_constants.ENV_WEB_PORT] = str(web_port) child_env[services_constants.ENV_WEB_ADDRESS] = bind_host child_env[services_constants.ENV_NODE_API_PORT] = str(node_port) child_env[services_constants.ENV_NODE_API_ADDRESS] = bind_host child_env[commons_constants.ENV_USE_MINIMAL_LIBS] = "false" + child_env[octobot_constants.ENV_PROCESS_BOT_SYNC_USER_ID] = sync_user_id + child_env[commons_constants.ENV_OCTOBOT_SYNC_DATA_ROOT] = os.path.normpath( + os.path.join(working_directory, commons_constants.USER_FOLDER) + ) return child_env @@ -611,28 +717,84 @@ def _ensure_start_cmd( return cmd +_child_listen_ports_reserved: dict[int, str] = {} +_child_listen_ports_lock = threading.Lock() + + +def _reserve_child_listen_ports(web_port: int, node_port: int, user_folder: str) -> None: + with _child_listen_ports_lock: + _child_listen_ports_reserved[web_port] = user_folder + _child_listen_ports_reserved[node_port] = user_folder + + +def _release_child_listen_ports(web_port: int, node_port: int, user_folder: str) -> None: + with _child_listen_ports_lock: + for listen_port in (web_port, node_port): + if _child_listen_ports_reserved.get(listen_port) == user_folder: + _child_listen_ports_reserved.pop(listen_port, None) + + def _listen_port_pair_with_shared_scan_offset( probe_host: str, primary_listen_port_base: int, secondary_listen_port_base: int, *, max_offset: int = 256, + extra_blocklist: set[int] | frozenset[int] | None = None, ) -> tuple[int, int]: """Delegates to ``find_first_free_listen_port_after_base`` paired scan (one loop).""" + primary_blocklist = list(extra_blocklist) if extra_blocklist else None primary_listen_port = os_util.find_first_free_listen_port_after_base( probe_host, primary_listen_port_base, max_offset=max_offset, + blocklist=primary_blocklist, ) + secondary_blocklist = set(extra_blocklist or ()) + secondary_blocklist.add(primary_listen_port) secondary_listen_port = os_util.find_first_free_listen_port_after_base( probe_host, secondary_listen_port_base, max_offset=max_offset, - blocklist=[primary_listen_port], + blocklist=list(secondary_blocklist), ) return primary_listen_port, secondary_listen_port +def _allocate_child_listen_port_pair( + probe_host: str, + primary_listen_port_base: int, + secondary_listen_port_base: int, + user_folder: str, + *, + max_offset: int = 256, +) -> tuple[int, int]: + with _child_listen_ports_lock: + reserved_ports = frozenset(_child_listen_ports_reserved) + web_port, node_port = _listen_port_pair_with_shared_scan_offset( + probe_host, + primary_listen_port_base, + secondary_listen_port_base, + max_offset=max_offset, + extra_blocklist=reserved_ports, + ) + _child_listen_ports_reserved[web_port] = user_folder + _child_listen_ports_reserved[node_port] = user_folder + return web_port, node_port + + +def _release_recall_state_listen_ports( + recall_state: typing.Optional[EnsureOctobotProcessState], +) -> None: + if recall_state is None: + return + _release_child_listen_ports( + recall_state.web_port, + recall_state.node_port, + recall_state.user_folder, + ) + + def create_octobot_process_operators( signals: typing.Optional[dsl_interpreter.OperatorSignals] = None, executor_id: str = "", @@ -691,13 +853,37 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: name="profile_data", description=( "Optional object compatible with octobot_commons.profiles.profile_data.ProfileData. " - "When omitted, the child uses the packaged default config and copies the " - f"{DEFAULT_DSL_PROFILE_ID!r} profile from the executor user profiles folder." + "When omitted, the child uses the packaged default config and selects the " + f"{DEFAULT_DSL_PROFILE_ID!r} profile from the executor user profiles folder " + "via master profile overlay." ), required=False, type=dict, default=None, ), + dsl_interpreter.OperatorParameter( + name="sync_profile_id", + description=( + "Optional sync profile id (strategy id). Used only when profile_data is omitted. " + "Validates the strategy exists in sync for user_id. When the strategy embeds " + "profile_data, selects it in child config.json without a local profiles/ tree; " + f"otherwise the child uses {DEFAULT_DSL_PROFILE_ID!r} from the executor profiles overlay." + ), + required=False, + type=str, + default=None, + ), + dsl_interpreter.OperatorParameter( + name="user_id", + description=( + "Sync wallet user id (same as node task user_id). Required to spawn the " + "process child (passed via environment, not config.json). Also required " + "when sync_profile_id is set for strategy validation." + ), + required=False, + type=str, + default=None, + ), dsl_interpreter.OperatorParameter( name="exchange_auth_data", description=( @@ -860,6 +1046,7 @@ async def _pre_compute_recall_path( resolved_pid = _resolve_bound_pid(recall_state, loaded_state) if resolved_pid is not None: self.pid = resolved_pid + _release_recall_state_listen_ports(recall_state) self.value = self.request_graceful_stop(logger=_get_logger()) raise commons_errors.DSLInterpreterError( "Timed out waiting for OctoBot process_bot_state.json during init (see ping_timeout).", @@ -897,6 +1084,8 @@ async def _pre_compute_recall_path( recall_state.http_base_url, logged_pid, ) + if not recall_state.init_state_ok: + _release_recall_state_listen_ports(recall_state) updated = recall_state.model_copy( update={"init_state_ok": True, "state_file_path": state_path} ) @@ -954,6 +1143,8 @@ async def _pre_compute_first_spawn( params.get("profile_data"), None, exchange_auth, + sync_profile_id=params.get("sync_profile_id"), + user_id=params.get("user_id"), ) user_root = init_info["user_root"] log_folder = _ensure_log_folder_path(working_directory, user_folder) @@ -962,17 +1153,30 @@ async def _pre_compute_first_spawn( params ) ) + prior_recall_state = _parse_ensure_recall_state(last_result) + _release_recall_state_listen_ports(prior_recall_state) web_b = int(params.get("web_port_base") or services_constants.DEFAULT_SERVER_PORT) node_b = int(params.get("node_port_base") or services_constants.DEFAULT_NODE_API_PORT) - web_port, node_port = _listen_port_pair_with_shared_scan_offset( - probe_host, web_b, node_b + web_port, node_port = _allocate_child_listen_port_pair( + probe_host, web_b, node_b, user_folder ) start_script = os.path.join(working_directory, "start.py") if not os.path.isfile(start_script): raise commons_errors.DSLInterpreterError( f"start.py not found at {start_script} (current working directory must be the OctoBot project root)." ) - child_env = _ensure_child_environ(web_port, node_port, bind_host) + process_sync_user_id = params.get("user_id") + if not process_sync_user_id or not str(process_sync_user_id).strip(): + raise commons_errors.DSLInterpreterError( + "run_octobot_process requires user_id to spawn a process child." + ) + child_env = _ensure_child_environ( + web_port, + node_port, + bind_host, + str(process_sync_user_id), + working_directory, + ) rel_user = os.path.relpath(user_root, working_directory) rel_log = os.path.relpath(log_folder, working_directory) state_file_path = os.path.normpath( @@ -985,6 +1189,8 @@ async def _pre_compute_first_spawn( bool(params.get("no_telegram", True)), state_file_path, ) + _assert_spawn_cmd_isolation(cmd, rel_user, rel_log) + _get_logger().info("Spawning OctoBot process child: cmd=%r", cmd) self.spawn_subprocess( cmd, working_directory=working_directory, @@ -1025,6 +1231,7 @@ async def _pre_compute_first_spawn( state_pid, ) ready = state.model_copy(update={"init_state_ok": True}) + _release_child_listen_ports(web_port, node_port, user_folder) self._emit_ensure_recall( state=ready, last_result=last_result, @@ -1092,6 +1299,7 @@ async def _pre_compute_update_config_refresh( logger=process_logger, timeout_seconds=ping_timeout, ) + _release_recall_state_listen_ports(recall_state) process_logger.info("configuration update: removing automation user and log directories") _remove_path_for_fresh_start(recall_state.user_root, logger=process_logger) _remove_path_for_fresh_start(recall_state.log_folder, logger=process_logger) @@ -1123,6 +1331,7 @@ async def pre_compute(self) -> None: if resolved_pid is not None: self.pid = resolved_pid self.value = self.request_graceful_stop(logger=_get_logger()) + _release_recall_state_listen_ports(recall_state) return # Grace with dead metadata pid: child restarting; no SIGTERM, report already_stopped. if _in_restart_grace_period( @@ -1136,6 +1345,7 @@ async def pre_compute(self) -> None: "run_octobot_process(STOP): child in restart grace; treating as already_stopped" ) self.value = {"status": "already_stopped", "reason": "not_running"} + _release_recall_state_listen_ports(recall_state) return working_directory = os.path.normpath(os.getcwd()) user_folder = params["user_folder"] diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py index b07143d73d..25f11ccf9d 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py @@ -36,10 +36,13 @@ import octobot_services.constants as services_constants import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_storage as profile_storage_module import octobot_commons.profiles.exchange_auth_data as exchange_auth_data_module import octobot_flow.entities as octobot_flow_entities import octobot_flow.entities.accounts.process_bot_state as process_bot_state_import import octobot_tentacles_manager.constants as tentacles_manager_constants +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_backend.errors as collection_errors import tentacles.Meta.DSL_operators.octobot_process_operators.octobot_process_ops as octobot_process_ops import tentacles.Trading.Mode.grid_trading_mode.grid_trading as grid_trading_module @@ -47,6 +50,7 @@ # Nested class from factory (not exposed on ``octobot_process_ops``). TEST_EXECUTOR_ID = "test-executor" +_PROCESS_TEST_USER_ID = "wallet-user" EnsureOctobotProcessOperator = octobot_process_ops.create_octobot_process_operators( None, TEST_EXECUTOR_ID )[0] @@ -215,6 +219,31 @@ def _require_octobot_project_root_for_subprocess_tests() -> str: return project_root +def _seed_executor_user_config(working_directory: pathlib.Path) -> None: + user_directory = working_directory / commons_constants.USER_FOLDER + user_directory.mkdir(parents=True, exist_ok=True) + config_path = user_directory / commons_constants.CONFIG_FILE + config_path.write_text("{}", encoding="utf-8") + + +def _generic_process_sync_strategy( + strategy_id: str, + *, + profile_data: dict | None = None, +) -> protocol_models.Strategy: + generic_process_configuration = protocol_models.GenericProcessConfiguration( + configuration_type=protocol_models.ActionConfigurationType.GENERIC_PROCESS, + profile_data=profile_data, + ) + return protocol_models.Strategy( + id=strategy_id, + version="1.0.0", + name="test-sync-strategy", + reference_market="USDT", + configuration=protocol_models.StrategyConfiguration(generic_process_configuration), + ) + + def _seed_executor_non_trading_profile(working_directory: pathlib.Path) -> None: source_profile_path = _octobot_project_root_from_test_file().joinpath( commons_constants.USER_FOLDER, @@ -242,7 +271,9 @@ def _seed_executor_non_trading_profile(working_directory: pathlib.Path) -> None: commons_constants.CONFIG_PROFILE: { commons_constants.CONFIG_ID: octobot_process_ops.DEFAULT_DSL_PROFILE_ID, commons_constants.CONFIG_NAME: octobot_process_ops.DEFAULT_DSL_PROFILE_ID, - } + commons_constants.CONFIG_READ_ONLY: True, + }, + commons_constants.PROFILE_CONFIG: {}, } (minimal_profile_path / commons_constants.PROFILE_CONFIG_FILE).write_text( json.dumps(profile_payload), @@ -294,7 +325,8 @@ async def _poll_dsl_until_init_state_ok( timeout_sec: float = 60.0, ) -> dict: base_arguments = ( - f"{user_folder!r}, exchange_auth_data={repr(exchange_auth_list)}, " + f"{user_folder!r}, user_id={_PROCESS_TEST_USER_ID!r}, " + f"exchange_auth_data={repr(exchange_auth_list)}, " f"waiting_time={_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SEC}, ping_timeout=30.0" ) deadline = time.monotonic() + timeout_sec @@ -329,9 +361,20 @@ def _fresh_default_like_cfg_template(): } +def _automation_child_config_path(tmp_path, automation_id: str = "test-automation") -> str: + automation_root = ( + tmp_path + / commons_constants.USER_FOLDER + / commons_constants.AUTOMATIONS_FOLDER + / automation_id + ) + automation_root.mkdir(parents=True, exist_ok=True) + return str(automation_root / commons_constants.CONFIG_FILE) + + class TestWriteUserRootConfigJson: def test_sets_profile_and_disables_browser_auto_open(self, tmp_path): - config_path = str(tmp_path / commons_constants.CONFIG_FILE) + config_path = _automation_child_config_path(tmp_path) profile_id = "dsl_profile_abc" with mock.patch.object( octobot_process_ops.json_util, @@ -349,7 +392,7 @@ def test_sets_profile_and_disables_browser_auto_open(self, tmp_path): assert written[commons_constants.CONFIG_EXCHANGES] == {} def test_seeds_exchanges_from_profile_data(self, tmp_path): - config_path = str(tmp_path / commons_constants.CONFIG_FILE) + config_path = _automation_child_config_path(tmp_path) profile_dict = { **_MINIMAL_PROFILE_DATA, "exchanges": [ @@ -379,7 +422,7 @@ def test_presets_encrypted_empty_credentials_when_default_config_exchange_has_no self, tmp_path ): """Mirrors packaged ``default_config.json`` rows that omit api-key/api-secret until setdefault.""" - config_path = str(tmp_path / commons_constants.CONFIG_FILE) + config_path = _automation_child_config_path(tmp_path) template = _fresh_default_like_cfg_template() template[commons_constants.CONFIG_EXCHANGES] = { "prefilled_exchange": { @@ -399,7 +442,7 @@ def test_presets_encrypted_empty_credentials_when_default_config_exchange_has_no assert exch[commons_constants.CONFIG_EXCHANGE_SECRET] == octobot_process_ops._DEFAULT_ENCRYPTED_VALUE def test_applies_exchange_auth_credentials(self, tmp_path): - config_path = str(tmp_path / commons_constants.CONFIG_FILE) + config_path = _automation_child_config_path(tmp_path) auth_list = [ exchange_auth_data_module.ExchangeAuthData( internal_name="binance_test", @@ -425,7 +468,7 @@ def test_applies_exchange_auth_credentials(self, tmp_path): assert exch[commons_constants.CONFIG_EXCHANGE_SANDBOXED] is True def test_profile_seed_then_auth_overlay(self, tmp_path): - config_path = str(tmp_path / commons_constants.CONFIG_FILE) + config_path = _automation_child_config_path(tmp_path) exchange_internal_name = "overlay_exchange" profile_dict = { **_MINIMAL_PROFILE_DATA, @@ -455,6 +498,106 @@ def test_profile_seed_then_auth_overlay(self, tmp_path): assert exch[commons_constants.CONFIG_EXCHANGE_KEY] == "overlay-key" assert exch[commons_constants.CONFIG_EXCHANGE_SECRET] == "overlay-secret" + def test_does_not_persist_sync_user_id_in_config(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) + with mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=lambda *_unused: _fresh_default_like_cfg_template(), + ): + octobot_process_ops._write_user_root_config_json( + config_path, + "sync-profile-id", + None, + None, + ) + written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) + assert written[commons_constants.CONFIG_PROFILE] == "sync-profile-id" + assert "sync_user_id" not in written + + def test_writes_master_overlay_config_when_provided(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) + master_profiles_path = str(tmp_path / "master" / commons_constants.PROFILES_FOLDER) + with mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=lambda *_unused: _fresh_default_like_cfg_template(), + ): + octobot_process_ops._write_user_root_config_json( + config_path, + octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + None, + None, + readonly_profiles_path=master_profiles_path, + ) + written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) + assert written[commons_constants.CONFIG_READONLY_PROFILES_PATH] == master_profiles_path + + +class TestAutomationChildPathGuards: + def test_write_user_root_config_json_rejects_master_config_path(self, tmp_path): + master_config_path = str( + tmp_path / commons_constants.USER_FOLDER / commons_constants.CONFIG_FILE + ) + with pytest.raises(commons_errors.DSLInterpreterError, match="user/automations"): + octobot_process_ops._write_user_root_config_json( + master_config_path, + octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + None, + None, + ) + + def test_write_user_root_config_json_rejects_automations_root_without_leaf(self, tmp_path): + automations_config_path = str( + tmp_path + / commons_constants.USER_FOLDER + / commons_constants.AUTOMATIONS_FOLDER + / commons_constants.CONFIG_FILE + ) + with pytest.raises(commons_errors.DSLInterpreterError, match="automation_id"): + octobot_process_ops._write_user_root_config_json( + automations_config_path, + octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + None, + None, + ) + + def test_assert_spawn_cmd_isolation_rejects_master_user_folder(self): + rel_user = commons_constants.USER_FOLDER + rel_log = os.path.join(octobot_node_constants.AUTOMATION_LOGS_FOLDER, "bot-1") + cmd = [ + sys.executable, + "start.py", + "--user-folder", + rel_user, + "--log-folder", + rel_log, + ] + with pytest.raises(commons_errors.DSLInterpreterError, match="user/automations"): + octobot_process_ops._assert_spawn_cmd_isolation(cmd, rel_user, rel_log) + + def test_assert_spawn_cmd_isolation_rejects_missing_log_folder_flag(self): + rel_user = os.path.join(commons_constants.USER_AUTOMATIONS_FOLDER, "bot-1") + rel_log = os.path.join(octobot_node_constants.AUTOMATION_LOGS_FOLDER, "bot-1") + cmd = [sys.executable, "start.py", "--user-folder", rel_user] + with pytest.raises(commons_errors.DSLInterpreterError, match="--log-folder"): + octobot_process_ops._assert_spawn_cmd_isolation(cmd, rel_user, rel_log) + + def test_assert_spawn_cmd_isolation_accepts_valid_cmd(self): + rel_user = os.path.join(commons_constants.USER_AUTOMATIONS_FOLDER, "bot-1") + rel_log = os.path.join(octobot_node_constants.AUTOMATION_LOGS_FOLDER, "bot-1") + cmd = [ + sys.executable, + "start.py", + "--user-folder", + rel_user, + "--log-folder", + rel_log, + "--dump-state", + "/tmp/state.json", + ] + octobot_process_ops._assert_spawn_cmd_isolation(cmd, rel_user, rel_log) + class TestEnsureUserProfileAndLayout: async def test_marked_prepared_is_skipped(self, tmp_path): @@ -488,70 +631,168 @@ def test_declares_optional_profile_data_parameter(self): assert profile_parameter.default is None -class TestCopyReadOnlyProfilesToUserRoot: - async def test_copies_read_only_profiles_and_skips_editable(self, tmp_path): - _seed_executor_non_trading_profile(tmp_path) - readonly_profile_id = "readonly_strategy" - editable_profile_id = "editable_strategy" - _seed_executor_profile(tmp_path, readonly_profile_id, read_only=True) - _seed_executor_profile(tmp_path, editable_profile_id, read_only=False) - user_root = tmp_path / "child_user_root" - user_root.mkdir() - await octobot_process_ops._copy_read_only_profiles_to_user_root( - str(tmp_path), - str(user_root), - active_profile_id=octobot_process_ops.DEFAULT_DSL_PROFILE_ID, - ) - profiles_root = user_root / commons_constants.PROFILES_FOLDER - readonly_profile_json = ( - profiles_root / readonly_profile_id / commons_constants.PROFILE_CONFIG_FILE - ) - editable_profile_json = ( - profiles_root / editable_profile_id / commons_constants.PROFILE_CONFIG_FILE +class TestEnsureOctobotProcessOperatorSyncProfileIdOptional: + def test_declares_optional_sync_profile_id_parameter(self): + params = EnsureOctobotProcessOperator.get_parameters() + sync_profile_parameter = next( + (parameter for parameter in params if parameter.name == "sync_profile_id"), + None, ) - non_trading_profile_json = ( - profiles_root - / octobot_process_ops.DEFAULT_DSL_PROFILE_ID - / commons_constants.PROFILE_CONFIG_FILE + assert sync_profile_parameter is not None + assert sync_profile_parameter.required is False + assert sync_profile_parameter.default is None + + def test_declares_optional_user_id_parameter(self): + params = EnsureOctobotProcessOperator.get_parameters() + user_id_parameter = next( + (parameter for parameter in params if parameter.name == "user_id"), + None, ) - assert readonly_profile_json.is_file() - assert not editable_profile_json.exists() - assert not non_trading_profile_json.exists() + assert user_id_parameter is not None + assert user_id_parameter.required is False + assert user_id_parameter.default is None - async def test_skips_active_profile_id(self, tmp_path): - _seed_executor_profile( - tmp_path, - octobot_process_ops.DEFAULT_DSL_PROFILE_ID, - read_only=True, + +class TestAssertSyncStrategyExists: + def test_accepts_existing_strategy(self): + stored_strategy = _generic_process_sync_strategy("strategy-1") + strategy_provider_mock = mock.Mock() + strategy_provider_mock.get_item.return_value = stored_strategy + with mock.patch.object( + octobot_process_ops.collection_providers, + "StrategyProvider", + ) as strategy_provider_class: + strategy_provider_class.instance.return_value = strategy_provider_mock + result = octobot_process_ops._assert_sync_strategy_exists( + "wallet-user", + "strategy-1", + ) + assert result is stored_strategy + strategy_provider_mock.get_item.assert_called_once_with("wallet-user", "strategy-1") + + def test_raises_when_strategy_not_found(self): + strategy_provider_mock = mock.Mock() + strategy_provider_mock.get_item.side_effect = collection_errors.ItemNotFoundError( + "missing" ) - user_root = tmp_path / "child_user_root" - user_root.mkdir() - destination_profile_path = ( - user_root - / commons_constants.PROFILES_FOLDER - / octobot_process_ops.DEFAULT_DSL_PROFILE_ID + with mock.patch.object( + octobot_process_ops.collection_providers, + "StrategyProvider", + ) as strategy_provider_class: + strategy_provider_class.instance.return_value = strategy_provider_mock + with pytest.raises(commons_errors.DSLInterpreterError, match="not found"): + octobot_process_ops._assert_sync_strategy_exists( + "wallet-user", + "strategy-1", + ) + + +class TestEnsureUserProfileAndLayoutSyncProfileId: + async def test_raises_when_user_id_missing_with_sync_profile_id(self, tmp_path): + with pytest.raises(commons_errors.DSLInterpreterError, match="requires user_id"): + await octobot_process_ops.ensure_user_profile_and_layout( + "sync_user_folder", + str(tmp_path), + None, + None, + None, + sync_profile_id="sync-strategy-1", + ) + + async def test_bare_generic_process_strategy_uses_non_trading_profile(self, tmp_path): + sync_profile_id = "sync-strategy-1" + executor_sync_user_id = "wallet-user" + bare_strategy = _generic_process_sync_strategy(sync_profile_id) + with mock.patch.object( + octobot_process_ops, + "_assert_sync_strategy_exists", + return_value=bare_strategy, + ), mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=lambda *_unused: _fresh_default_like_cfg_template(), + ): + result = await octobot_process_ops.ensure_user_profile_and_layout( + "sync_user_folder", + str(tmp_path), + None, + None, + None, + sync_profile_id=sync_profile_id, + user_id=executor_sync_user_id, + ) + user_root = pathlib.Path(result["user_root"]) + assert result["profile_id"] == octobot_process_ops.DEFAULT_DSL_PROFILE_ID + assert not (user_root / commons_constants.PROFILES_FOLDER / sync_profile_id).exists() + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + assert root_cfg[commons_constants.CONFIG_PROFILE] == octobot_process_ops.DEFAULT_DSL_PROFILE_ID + assert "sync_user_id" not in root_cfg + + async def test_strategy_with_profile_data_uses_sync_profile_id(self, tmp_path): + sync_profile_id = "sync-strategy-1" + executor_sync_user_id = "wallet-user" + strategy_with_profile_data = _generic_process_sync_strategy( + sync_profile_id, + profile_data={"profile_details": {"id": sync_profile_id}}, ) - destination_profile_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copytree( - tmp_path.joinpath( - commons_constants.USER_FOLDER, - commons_constants.PROFILES_FOLDER, - octobot_process_ops.DEFAULT_DSL_PROFILE_ID, - ), - destination_profile_path, + with mock.patch.object( + octobot_process_ops, + "_assert_sync_strategy_exists", + return_value=strategy_with_profile_data, + ), mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=lambda *_unused: _fresh_default_like_cfg_template(), + ): + result = await octobot_process_ops.ensure_user_profile_and_layout( + "sync_user_folder", + str(tmp_path), + None, + None, + None, + sync_profile_id=sync_profile_id, + user_id=executor_sync_user_id, + ) + user_root = pathlib.Path(result["user_root"]) + assert result["profile_id"] == sync_profile_id + assert not (user_root / commons_constants.PROFILES_FOLDER / sync_profile_id).exists() + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + assert root_cfg[commons_constants.CONFIG_PROFILE] == sync_profile_id + assert "sync_user_id" not in root_cfg + assert root_cfg[commons_constants.CONFIG_READONLY_PROFILES_PATH] == ( + octobot_process_ops._executor_profiles_directory(str(tmp_path)) ) - profile_json_path = destination_profile_path / commons_constants.PROFILE_CONFIG_FILE - original_mtime = profile_json_path.stat().st_mtime - await octobot_process_ops._copy_read_only_profiles_to_user_root( + + +class TestEnsureUserProfileAndLayoutProfileDataPriority: + async def test_profile_data_wins_over_sync_profile_id(self, tmp_path): + sync_profile_id = "ignored-sync-profile" + result = await octobot_process_ops.ensure_user_profile_and_layout( + "priority_user_folder", str(tmp_path), - str(user_root), - active_profile_id=octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + _MINIMAL_PROFILE_DATA, + None, + None, + sync_profile_id=sync_profile_id, ) - assert profile_json_path.stat().st_mtime == original_mtime + user_root = pathlib.Path(result["user_root"]) + profile_id = result["profile_id"] + assert profile_id + assert profile_id != sync_profile_id + assert ( + user_root + / commons_constants.PROFILES_FOLDER + / profile_id + / commons_constants.PROFILE_CONFIG_FILE + ).is_file() + assert not (user_root / commons_constants.PROFILES_FOLDER / sync_profile_id).exists() + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + assert root_cfg[commons_constants.CONFIG_PROFILE] == profile_id + assert "sync_user_id" not in root_cfg class TestEnsureUserProfileAndLayoutDefaultProfile: - async def test_copies_non_trading_profile_and_writes_default_config(self, tmp_path): + async def test_pins_non_trading_profile_via_overlay_and_writes_default_config(self, tmp_path): _seed_executor_non_trading_profile(tmp_path) user_leaf = "default_profile_layout_user" result = await octobot_process_ops.ensure_user_profile_and_layout( @@ -570,10 +811,13 @@ async def test_copies_non_trading_profile_and_writes_default_config(self, tmp_pa / octobot_process_ops.DEFAULT_DSL_PROFILE_ID / commons_constants.PROFILE_CONFIG_FILE ) - assert profile_json_path.is_file() + assert not profile_json_path.exists() root_config_path = user_root / commons_constants.CONFIG_FILE root_cfg = json.loads(root_config_path.read_text(encoding="utf-8")) assert root_cfg[commons_constants.CONFIG_PROFILE] == octobot_process_ops.DEFAULT_DSL_PROFILE_ID + assert root_cfg[commons_constants.CONFIG_READONLY_PROFILES_PATH] == ( + octobot_process_ops._executor_profiles_directory(str(tmp_path)) + ) assert ( root_cfg[services_constants.CONFIG_CATEGORY_SERVICES][services_constants.CONFIG_WEB][ services_constants.CONFIG_AUTO_OPEN_IN_WEB_BROWSER @@ -582,7 +826,28 @@ async def test_copies_non_trading_profile_and_writes_default_config(self, tmp_pa ) assert root_cfg[commons_constants.CONFIG_ACCEPTED_TERMS] is True - async def test_copies_read_only_profiles_on_default_layout(self, tmp_path): + async def test_resolves_non_trading_profile_from_master_overlay(self, tmp_path): + _seed_executor_non_trading_profile(tmp_path) + result = await octobot_process_ops.ensure_user_profile_and_layout( + "default_overlay_resolution_user", + str(tmp_path), + None, + None, + None, + ) + user_root = pathlib.Path(result["user_root"]) + child_profiles_path = user_root / commons_constants.PROFILES_FOLDER + child_profiles_path.mkdir(parents=True, exist_ok=True) + profile_storage = profile_storage_module.ProfileStorage(str(child_profiles_path), None) + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + profile_storage.configure_readonly_profiles_path( + root_cfg[commons_constants.CONFIG_READONLY_PROFILES_PATH], + ) + resolved_profile = profile_storage.get_profile(octobot_process_ops.DEFAULT_DSL_PROFILE_ID) + assert resolved_profile is not None + assert resolved_profile.profile_id == octobot_process_ops.DEFAULT_DSL_PROFILE_ID + + async def test_default_layout_does_not_copy_read_only_profiles(self, tmp_path): _seed_executor_non_trading_profile(tmp_path) readonly_profile_id = "readonly_strategy" _seed_executor_profile(tmp_path, readonly_profile_id, read_only=True) @@ -596,14 +861,29 @@ async def test_copies_read_only_profiles_on_default_layout(self, tmp_path): ) user_root = pathlib.Path(result["user_root"]) profiles_root = user_root / commons_constants.PROFILES_FOLDER - assert ( + assert not ( profiles_root / octobot_process_ops.DEFAULT_DSL_PROFILE_ID / commons_constants.PROFILE_CONFIG_FILE - ).is_file() - assert ( + ).exists() + assert not ( profiles_root / readonly_profile_id / commons_constants.PROFILE_CONFIG_FILE - ).is_file() + ).exists() + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + assert commons_constants.CONFIG_READONLY_PROFILES_PATH in root_cfg + + async def test_default_layout_omits_sync_user_id_without_user_id_arg(self, tmp_path): + _seed_executor_non_trading_profile(tmp_path) + result = await octobot_process_ops.ensure_user_profile_and_layout( + "default_layout_without_sync_user", + str(tmp_path), + None, + None, + None, + ) + user_root = pathlib.Path(result["user_root"]) + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + assert "sync_user_id" not in root_cfg async def test_applies_exchange_auth_without_profile_data(self, tmp_path): _seed_executor_non_trading_profile(tmp_path) @@ -885,6 +1165,129 @@ def test_skips_port_occupied_on_host(self): assert web_port != occupied_port +class TestChildListenPortReservation: + @pytest.fixture(autouse=True) + def _clear_reserved_ports(self): + octobot_process_ops._child_listen_ports_reserved.clear() + yield + octobot_process_ops._child_listen_ports_reserved.clear() + + def test_parallel_allocations_get_distinct_web_ports(self): + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + first_web_port, first_node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 20000, 30000, "automation-a", max_offset=100 + ) + second_web_port, second_node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 20000, 30000, "automation-b", max_offset=100 + ) + try: + assert first_web_port != second_web_port + assert first_node_port != second_node_port + finally: + octobot_process_ops._release_child_listen_ports(first_web_port, first_node_port, "automation-a") + octobot_process_ops._release_child_listen_ports(second_web_port, second_node_port, "automation-b") + + def test_release_allows_reuse_when_no_listener(self): + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + web_port, node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 20000, 30000, "automation-a", max_offset=100 + ) + octobot_process_ops._release_child_listen_ports(web_port, node_port, "automation-a") + reused_web_port, reused_node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 20000, 30000, "automation-a", max_offset=100 + ) + try: + assert reused_web_port == web_port + assert reused_node_port == node_port + finally: + octobot_process_ops._release_child_listen_ports(reused_web_port, reused_node_port, "automation-a") + + def test_second_allocation_skips_reserved_port(self): + octobot_process_ops._reserve_child_listen_ports(5002, 5999, "automation-a") + try: + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + web_port, _node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 5002, 6000, "automation-b", max_offset=10 + ) + assert web_port != 5002 + finally: + octobot_process_ops._release_child_listen_ports(5002, 5999, "automation-a") + + def test_stale_prior_recall_release_does_not_free_other_automation_ports(self): + user_folder_a = "03e38366-99ea-4c47-84d5-4329c7aa00df" + user_folder_b = "7ae4e140-1dc0-4d7c-a4cf-38c121a80f72" + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + web_port_a, node_port_a = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 5002, 6000, user_folder_a, max_offset=10 + ) + assert web_port_a == 5002 + stale_recall_b = octobot_process_ops.EnsureOctobotProcessState( + http_base_url="http://127.0.0.1:5002", + web_port=5002, + node_port=node_port_a, + user_root="/x/b", + user_folder=user_folder_b, + log_folder="/x/logs/b", + profile_id=None, + pid=0, + executor_id=TEST_EXECUTOR_ID, + ) + octobot_process_ops._release_recall_state_listen_ports(stale_recall_b) + assert octobot_process_ops._child_listen_ports_reserved.get(5002) == user_folder_a + try: + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + web_port_b, node_port_b = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 5002, 6000, user_folder_b, max_offset=10 + ) + assert web_port_b != 5002 + finally: + octobot_process_ops._release_child_listen_ports(web_port_a, node_port_a, user_folder_a) + octobot_process_ops._release_child_listen_ports(web_port_b, node_port_b, user_folder_b) + + def test_same_automation_recall_release_frees_ports_for_reuse(self): + user_folder = "automation-a" + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + web_port, node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 5002, 6000, user_folder, max_offset=10 + ) + recall_state = octobot_process_ops.EnsureOctobotProcessState( + http_base_url=f"http://127.0.0.1:{web_port}", + web_port=web_port, + node_port=node_port, + user_root="/x/a", + user_folder=user_folder, + log_folder="/x/logs/a", + profile_id=None, + pid=0, + executor_id=TEST_EXECUTOR_ID, + ) + octobot_process_ops._release_recall_state_listen_ports(recall_state) + assert web_port not in octobot_process_ops._child_listen_ports_reserved + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + reused_web_port, reused_node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 5002, 6000, user_folder, max_offset=10 + ) + try: + assert reused_web_port == web_port + assert reused_node_port == node_port + finally: + octobot_process_ops._release_child_listen_ports(reused_web_port, reused_node_port, user_folder) + + class TestEnsureOctobotProcessOperatorExchangeAuthData: def test_declares_optional_exchange_auth_parameter(self): params = EnsureOctobotProcessOperator.get_parameters() @@ -919,6 +1322,7 @@ async def test_pre_compute_passes_dict_exchange_auth_into_ensure_layout(self, tm start_script.write_text("#", encoding="utf-8") operator_instance = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, exchange_auth_data=exchange_auth_dicts, last_execution_result=None, @@ -955,12 +1359,61 @@ async def test_pre_compute_passes_dict_exchange_auth_into_ensure_layout(self, tm assert parsed_exchange_auth[0].exchange_type == commons_constants.CONFIG_EXCHANGE_SPOT +class TestEnsureChildEnviron: + def test_sets_process_bot_sync_user_id_env(self, tmp_path): + working_directory = str(tmp_path) + child_env = octobot_process_ops._ensure_child_environ( + 20050, + 30050, + "127.0.0.1", + "wallet-user", + working_directory, + ) + assert child_env[octobot_constants.ENV_PROCESS_BOT_SYNC_USER_ID] == "wallet-user" + assert child_env[commons_constants.ENV_OCTOBOT_SYNC_DATA_ROOT] == os.path.normpath( + os.path.join(working_directory, commons_constants.USER_FOLDER) + ) + + class TestEnsureOctobotProcessOperatorPrecompute: + async def test_raises_when_user_id_missing_at_spawn(self, tmp_path): + start_script = tmp_path / "start.py" + start_script.write_text("#", encoding="utf-8") + op = EnsureOctobotProcessOperator( + user_folder="ub", + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=None, + ) + with mock.patch.object( + octobot_process_ops.os, + "getcwd", + return_value=str(tmp_path), + ), mock.patch.object( + octobot_process_ops, + "ensure_user_profile_and_layout", + new=mock.AsyncMock( + return_value={ + "user_root": str( + tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" + ), + "profile_id": "x", + "already_prepared": True, + } + ), + ), mock.patch.object( + octobot_process_ops, + "_listen_port_pair_with_shared_scan_offset", + return_value=(20050, 30050), + ): + with pytest.raises(commons_errors.DSLInterpreterError, match="requires user_id"): + await op.pre_compute() + async def test_returns_recallable_when_process_bot_state_not_live(self, tmp_path): start_script = tmp_path / "start.py" start_script.write_text("#", encoding="utf-8") op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=None, ) @@ -1007,6 +1460,7 @@ async def test_returns_recallable_with_init_state_ok_after_first_spawn(self, tmp start_script.write_text("#", encoding="utf-8") op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=None, ) @@ -1065,6 +1519,7 @@ async def test_returns_recallable_with_init_state_ok_on_recall_path(self, tmp_pa start_script.write_text("#", encoding="utf-8") op1 = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=None, ) @@ -1105,6 +1560,7 @@ async def test_returns_recallable_with_init_state_ok_on_recall_path(self, tmp_pa anchor = first_le["started_waiting_at"] op2 = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=first_value, ) @@ -1160,6 +1616,7 @@ async def test_init_timeout_kills_and_raises_dsl_error(self, tmp_path): } op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1218,6 +1675,7 @@ async def test_does_not_apply_init_timeout_after_init_state_ok(self, tmp_path): } op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1252,6 +1710,7 @@ async def test_waiting_time_uses_parameter_for_recall_emissions(self, tmp_path): start_script.write_text("#", encoding="utf-8") op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=None, waiting_time=7.0, @@ -1305,7 +1764,10 @@ async def test_run_octobot_process_via_dsl(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) (tmp_path / "start.py").write_text("#", encoding="utf-8") user_folder = "integration_dsl_bot" - expression = f"run_octobot_process({user_folder!r}, {repr(_MINIMAL_PROFILE_DATA_DSL_LITERAL)})" + expression = ( + f"run_octobot_process({user_folder!r}, {repr(_MINIMAL_PROFILE_DATA_DSL_LITERAL)}, " + f"user_id={_PROCESS_TEST_USER_ID!r})" + ) # Contextual operator is excluded from get_all_operators(); append it explicitly. interpreter = dsl_interpreter.Interpreter( dsl_interpreter.get_all_operators() @@ -1380,6 +1842,7 @@ async def test_run_octobot_process_via_dsl(self, tmp_path, monkeypatch): assert child_env[services_constants.ENV_WEB_ADDRESS] == "127.0.0.1" assert child_env[services_constants.ENV_NODE_API_PORT] == str(last_execution["node_port"]) assert child_env[services_constants.ENV_NODE_API_ADDRESS] == "127.0.0.1" + assert child_env[octobot_constants.ENV_PROCESS_BOT_SYNC_USER_ID] == _PROCESS_TEST_USER_ID assert spawn_kwargs.get("hide_console_window") is True finally: # Redundant with pytest’s tmp_path teardown; makes intent obvious if the test is copied elsewhere. @@ -1414,7 +1877,7 @@ async def test_run_octobot_process_via_dsl_writes_exchange_auth_into_user_config ] expression = ( f"run_octobot_process({user_folder!r}, {repr(_MINIMAL_PROFILE_DATA_DSL_LITERAL)}, " - f"{repr(exchange_auth_list)})" + f"exchange_auth_data={repr(exchange_auth_list)}, user_id={_PROCESS_TEST_USER_ID!r})" ) interpreter = dsl_interpreter.Interpreter( dsl_interpreter.get_all_operators() @@ -1472,7 +1935,8 @@ async def test_run_octobot_process_via_dsl_without_profile_data_accepts_exchange } ] expression = ( - f"run_octobot_process({user_folder!r}, exchange_auth_data={repr(exchange_auth_list)}, " + f"run_octobot_process({user_folder!r}, user_id={_PROCESS_TEST_USER_ID!r}, " + f"exchange_auth_data={repr(exchange_auth_list)}, " f"waiting_time={_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SEC}, ping_timeout=30.0)" ) interpreter = dsl_interpreter.Interpreter( @@ -1502,13 +1966,16 @@ async def test_run_octobot_process_via_dsl_without_profile_data_accepts_exchange assert root_config_path.is_file() written_root_cfg = json.loads(root_config_path.read_text(encoding="utf-8")) assert written_root_cfg[commons_constants.CONFIG_PROFILE] == octobot_process_ops.DEFAULT_DSL_PROFILE_ID + assert written_root_cfg[commons_constants.CONFIG_READONLY_PROFILES_PATH] == ( + octobot_process_ops._executor_profiles_directory(str(tmp_path)) + ) profile_json_path = ( user_data_root / commons_constants.PROFILES_FOLDER / octobot_process_ops.DEFAULT_DSL_PROFILE_ID / commons_constants.PROFILE_CONFIG_FILE ) - assert profile_json_path.is_file() + assert not profile_json_path.exists() exchange_cfg = written_root_cfg[commons_constants.CONFIG_EXCHANGES][exchange_internal_name] assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_KEY] == "no-profile-key" assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_SECRET] == "no-profile-secret" @@ -1575,15 +2042,19 @@ async def _run_default_config_lifecycle( else: assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_SANDBOXED] is exchange_auth_list[0]["sandboxed"] assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_TYPE] == exchange_auth_list[0]["exchange_type"] + assert root_cfg[commons_constants.CONFIG_READONLY_PROFILES_PATH] == ( + octobot_process_ops._executor_profiles_directory(project_root) + ) profile_json_path = ( user_root / commons_constants.PROFILES_FOLDER / octobot_process_ops.DEFAULT_DSL_PROFILE_ID / commons_constants.PROFILE_CONFIG_FILE ) - assert profile_json_path.is_file() + assert not profile_json_path.exists() stop_expression = ( - f"run_octobot_process({user_folder!r}, exchange_auth_data={repr(exchange_auth_list)}, " + f"run_octobot_process({user_folder!r}, user_id={_PROCESS_TEST_USER_ID!r}, " + f"exchange_auth_data={repr(exchange_auth_list)}, " f"waiting_time={_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SEC}, ping_timeout=30.0, " f"last_execution_result={repr(_re_calling_ensure_value(inner))})" ) @@ -1665,6 +2136,7 @@ async def test_execution_stop_dead_child_is_already_stopped(self): }) op = operator_under_test( user_folder="u1", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1690,6 +2162,7 @@ async def test_execution_stop_short_circuits_without_sigterm_when_not_running(se }) op = operator_under_test( user_folder="u1", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1722,6 +2195,7 @@ async def test_execution_stop_os_kill_failure_raises(self): }) op = operator_under_test( user_folder="u1", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1799,6 +2273,7 @@ async def test_update_config_triggers_respawn_and_recallable_result(self, tmp_pa node_port=5002, user_root=str(user_automation), user_folder="nested/upd_bot", + user_id=_PROCESS_TEST_USER_ID, log_folder=str(log_dir), profile_id="p1", pid=4242, @@ -1819,6 +2294,7 @@ async def test_update_config_triggers_respawn_and_recallable_result(self, tmp_pa }) op = operator_under_test( user_folder="nested/upd_bot", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1880,6 +2356,7 @@ async def test_adopts_pid_from_live_state_without_spawn(self, tmp_path): inner = _healthy_recall_inner(pid=10002, tmp_path=tmp_path) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1912,6 +2389,7 @@ async def test_recall_without_spawn_during_init(self, tmp_path): inner["started_waiting_at"] = octobot_process_ops.time.time() op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1941,6 +2419,7 @@ async def test_recall_when_pid_running_without_state_file(self, tmp_path): inner = _healthy_recall_inner(pid=10002, tmp_path=tmp_path) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1974,6 +2453,7 @@ async def test_recall_during_restart_grace(self, tmp_path): ) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ping_timeout=120.0, @@ -2006,6 +2486,7 @@ async def test_respawns_when_grace_expired(self, tmp_path): stale_state = _stale_process_bot_state_for_grace(age_seconds=200.0, metadata_pid=10002) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ping_timeout=120.0, @@ -2051,6 +2532,7 @@ async def test_respawns_when_no_state_file_and_pid_dead(self, tmp_path): inner = _healthy_recall_inner(pid=10002, tmp_path=tmp_path) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -2102,6 +2584,7 @@ async def test_stop_signals_adopted_pid(self): }) op = operator_under_test( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -2142,6 +2625,7 @@ async def test_marker_mismatch_forces_first_spawn(self, tmp_path): None, TEST_EXECUTOR_ID )[0]( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -2191,6 +2675,7 @@ async def test_marker_mismatch_but_metadata_pid_running_recalls(self, tmp_path): None, TEST_EXECUTOR_ID )[0]( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -2230,6 +2715,7 @@ async def test_recall_during_grace_without_spawn(self, tmp_path): ) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ping_timeout=120.0, @@ -2264,6 +2750,7 @@ async def test_recall_during_grace_when_marker_matches(self, tmp_path): ) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ping_timeout=120.0, @@ -2296,6 +2783,7 @@ async def test_first_spawn_when_grace_expired(self, tmp_path): stale_state = _stale_process_bot_state_for_grace(age_seconds=200.0, metadata_pid=10002) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ping_timeout=120.0, @@ -2341,6 +2829,7 @@ async def test_first_spawn_emits_executor_id(self, tmp_path, monkeypatch): (tmp_path / "start.py").write_text("#", encoding="utf-8") op = EnsureOctobotProcessOperator( user_folder="emit_master_bot", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, ) with mock.patch.object( @@ -2383,6 +2872,7 @@ async def test_missing_executor_id_falls_through_to_first_spawn(self, tmp_path): del inner["executor_id"] op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/node_api.py b/packages/tentacles/Services/Interfaces/node_api_interface/node_api.py index a9eb447de0..8075d2c4fc 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/node_api.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/node_api.py @@ -13,6 +13,8 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +import asyncio +import threading from contextlib import asynccontextmanager import uvicorn @@ -25,6 +27,7 @@ import octobot.community.authentication as community_auth import octobot_services.interfaces as services_interfaces import octobot_node.config as node_config +import octobot_node.constants as node_constants import octobot_node.scheduler as scheduler # noqa: F401 import octobot_sync.server as sync_server @@ -72,6 +75,7 @@ def __init__(self, config): self.host = None self.port = None self.node_api_service = None + self._serve_finished: threading.Event | None = None async def _inner_start(self) -> bool: return self.threaded_start() @@ -110,12 +114,31 @@ async def _async_run(self) -> bool: ) config = uvicorn.Config(self.app, host=host, port=port, log_level="info") self.server = uvicorn.Server(config) - await self.server.serve() + self._serve_finished = threading.Event() + try: + await self.server.serve() + finally: + if self._serve_finished is not None: + self._serve_finished.set() return True async def stop(self): - if self.server is not None: - self.server.should_exit = True + if self.server is None: + return + self.server.should_exit = True + serve_finished = self._serve_finished + if serve_finished is None: + return + try: + await asyncio.wait_for( + asyncio.to_thread(serve_finished.wait), + timeout=node_constants.NODE_API_STOP_TIMEOUT_SECONDS, + ) + except asyncio.TimeoutError: + self.logger.warning( + "Timed out waiting for Node API server to stop after %ss", + node_constants.NODE_API_STOP_TIMEOUT_SECONDS, + ) @classmethod def create_app(cls) -> FastAPI: diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_node_api_stop.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_node_api_stop.py new file mode 100644 index 0000000000..c05ac248ba --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_node_api_stop.py @@ -0,0 +1,51 @@ +# Drakkar-Software OctoBot-Interfaces +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import asyncio +import threading + +import mock +import pytest + +import tentacles.Services.Interfaces.node_api_interface.node_api as node_api_module + + +pytestmark = pytest.mark.asyncio + + +class TestNodeApiInterfaceStop: + async def test_stop_sets_should_exit_and_waits_for_serve_finished(self): + interface = node_api_module.NodeApiInterface({}) + interface.logger = mock.Mock() + interface.server = mock.Mock() + interface.server.should_exit = False + serve_finished = threading.Event() + serve_finished.set() + interface._serve_finished = serve_finished + + await interface.stop() + + assert interface.server.should_exit is True + + async def test_stop_logs_warning_when_serve_does_not_finish_in_time(self): + interface = node_api_module.NodeApiInterface({}) + interface.logger = mock.Mock() + interface.server = mock.Mock() + interface._serve_finished = threading.Event() + + with mock.patch("asyncio.wait_for", side_effect=asyncio.TimeoutError): + await interface.stop() + + interface.logger.warning.assert_called_once() diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx index aad772b86c..6ecfc2e5a8 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx @@ -21,6 +21,7 @@ import { buildAccountEditUserActionJson, buildAutomationCreateUserActionJsonForAccount, buildAutomationCreateUserActionJsonForStrategy, + buildAutomationRestartUserActionJson, buildAutomationSignalUserActionJson, buildAutomationStopUserActionJson, buildExchangeConfigEditUserActionJson, @@ -166,6 +167,12 @@ function DebugTabsPanelComponent({ jsonText: buildAutomationStopUserActionJson(automation.id), }) } + onRestart={(automation) => + onOpenExecuteAction({ + actionType: "automation_restart", + jsonText: buildAutomationRestartUserActionJson(automation.id), + }) + } /> diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AutomationsTable.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AutomationsTable.tsx index a392446cf9..aeae90dd1f 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AutomationsTable.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AutomationsTable.tsx @@ -1,4 +1,4 @@ -import { Eye, X, Zap } from "lucide-react" +import { Eye, RotateCcw, X, Zap } from "lucide-react" import { useMemo, useState } from "react" import type { AccountTradingWithAccountId, AutomationState } from "@/client" @@ -32,6 +32,7 @@ import { formatActionProgress, getAutomationErrorTooltipLines, getAutomationUpdatedAt, + isRestartableAutomation, isRunningAutomation, } from "@/lib/debug/automation" import { @@ -63,6 +64,7 @@ type AutomationsTableProps = { accountTradings: AccountTradingWithAccountId[] onSuccess?: () => void onStop?: (automation: AutomationState) => void + onRestart?: (automation: AutomationState) => void onSignal?: (automation: AutomationState) => void readOnly?: boolean selectionMode?: boolean @@ -77,6 +79,7 @@ export function AutomationsTable({ accountTradings, onSuccess, onStop, + onRestart, onSignal, readOnly = false, selectionMode = false, @@ -240,9 +243,9 @@ export function AutomationsTable({ const canSignal = readOnly ? Boolean(onSignal) : isRunningAutomation(row) - const canStop = readOnly - ? Boolean(onStop) - : isRunningAutomation(row) + const canStop = Boolean(onStop) && isRunningAutomation(row) + const canRestart = + Boolean(onRestart) && isRestartableAutomation(row) const signalButton = ( - ) : ( - - - - - - - - {readOnly - ? "Stop action unavailable" - : "Only running automations can be stopped"} - - - )} + ) : null} + {canRestart ? ( + + ) : null} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/automation.test.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/automation.test.ts index ab29ca19c9..df7c75771d 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/automation.test.ts +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/automation.test.ts @@ -11,6 +11,7 @@ import { getNextPendingAction, getRunningAction, isActionExecuted, + isRestartableAutomation, isRunningAutomation, signalTypeRequiresPayload, validateAutomationCanReceiveSignal, @@ -46,6 +47,26 @@ describe("isRunningAutomation", () => { }) }) +describe("isRestartableAutomation", () => { + it("returns true for completed and failed statuses", () => { + expect(isRestartableAutomation(makeAutomation({ status: "completed" }))).toBe( + true, + ) + expect(isRestartableAutomation(makeAutomation({ status: "failed" }))).toBe( + true, + ) + }) + + it("returns false for running and pending statuses", () => { + expect(isRestartableAutomation(makeAutomation({ status: "running" }))).toBe( + false, + ) + expect(isRestartableAutomation(makeAutomation({ status: "pending" }))).toBe( + false, + ) + }) +}) + describe("getAutomationActions", () => { it("returns an empty array when actions are missing", () => { expect(getAutomationActions(makeAutomation())).toEqual([]) diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/user-action-templates.test.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/user-action-templates.test.ts index 79d083bd7d..b2b78eada3 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/user-action-templates.test.ts +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/user-action-templates.test.ts @@ -5,6 +5,7 @@ import { buildAccountEditUserActionJson, buildAutomationCreateUserActionJsonForAccount, buildAutomationCreateUserActionJsonForStrategy, + buildAutomationRestartUserActionJson, buildAutomationSignalUserActionJson, buildAutomationStopUserActionJson, buildExchangeConfigEditUserActionJson, @@ -36,6 +37,15 @@ describe("buildUserActionTemplate", () => { }) }) + it("builds an automation restart template", () => { + const action = buildUserActionTemplate("automation_restart") + expect(action.id).toContain("automation_restart") + expect(action.configuration).toMatchObject({ + action_type: "automation_restart", + id: "", + }) + }) + it("builds an automation create template with a random configuration id", () => { const action = buildUserActionTemplate("automation_create") expect(action.configuration).toMatchObject({ @@ -340,6 +350,16 @@ describe("buildAutomationStopUserActionJson", () => { }) }) +describe("buildAutomationRestartUserActionJson", () => { + it("targets the automation id", () => { + const json = JSON.parse(buildAutomationRestartUserActionJson("auto-1")) + expect(json.configuration).toEqual({ + action_type: "automation_restart", + id: "auto-1", + }) + }) +}) + describe("buildAutomationSignalUserActionJson", () => { it("includes the selected signal type", () => { const json = JSON.parse( diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/automation.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/automation.ts index 5ba556143d..535ed078bc 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/automation.ts +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/automation.ts @@ -11,6 +11,12 @@ export function isRunningAutomation(automation: AutomationState): boolean { return automation.status === "running" } +export function isRestartableAutomation(automation: AutomationState): boolean { + return ( + automation.status === "completed" || automation.status === "failed" + ) +} + export function getAutomationErrorTooltipLines( automation: AutomationState, ): string[] { diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/user-action-templates.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/user-action-templates.ts index a2d5339e4b..9850b931ab 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/user-action-templates.ts +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/user-action-templates.ts @@ -28,6 +28,7 @@ import type { MarketMakingConfiguration, MarketMakingSymbolConfiguration, RefreshAccountsConfiguration, + RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, Strategy, @@ -62,6 +63,7 @@ export const USER_ACTION_TEMPLATE_OPTIONS: { { value: "automation_create", label: "Automation create" }, { value: "automation_edit", label: "Automation edit" }, { value: "automation_stop", label: "Automation stop" }, + { value: "automation_restart", label: "Automation restart" }, { value: "automation_signal", label: "Automation signal" }, { value: "account_create", label: "Account create" }, { value: "account_edit", label: "Account edit" }, @@ -117,6 +119,7 @@ type DebugUserActionConfiguration = | CreateAutomationConfiguration | EditAutomationConfiguration | StopAutomationConfiguration + | RestartAutomationConfiguration | SignalAutomationConfiguration | CreateStrategyConfiguration | EditStrategyConfiguration @@ -615,6 +618,11 @@ export function buildUserActionTemplate( action_type: actionType, id: "", } satisfies StopAutomationConfiguration) + case "automation_restart": + return userAction(id, { + action_type: actionType, + id: "", + } satisfies RestartAutomationConfiguration) case "automation_signal": return userAction(id, { action_type: actionType, @@ -755,6 +763,17 @@ export function buildAutomationStopUserActionJson( ) } +export function buildAutomationRestartUserActionJson( + automationId: string, +): string { + return userActionJson( + userAction(`ua-restart-${automationId}`, { + action_type: "automation_restart", + id: automationId, + } satisfies RestartAutomationConfiguration), + ) +} + export function buildAutomationSignalUserActionJson( automationId: string, signalType: AutomationSignalType = "forced_trigger", diff --git a/packages/tentacles/Services/Interfaces/web_interface/controllers/configuration.py b/packages/tentacles/Services/Interfaces/web_interface/controllers/configuration.py index cafa62b063..91b93f5357 100644 --- a/packages/tentacles/Services/Interfaces/web_interface/controllers/configuration.py +++ b/packages/tentacles/Services/Interfaces/web_interface/controllers/configuration.py @@ -20,6 +20,7 @@ import octobot_commons.constants as commons_constants import octobot_commons.enums as commons_enums +import octobot_commons.errors as errors import octobot_commons.authentication as authentication import octobot_services.constants as services_constants import tentacles.Services.Interfaces.web_interface.constants as constants @@ -40,11 +41,15 @@ def profile(): selected_profile = flask.request.args.get("select", None) next_url = flask.request.args.get("next", None) if selected_profile is not None and selected_profile != models.get_current_profile().profile_id: - models.select_profile(selected_profile) - current_profile = models.get_current_profile() - flask.flash( - f"Selected the {current_profile.name} profile", "success" - ) + try: + models.select_profile(selected_profile) + current_profile = models.get_current_profile() + flask.flash( + f"Selected the {current_profile.name} profile", "success" + ) + except errors.NoProfileError: + flask.flash("The requested profile no longer exists.", "warning") + current_profile = models.get_current_profile() else: current_profile = models.get_current_profile() if next_url is not None: diff --git a/packages/tentacles/Services/Interfaces/web_interface/controllers/tentacles_config.py b/packages/tentacles/Services/Interfaces/web_interface/controllers/tentacles_config.py index 73027675db..cda0286be4 100644 --- a/packages/tentacles/Services/Interfaces/web_interface/controllers/tentacles_config.py +++ b/packages/tentacles/Services/Interfaces/web_interface/controllers/tentacles_config.py @@ -65,6 +65,7 @@ def config_tentacle(): return util.get_rest_reply(response, 500) else: if flask.request.args: + models.refresh_sync_profiles_for_display() tentacle_name = flask.request.args.get("name") missing_tentacles = set() media_url = flask.url_for("tentacle_media", _external=True) @@ -113,6 +114,7 @@ def config_tentacle(): @login.login_required_when_activated def config_tentacle_edit_details(tentacle): try: + models.refresh_sync_profiles_for_display() profile_id = flask.request.args.get("profile", None) return util.get_rest_reply( models.get_tentacle_config_and_edit_display(tentacle, profile_id=profile_id) diff --git a/packages/tentacles/Services/Interfaces/web_interface/models/__init__.py b/packages/tentacles/Services/Interfaces/web_interface/models/__init__.py index 72a2354d48..7ec69423e3 100644 --- a/packages/tentacles/Services/Interfaces/web_interface/models/__init__.py +++ b/packages/tentacles/Services/Interfaces/web_interface/models/__init__.py @@ -231,6 +231,7 @@ convert_to_live_profile, select_profile, get_profiles, + refresh_sync_profiles_for_display, get_profiles_tentacles_details, update_profile, remove_profile, @@ -446,6 +447,7 @@ "convert_to_live_profile", "select_profile", "get_profiles", + "refresh_sync_profiles_for_display", "get_profiles_tentacles_details", "update_profile", "remove_profile", diff --git a/packages/tentacles/Services/Interfaces/web_interface/models/configuration.py b/packages/tentacles/Services/Interfaces/web_interface/models/configuration.py index 79ae02f7b9..7e126463cc 100644 --- a/packages/tentacles/Services/Interfaces/web_interface/models/configuration.py +++ b/packages/tentacles/Services/Interfaces/web_interface/models/configuration.py @@ -109,6 +109,10 @@ ] _TENTACLE_CONFIG_CACHE = {} + +def clear_tentacle_config_cache(): + _TENTACLE_CONFIG_CACHE.clear() + DEFAULT_EXCHANGE = "binance" MERGED_CCXT_EXCHANGES = { result.__name__: [merged_exchange.__name__ for merged_exchange in merged] @@ -484,6 +488,24 @@ def get_tentacles_activation_desc_by_group(media_url, missing_tentacles: set): if len(tentacles) > 1} +def _persist_profile_tentacles_changes(tentacles_setup_config): + config = interfaces_util.get_edited_config(dict_only=False) + profile = config.profile + if profile is None: + tentacles_manager_api.save_tentacles_setup_configuration(tentacles_setup_config) + return + if profile.is_profile_data_tentacle_backed(): + edited_profile = tentacles_setup_config.profile + if edited_profile is not None: + profile.get_profile_data().tentacles = list( + edited_profile.get_profile_data().tentacles + ) + profile.bind_tentacles_setup_config(tentacles_setup_config) + config.save(save_profile=True) + return + tentacles_manager_api.save_tentacles_setup_configuration(tentacles_setup_config) + + def update_tentacle_config(tentacle_name, config_update, tentacle_class=None, tentacles_setup_config=None): try: tentacle_class = tentacle_class or get_tentacle_from_string(tentacle_name, None, with_info=False)[0] @@ -494,6 +516,9 @@ def update_tentacle_config(tentacle_name, config_update, tentacle_class=None, te tentacle_class, config_update ) + _persist_profile_tentacles_changes( + tentacles_setup_config or interfaces_util.get_edited_tentacles_config() + ) return True, f"{tentacle_name} updated" except errors.InvalidAutomationConfigError: raise # propagate the error to the caller @@ -519,6 +544,9 @@ def reset_config_to_default(tentacle_name, tentacle_class=None, tentacles_setup_ tentacles_setup_config or interfaces_util.get_edited_tentacles_config(), tentacle_class ) + _persist_profile_tentacles_changes( + tentacles_setup_config or interfaces_util.get_edited_tentacles_config() + ) return True, f"{tentacle_name} configuration reset to default values" except FileNotFoundError as e: error_message = f"Error when resetting factory tentacle config: no default values file at {e.filename}" @@ -717,7 +745,7 @@ def update_tentacles_activation_config(new_config, deactivate_others=False, tent if tentacles_manager_api.update_activation_configuration( tentacles_setup_configuration, updated_config, deactivate_others ): - tentacles_manager_api.save_tentacles_setup_configuration(tentacles_setup_configuration) + _persist_profile_tentacles_changes(tentacles_setup_configuration) return True except Exception as e: _get_logger().exception(e, True, f"Error when updating tentacles activation {e}") @@ -906,6 +934,12 @@ async def _load_market(exchange, results): ) as client: await client.load_markets() symbols = client.symbols + elif not hasattr(ccxt.async_support, exchange): + _get_logger().warning( + "Skipping symbol list load for unknown exchange %r: not available in ccxt", + exchange, + ) + return else: async with getattr(ccxt.async_support, exchange)({'verbose': False}) as client: client.logger.setLevel(logging.INFO) # prevent log of each request (huge on market statuses) diff --git a/packages/tentacles/Services/Interfaces/web_interface/models/profiles.py b/packages/tentacles/Services/Interfaces/web_interface/models/profiles.py index 5ef719bd66..570a7404b3 100644 --- a/packages/tentacles/Services/Interfaces/web_interface/models/profiles.py +++ b/packages/tentacles/Services/Interfaces/web_interface/models/profiles.py @@ -15,6 +15,7 @@ # License along with this library. import os +import octobot_commons.logging as logging import octobot_services.interfaces.util as interfaces_util import octobot_commons.profiles as profiles import octobot_commons.errors as errors @@ -22,10 +23,13 @@ import octobot_commons.authentication as authentication import octobot_trading.util as trading_util import octobot_tentacles_manager.api as tentacles_manager_api +import octobot_tentacles_manager.constants as tentacles_manager_constants import octobot.constants as constants import octobot.community as community import octobot.community.errors as community_errors +import tentacles.Services.Interfaces.web_interface.models.configuration as configuration_model + ACTIVATION = "activation" VERSION = "version" @@ -35,6 +39,42 @@ _PROFILE_TENTACLES_CONFIG_CACHE = {} +def _get_logger(): + return logging.get_logger("WebProfileModel") + + +def _fallback_tentacles_details(profile, *, read_error: bool = True) -> dict: + return { + ACTIVATION: [], + VERSION: tentacles_manager_constants.TENTACLE_INSTALLATION_CONTEXT_OCTOBOT_VERSION_UNKNOWN, + IMPORTED: profile.imported, + REQUIRE_EXACT_VERSION: False, + READ_ERROR: read_error, + } + + +def _resolve_profile_tentacles_setup_config(profile, *, force_reload: bool = False): + if profile.is_profile_data_tentacle_backed(): + if force_reload or profile.tentacles_setup_config is None: + profile.init_tentacles_setup_config() + tentacles_setup_config = profile.tentacles_setup_config + return profile.bind_tentacles_setup_config(tentacles_setup_config) + return tentacles_manager_api.get_tentacles_setup_config( + profile.get_tentacles_config_path(), + profile=profile, + ) + + +def _ensure_profile_in_config(config, profile_id): + if profile_id in config.profile_by_id: + return config.profile_by_id[profile_id] + config.load_profiles() + if profile_id in config.profile_by_id: + return config.profile_by_id[profile_id] + profile = config.profile_storage.load_profile_by_id(profile_id) + config.profile_by_id[profile_id] = profile + return profile + def get_current_profile(): return interfaces_util.get_edited_config(dict_only=False).profile @@ -43,10 +83,13 @@ def get_current_profile(): def duplicate_profile(profile_id): to_duplicate = get_profile(profile_id) new_profile = to_duplicate.duplicate(name=f"{to_duplicate.name}_(copy)", description=to_duplicate.description) - tentacles_manager_api.refresh_profile_tentacles_setup_config(new_profile.path) + if not new_profile.is_profile_data_tentacle_backed(): + tentacles_manager_api.refresh_profile_tentacles_setup_config(new_profile.path) interfaces_util.get_edited_config(dict_only=False).load_profiles() - return get_profile(new_profile.profile_id) - + new_profile = get_profile(new_profile.profile_id) + if new_profile.is_profile_data_tentacle_backed(): + new_profile.init_tentacles_setup_config() + return new_profile def convert_to_live_profile(profile_id): profile = get_profile(profile_id) @@ -55,7 +98,9 @@ def convert_to_live_profile(profile_id): def select_profile(profile_id): - _select_and_save(interfaces_util.get_edited_config(dict_only=False), profile_id) + config = interfaces_util.get_edited_config(dict_only=False) + _ensure_profile_in_config(config, profile_id) + _select_and_save(config, profile_id) def _select_and_save(config, profile_id): @@ -64,13 +109,30 @@ def _select_and_save(config, profile_id): config.save() -def _update_edited_tentacles_config(config): - updated_tentacles_config = tentacles_manager_api.get_tentacles_setup_config(config.get_tentacles_config_path()) +def _update_edited_tentacles_config(config, *, force_reload: bool = False): + updated_tentacles_config = _resolve_profile_tentacles_setup_config( + config.profile, + force_reload=force_reload, + ) interfaces_util.set_edited_tentacles_config(updated_tentacles_config) +def refresh_sync_profiles_for_display(config=None): + if config is None: + config = interfaces_util.get_edited_config(dict_only=False) + config.refresh_sync_profiles() + force_reload = config.profile is not None and config.profile.is_sync_backed() + if force_reload: + _PROFILE_TENTACLES_CONFIG_CACHE.pop(config.profile.profile_id, None) + _update_edited_tentacles_config(config, force_reload=force_reload) + + configuration_model.clear_tentacle_config_cache() + return config + + def get_profile(profile_id): - return interfaces_util.get_edited_config(dict_only=False).profile_by_id[profile_id] + config = interfaces_util.get_edited_config(dict_only=False) + return _ensure_profile_in_config(config, profile_id) def get_tentacles_setup_config_from_profile_id(profile_id): @@ -78,32 +140,29 @@ def get_tentacles_setup_config_from_profile_id(profile_id): def get_tentacles_setup_config_from_profile(profile): - return tentacles_manager_api.get_tentacles_setup_config( - profile.get_tentacles_config_path() - ) + return _resolve_profile_tentacles_setup_config(profile) def get_profiles(profile_type: commons_enums.ProfileType = None): + config = refresh_sync_profiles_for_display() return { identifier: profile - for identifier, profile in interfaces_util.get_edited_config(dict_only=False).profile_by_id.items() + for identifier, profile in config.profile_by_id.items() if profile_type is None or profile.profile_type is profile_type } def _get_profile_setup_config(profile, reloading_profile): - if profile.profile_id == reloading_profile: - _PROFILE_TENTACLES_CONFIG_CACHE.pop(reloading_profile, None) - return tentacles_manager_api.get_tentacles_setup_config( - profile.get_tentacles_config_path() + force_reload = profile.profile_id == reloading_profile + if force_reload: + _PROFILE_TENTACLES_CONFIG_CACHE.pop(profile.profile_id, None) + if profile.is_profile_data_tentacle_backed(): + return _resolve_profile_tentacles_setup_config(profile, force_reload=force_reload) + if profile.profile_id not in _PROFILE_TENTACLES_CONFIG_CACHE or force_reload: + _PROFILE_TENTACLES_CONFIG_CACHE[profile.profile_id] = _resolve_profile_tentacles_setup_config( + profile, + force_reload=force_reload, ) - try: - _PROFILE_TENTACLES_CONFIG_CACHE[profile.profile_id] - except KeyError: - _PROFILE_TENTACLES_CONFIG_CACHE[profile.profile_id] = \ - tentacles_manager_api.get_tentacles_setup_config( - profile.get_tentacles_config_path() - ) return _PROFILE_TENTACLES_CONFIG_CACHE[profile.profile_id] @@ -122,12 +181,15 @@ def get_profiles_tentacles_details(profiles_list): READ_ERROR: not tentacles_manager_api.is_tentacles_setup_config_successfully_loaded(tentacles_setup_config), } - except Exception: - # do not raise here to prevent avoid config display - pass + except Exception as err: + _get_logger().warning( + "Failed to load tentacles details for profile %r: %s", + profile.profile_id, + err, + ) + tentacles_by_profile_id[profile.profile_id] = _fallback_tentacles_details(profile) return tentacles_by_profile_id - def update_profile(profile_id, json_profile_desc, json_profile_content=None): profile = get_profile(profile_id) new_name = json_profile_desc.get("name", profile.name) @@ -142,11 +204,10 @@ def update_profile(profile_id, json_profile_desc, json_profile_content=None): if json_profile_content is not None: profile.config = json_profile_content profile.validate_and_save_config() - if renamed: + if renamed and not profile.is_sync_backed(): profile.rename_folder(new_name, False) return True, "Profile updated" - def remove_profile(profile_id): profile = None if get_current_profile().profile_id == profile_id: diff --git a/packages/tentacles/Services/Interfaces/web_interface/tests/test_configuration_models.py b/packages/tentacles/Services/Interfaces/web_interface/tests/test_configuration_models.py new file mode 100644 index 0000000000..69b1a50929 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/web_interface/tests/test_configuration_models.py @@ -0,0 +1,151 @@ +# Drakkar-Software OctoBot-Tentacles +# Copyright (c) Drakkar-Software, All rights reserved. + +import mock +import pytest + +import tentacles.Services.Interfaces.web_interface.models.configuration as configuration_model + + +# All test coroutines will be treated as marked. +pytestmark = pytest.mark.asyncio + + +class TestPersistProfileTentaclesChanges: + def test_sync_profile_calls_config_save(self): + edited_tentacles = [mock.Mock(name="edited-tentacle")] + edited_profile_data = mock.Mock() + edited_profile_data.tentacles = edited_tentacles + edited_profile = mock.Mock() + edited_profile.get_profile_data.return_value = edited_profile_data + + active_profile_data = mock.Mock() + active_profile_data.tentacles = [] + active_profile = mock.Mock() + active_profile.is_profile_data_tentacle_backed.return_value = True + active_profile.get_profile_data.return_value = active_profile_data + + tentacles_setup_config = mock.Mock() + tentacles_setup_config.profile = edited_profile + + config = mock.Mock() + config.profile = active_profile + + with mock.patch.object( + configuration_model.interfaces_util, + "get_edited_config", + mock.Mock(return_value=config), + ), mock.patch.object( + configuration_model.tentacles_manager_api, + "save_tentacles_setup_configuration", + mock.Mock(), + ) as save_setup_mock: + configuration_model._persist_profile_tentacles_changes(tentacles_setup_config) + + assert active_profile_data.tentacles == edited_tentacles + active_profile.bind_tentacles_setup_config.assert_called_once_with(tentacles_setup_config) + config.save.assert_called_once_with(save_profile=True) + save_setup_mock.assert_not_called() + + def test_filesystem_profile_calls_save_tentacles_setup(self): + profile = mock.Mock() + profile.is_profile_data_tentacle_backed.return_value = False + config = mock.Mock() + config.profile = profile + tentacles_setup_config = mock.Mock() + + with mock.patch.object( + configuration_model.interfaces_util, + "get_edited_config", + mock.Mock(return_value=config), + ), mock.patch.object( + configuration_model.tentacles_manager_api, + "save_tentacles_setup_configuration", + mock.Mock(), + ) as save_setup_mock: + configuration_model._persist_profile_tentacles_changes(tentacles_setup_config) + + save_setup_mock.assert_called_once_with(tentacles_setup_config) + config.save.assert_not_called() + + def test_leaves_startup_copy_unchanged(self): + startup_setup = mock.Mock(name="startup-setup") + original_startup_profile = mock.Mock(name="startup-profile") + startup_setup.profile = original_startup_profile + + edited_tentacles = [mock.Mock(name="edited-tentacle")] + edited_profile_data = mock.Mock() + edited_profile_data.tentacles = edited_tentacles + edited_profile = mock.Mock() + edited_profile.get_profile_data.return_value = edited_profile_data + + active_profile_data = mock.Mock() + active_profile_data.tentacles = [] + active_profile = mock.Mock() + active_profile.is_profile_data_tentacle_backed.return_value = True + active_profile.get_profile_data.return_value = active_profile_data + + tentacles_setup_config = mock.Mock(name="edited-setup") + tentacles_setup_config.profile = edited_profile + + config = mock.Mock() + config.profile = active_profile + + with mock.patch.object( + configuration_model.interfaces_util, + "get_edited_config", + mock.Mock(return_value=config), + ): + configuration_model._persist_profile_tentacles_changes(tentacles_setup_config) + + active_profile.bind_tentacles_setup_config.assert_called_once_with(tentacles_setup_config) + assert startup_setup.profile is original_startup_profile + assert tentacles_setup_config is not startup_setup + + +class TestUpdateTentaclesActivationConfig: + def test_persists_sync_profile(self): + tentacles_setup_config = mock.Mock() + with mock.patch.object( + configuration_model.interfaces_util, + "get_edited_tentacles_config", + mock.Mock(return_value=tentacles_setup_config), + ), mock.patch.object( + configuration_model.tentacles_manager_api, + "update_activation_configuration", + mock.Mock(return_value=True), + ), mock.patch.object( + configuration_model, + "_persist_profile_tentacles_changes", + mock.Mock(), + ) as persist_mock, mock.patch.object( + configuration_model.tentacles_manager_api, + "save_tentacles_setup_configuration", + mock.Mock(), + ) as save_setup_mock: + result = configuration_model.update_tentacles_activation_config( + {"IndexTradingMode": "true"} + ) + + assert result is True + persist_mock.assert_called_once_with(tentacles_setup_config) + save_setup_mock.assert_not_called() + + +class TestLoadMarketUnknownExchange: + async def test_skips_unknown_exchange_without_exception(self): + results = [] + logger = mock.Mock() + with mock.patch.object( + configuration_model, + "auto_filled_exchanges", + mock.Mock(return_value=[]), + ), mock.patch.object( + configuration_model, + "_get_logger", + mock.Mock(return_value=logger), + ): + await configuration_model._load_market("earn_curve", results) + assert results == [] + logger.warning.assert_called_once() + logger.exception.assert_not_called() diff --git a/packages/tentacles/Services/Interfaces/web_interface/tests/test_profiles_models.py b/packages/tentacles/Services/Interfaces/web_interface/tests/test_profiles_models.py new file mode 100644 index 0000000000..6b3af87415 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/web_interface/tests/test_profiles_models.py @@ -0,0 +1,415 @@ +# Drakkar-Software OctoBot-Tentacles +# Copyright (c) Drakkar-Software, All rights reserved. + +import mock +import pytest + +import octobot_commons.errors as commons_errors +import octobot_tentacles_manager.constants as tentacles_manager_constants +import tentacles.Services.Interfaces.web_interface.models.profiles as profiles_model + + +class TestResolveProfileTentaclesSetupConfig: + def test_profile_data_backed_calls_init_tentacles_setup_config_when_uninitialized(self): + profile = mock.Mock() + profile.is_profile_data_tentacle_backed.return_value = True + initialized_setup = mock.Mock() + + def init_side_effect(): + profile.tentacles_setup_config = initialized_setup + + profile.tentacles_setup_config = None + profile.init_tentacles_setup_config.side_effect = init_side_effect + + def bind_side_effect(setup): + setup.profile = profile + return setup + + profile.bind_tentacles_setup_config.side_effect = bind_side_effect + + result = profiles_model._resolve_profile_tentacles_setup_config(profile) + + profile.init_tentacles_setup_config.assert_called_once_with() + profile.bind_tentacles_setup_config.assert_called_once_with(initialized_setup) + profile.get_tentacles_config_path.assert_not_called() + assert result is initialized_setup + assert result.profile is profile + + def test_profile_data_backed_reuses_existing_setup_config(self): + profile = mock.Mock() + profile.is_profile_data_tentacle_backed.return_value = True + profile.tentacles_setup_config = mock.Mock() + + def bind_side_effect(setup): + setup.profile = profile + return setup + + profile.bind_tentacles_setup_config.side_effect = bind_side_effect + + result = profiles_model._resolve_profile_tentacles_setup_config(profile) + + profile.init_tentacles_setup_config.assert_not_called() + profile.bind_tentacles_setup_config.assert_called_once_with(profile.tentacles_setup_config) + assert result is profile.tentacles_setup_config + assert result.profile is profile + + def test_filesystem_profile_uses_tentacles_config_path(self): + profile = mock.Mock() + profile.is_profile_data_tentacle_backed.return_value = False + profile.get_tentacles_config_path.return_value = "/profiles/test/tentacles_config.json" + setup_config = mock.Mock() + with mock.patch.object( + profiles_model.tentacles_manager_api, + "get_tentacles_setup_config", + mock.Mock(return_value=setup_config), + ) as get_setup_config_mock: + result = profiles_model._resolve_profile_tentacles_setup_config(profile) + get_setup_config_mock.assert_called_once_with( + "/profiles/test/tentacles_config.json", + profile=profile, + ) + assert result is setup_config + + +class TestUpdateEditedTentaclesConfig: + def test_profile_data_backed_uses_resolve_not_filesystem_path(self): + profile = mock.Mock() + profile.is_profile_data_tentacle_backed.return_value = True + config = mock.Mock() + config.profile = profile + resolved_setup = mock.Mock() + + with mock.patch.object( + profiles_model, + "_resolve_profile_tentacles_setup_config", + mock.Mock(return_value=resolved_setup), + ) as resolve_mock, mock.patch.object( + profiles_model.interfaces_util, + "set_edited_tentacles_config", + mock.Mock(), + ) as set_edited_mock, mock.patch.object( + profiles_model.tentacles_manager_api, + "get_tentacles_setup_config", + mock.Mock(), + ) as get_setup_config_mock: + profiles_model._update_edited_tentacles_config(config) + + resolve_mock.assert_called_once_with(profile, force_reload=False) + set_edited_mock.assert_called_once_with(resolved_setup) + config.get_tentacles_config_path.assert_not_called() + get_setup_config_mock.assert_not_called() + + +class TestRefreshSyncProfilesForDisplay: + def test_refreshes_sync_profiles_rebinds_setup_and_clears_cache(self): + sync_profile = mock.Mock() + sync_profile.profile_id = "sync-profile-id" + sync_profile.is_sync_backed.return_value = True + config = mock.Mock() + config.profile = sync_profile + resolved_setup = mock.Mock() + + with mock.patch.object( + profiles_model.interfaces_util, + "get_edited_config", + mock.Mock(return_value=config), + ), mock.patch.object( + profiles_model, + "_update_edited_tentacles_config", + mock.Mock(), + ) as update_edited_mock, mock.patch( + "tentacles.Services.Interfaces.web_interface.models.configuration.clear_tentacle_config_cache", + mock.Mock(), + ) as clear_cache_mock, mock.patch.object( + profiles_model, + "_resolve_profile_tentacles_setup_config", + mock.Mock(return_value=resolved_setup), + ): + result = profiles_model.refresh_sync_profiles_for_display() + + config.refresh_sync_profiles.assert_called_once_with() + update_edited_mock.assert_called_once_with(config, force_reload=True) + clear_cache_mock.assert_called_once_with() + assert result is config + assert "sync-profile-id" not in profiles_model._PROFILE_TENTACLES_CONFIG_CACHE + + def test_skips_profile_cache_pop_for_filesystem_profile(self): + filesystem_profile = mock.Mock() + filesystem_profile.profile_id = "filesystem-profile-id" + filesystem_profile.is_sync_backed.return_value = False + config = mock.Mock() + config.profile = filesystem_profile + profiles_model._PROFILE_TENTACLES_CONFIG_CACHE["filesystem-profile-id"] = mock.Mock() + + with mock.patch.object( + profiles_model, + "_update_edited_tentacles_config", + mock.Mock(), + ) as update_edited_mock, mock.patch( + "tentacles.Services.Interfaces.web_interface.models.configuration.clear_tentacle_config_cache", + mock.Mock(), + ): + profiles_model.refresh_sync_profiles_for_display(config) + + update_edited_mock.assert_called_once_with(config, force_reload=False) + assert "filesystem-profile-id" in profiles_model._PROFILE_TENTACLES_CONFIG_CACHE + + +class TestGetProfiles: + def test_calls_refresh_sync_profiles_before_filtering(self): + live_profile = mock.Mock() + live_profile.profile_type = profiles_model.commons_enums.ProfileType.LIVE + simulator_profile = mock.Mock() + simulator_profile.profile_type = profiles_model.commons_enums.ProfileType.BACKTESTING + config = mock.Mock() + config.profile_by_id = { + "live-profile-id": live_profile, + "simulator-profile-id": simulator_profile, + } + + with mock.patch.object( + profiles_model, + "refresh_sync_profiles_for_display", + mock.Mock(return_value=config), + ) as refresh_mock: + result = profiles_model.get_profiles(profiles_model.commons_enums.ProfileType.LIVE) + + refresh_mock.assert_called_once_with() + assert result == {"live-profile-id": live_profile} + + +class TestEnsureProfileInConfig: + def test_returns_cached_profile_without_reload(self): + cached_profile = mock.Mock() + config = mock.Mock() + config.profile_by_id = {"cached-profile-id": cached_profile} + + result = profiles_model._ensure_profile_in_config(config, "cached-profile-id") + + assert result is cached_profile + config.load_profiles.assert_not_called() + config.profile_storage.load_profile_by_id.assert_not_called() + + def test_loads_from_profile_storage_when_missing_from_cache(self): + config = mock.Mock() + config.profile_by_id = {} + loaded_profile = mock.Mock() + config.profile_storage.load_profile_by_id.return_value = loaded_profile + + result = profiles_model._ensure_profile_in_config(config, "storage-profile-id") + + config.load_profiles.assert_called_once_with() + config.profile_storage.load_profile_by_id.assert_called_once_with("storage-profile-id") + assert config.profile_by_id["storage-profile-id"] is loaded_profile + assert result is loaded_profile + + def test_raises_no_profile_error_when_unknown(self): + config = mock.Mock() + config.profile_by_id = {} + config.profile_storage.load_profile_by_id.side_effect = commons_errors.NoProfileError( + "No profile with id: missing-profile-id" + ) + + with pytest.raises(commons_errors.NoProfileError): + profiles_model._ensure_profile_in_config(config, "missing-profile-id") + + +class TestSelectProfile: + def test_select_profile_updates_edited_tentacles_for_sync_profile(self): + sync_profile = mock.Mock() + sync_profile.is_profile_data_tentacle_backed.return_value = True + sync_profile.tentacles_setup_config = mock.Mock() + config = mock.Mock() + config.profile = mock.Mock() + config.profile_by_id = {"sync-profile-id": sync_profile} + + def select_side_effect(profile_id): + config.profile = sync_profile + + config.select_profile.side_effect = select_side_effect + resolved_setup = mock.Mock() + + with mock.patch.object( + profiles_model.interfaces_util, + "get_edited_config", + mock.Mock(return_value=config), + ), mock.patch.object( + profiles_model, + "_resolve_profile_tentacles_setup_config", + mock.Mock(return_value=resolved_setup), + ) as resolve_mock, mock.patch.object( + profiles_model.interfaces_util, + "set_edited_tentacles_config", + mock.Mock(), + ) as set_edited_mock: + profiles_model.select_profile("sync-profile-id") + + config.select_profile.assert_called_once_with("sync-profile-id") + resolve_mock.assert_called_once_with(sync_profile, force_reload=False) + set_edited_mock.assert_called_once_with(resolved_setup) + config.save.assert_called_once_with() + config.get_tentacles_config_path.assert_not_called() + + def test_select_profile_raises_no_profile_error_for_stale_id(self): + config = mock.Mock() + config.profile_by_id = {} + config.profile_storage.load_profile_by_id.side_effect = commons_errors.NoProfileError( + "No profile with id: stale-profile-id" + ) + + with mock.patch.object( + profiles_model.interfaces_util, + "get_edited_config", + mock.Mock(return_value=config), + ), pytest.raises(commons_errors.NoProfileError): + profiles_model.select_profile("stale-profile-id") + + config.select_profile.assert_not_called() + config.save.assert_not_called() + + +class TestGetProfilesTentaclesDetails: + @pytest.fixture(autouse=True) + def _clear_setup_config_cache(self): + profiles_model._PROFILE_TENTACLES_CONFIG_CACHE.clear() + yield + profiles_model._PROFILE_TENTACLES_CONFIG_CACHE.clear() + + def test_returns_details_for_filesystem_and_profile_data_backed_profiles(self): + filesystem_profile = mock.Mock() + filesystem_profile.profile_id = "filesystem-profile-id" + filesystem_profile.is_profile_data_tentacle_backed.return_value = False + filesystem_profile.imported = False + filesystem_profile.get_tentacles_config_path.return_value = "/profiles/fs/tentacles_config.json" + + sync_profile = mock.Mock() + sync_profile.profile_id = "sync-profile-id" + sync_profile.is_profile_data_tentacle_backed.return_value = True + sync_profile.imported = False + sync_profile.tentacles_setup_config = None + initialized_sync_setup = mock.Mock() + + def init_sync_setup(): + sync_profile.tentacles_setup_config = initialized_sync_setup + + sync_profile.init_tentacles_setup_config.side_effect = init_sync_setup + + current_profile = mock.Mock() + current_profile.profile_id = "filesystem-profile-id" + + filesystem_setup = mock.Mock() + with mock.patch.object(profiles_model, "get_current_profile", mock.Mock(return_value=current_profile)), mock.patch.object( + profiles_model.tentacles_manager_api, + "get_tentacles_setup_config", + mock.Mock(return_value=filesystem_setup), + ), mock.patch.object( + profiles_model.tentacles_manager_api, + "get_activated_tentacles", + mock.Mock(side_effect=[["ModeA"], ["ModeB"]]), + ), mock.patch.object( + profiles_model.tentacles_manager_api, + "get_tentacles_installation_version", + mock.Mock(side_effect=["2.1.1", "2.1.1"]), + ), mock.patch.object( + profiles_model.tentacles_manager_api, + "is_tentacles_setup_config_successfully_loaded", + mock.Mock(return_value=True), + ): + details = profiles_model.get_profiles_tentacles_details( + { + filesystem_profile.profile_id: filesystem_profile, + sync_profile.profile_id: sync_profile, + } + ) + + assert set(details) == {"filesystem-profile-id", "sync-profile-id"} + assert details["filesystem-profile-id"][profiles_model.ACTIVATION] == ["ModeA"] + assert details["sync-profile-id"][profiles_model.ACTIVATION] == ["ModeB"] + sync_profile.init_tentacles_setup_config.assert_called() + + def test_failed_profile_gets_fallback_entry(self): + broken_profile = mock.Mock() + broken_profile.profile_id = "broken-profile-id" + broken_profile.imported = True + + current_profile = mock.Mock() + current_profile.profile_id = "other-profile-id" + + with mock.patch.object(profiles_model, "get_current_profile", mock.Mock(return_value=current_profile)), mock.patch.object( + profiles_model, + "_get_profile_setup_config", + mock.Mock(side_effect=RuntimeError("load failed")), + ): + details = profiles_model.get_profiles_tentacles_details( + {broken_profile.profile_id: broken_profile} + ) + + assert broken_profile.profile_id in details + assert details[broken_profile.profile_id][profiles_model.READ_ERROR] is True + assert details[broken_profile.profile_id][profiles_model.ACTIVATION] == [] + assert details[broken_profile.profile_id][profiles_model.VERSION] == ( + tentacles_manager_constants.TENTACLE_INSTALLATION_CONTEXT_OCTOBOT_VERSION_UNKNOWN + ) + + +class TestDuplicateProfile: + def test_sync_duplicate_skips_filesystem_refresh_and_inits_tentacles_setup(self): + source_profile = mock.Mock() + source_profile.name = "Source" + source_profile.description = "desc" + duplicated_profile = mock.Mock() + duplicated_profile.profile_id = "new-sync-profile-id" + duplicated_profile.is_profile_data_tentacle_backed.return_value = True + source_profile.duplicate.return_value = duplicated_profile + + config = mock.Mock() + reloaded_profile = mock.Mock() + reloaded_profile.profile_id = "new-sync-profile-id" + reloaded_profile.is_profile_data_tentacle_backed.return_value = True + + with mock.patch.object(profiles_model, "get_profile", mock.Mock(side_effect=[source_profile, reloaded_profile])), mock.patch.object( + profiles_model.interfaces_util, + "get_edited_config", + mock.Mock(return_value=config), + ), mock.patch.object( + profiles_model.tentacles_manager_api, + "refresh_profile_tentacles_setup_config", + mock.Mock(), + ) as refresh_mock: + result = profiles_model.duplicate_profile("source-profile-id") + + source_profile.duplicate.assert_called_once_with(name="Source_(copy)", description="desc") + refresh_mock.assert_not_called() + config.load_profiles.assert_called_once_with() + reloaded_profile.init_tentacles_setup_config.assert_called_once_with() + assert result is reloaded_profile + + def test_filesystem_duplicate_refreshes_tentacles_on_disk(self): + source_profile = mock.Mock() + source_profile.name = "Source" + source_profile.description = "desc" + duplicated_profile = mock.Mock() + duplicated_profile.profile_id = "new-filesystem-profile-id" + duplicated_profile.path = "/profiles/new" + duplicated_profile.is_profile_data_tentacle_backed.return_value = False + source_profile.duplicate.return_value = duplicated_profile + + config = mock.Mock() + reloaded_profile = mock.Mock() + reloaded_profile.profile_id = "new-filesystem-profile-id" + reloaded_profile.is_profile_data_tentacle_backed.return_value = False + + with mock.patch.object(profiles_model, "get_profile", mock.Mock(side_effect=[source_profile, reloaded_profile])), mock.patch.object( + profiles_model.interfaces_util, + "get_edited_config", + mock.Mock(return_value=config), + ), mock.patch.object( + profiles_model.tentacles_manager_api, + "refresh_profile_tentacles_setup_config", + mock.Mock(), + ) as refresh_mock: + result = profiles_model.duplicate_profile("source-profile-id") + + refresh_mock.assert_called_once_with("/profiles/new") + reloaded_profile.init_tentacles_setup_config.assert_not_called() + assert result is reloaded_profile diff --git a/packages/tentacles/Trading/Mode/dca_trading_mode/dca_trading.py b/packages/tentacles/Trading/Mode/dca_trading_mode/dca_trading.py index 1ee3a4f5e2..d2407a44a8 100644 --- a/packages/tentacles/Trading/Mode/dca_trading_mode/dca_trading.py +++ b/packages/tentacles/Trading/Mode/dca_trading_mode/dca_trading.py @@ -1058,12 +1058,14 @@ def init_user_inputs(self, inputs: dict) -> None: DCATradingMode.TRADING_PAIRS, commons_enums.UserInputTypes.MULTIPLE_OPTIONS, default_config[DCATradingMode.TRADING_PAIRS], inputs, options=[], + other_schema_values={"minItems": 0}, title="Traded pairs: symbols to apply DCA on.", ) self.UI.user_input( DCATradingMode.TIME_FRAMES, commons_enums.UserInputTypes.MULTIPLE_OPTIONS, default_config[DCATradingMode.TIME_FRAMES], inputs, options=[time_frame.value for time_frame in commons_enums.TimeFrames], + other_schema_values={"minItems": 0}, title="Required time frames: time frames to apply DCA on.", ) diff --git a/packages/tentacles_manager/octobot_tentacles_manager/api/configurator.py b/packages/tentacles_manager/octobot_tentacles_manager/api/configurator.py index f821b0defe..bb2e9517ac 100644 --- a/packages/tentacles_manager/octobot_tentacles_manager/api/configurator.py +++ b/packages/tentacles_manager/octobot_tentacles_manager/api/configurator.py @@ -27,6 +27,7 @@ import octobot_tentacles_manager.managers as managers if TYPE_CHECKING: + import octobot_commons.profiles.profile_types.profile as profile_module from octobot_tentacles_manager.configuration import TentaclesSetupConfiguration @@ -64,12 +65,15 @@ def refresh_all_tentacles_setup_configs( def get_tentacles_setup_config( - config_path: str = None + config_path: str = None, + profile: "profile_module.Profile | None" = None, ) -> "TentaclesSetupConfiguration": if config_path is None: config_path = user_root_folder_provider.get_user_reference_tentacle_config_file_path() setup_config = configuration.TentaclesSetupConfiguration(config_path=config_path) setup_config.read_config() + if profile is not None: + setup_config.profile = profile return setup_config @@ -195,8 +199,8 @@ def get_tentacle_config(tentacles_setup_config: "TentaclesSetupConfiguration", k return configuration.get_config(tentacles_setup_config, klass) -def set_tentacle_config_proxy(new_proxy) -> dict: - return configuration.set_get_config_proxy(new_proxy) +def set_tentacle_config_proxy(new_proxy): + return configuration.local_get_config_proxy(new_proxy) def local_tentacle_config_proxy(new_proxy): diff --git a/packages/tentacles_manager/octobot_tentacles_manager/api/inspector.py b/packages/tentacles_manager/octobot_tentacles_manager/api/inspector.py index 5804883223..26eb286caa 100644 --- a/packages/tentacles_manager/octobot_tentacles_manager/api/inspector.py +++ b/packages/tentacles_manager/octobot_tentacles_manager/api/inspector.py @@ -13,6 +13,8 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +from __future__ import annotations + import functools import packaging.version as packaging_version import typing @@ -92,46 +94,56 @@ def check_tentacle_version(version, name, origin_package, verbose=True) -> bool: return True +def _get_tentacle_class_from_module(tentacle_name, parent, module, parent_inspection): + return tentacles_management.get_class_from_string( + tentacle_name, + parent, + module, + parent_inspection, + case_insensitive=True, + ) + + def _load_tentacle_class(tentacle_name): # Lazy import of tentacles to let tentacles manager handle imports try: import octobot_evaluators.evaluators as evaluators import tentacles.Evaluator as tentacles_Evaluator - if tentacle_class := tentacles_management.get_class_from_string( + if tentacle_class := _get_tentacle_class_from_module( tentacle_name, evaluators.StrategyEvaluator, tentacles_Evaluator.Strategies, tentacles_management.evaluator_parent_inspection): return tentacle_class - if tentacle_class := tentacles_management.get_class_from_string( + if tentacle_class := _get_tentacle_class_from_module( tentacle_name, evaluators.TAEvaluator, tentacles_Evaluator.TA, tentacles_management.evaluator_parent_inspection): return tentacle_class - if tentacle_class := tentacles_management.get_class_from_string( + if tentacle_class := _get_tentacle_class_from_module( tentacle_name, evaluators.SocialEvaluator, tentacles_Evaluator.Social, tentacles_management.evaluator_parent_inspection): return tentacle_class - if tentacle_class := tentacles_management.get_class_from_string( + if tentacle_class := _get_tentacle_class_from_module( tentacle_name, evaluators.RealTimeEvaluator, tentacles_Evaluator.RealTime, tentacles_management.evaluator_parent_inspection): return tentacle_class - if tentacle_class := tentacles_management.get_class_from_string( + if tentacle_class := _get_tentacle_class_from_module( tentacle_name, evaluators.ScriptedEvaluator, tentacles_Evaluator.Scripted, tentacles_management.evaluator_parent_inspection): return tentacle_class import octobot_trading.modes as trading_modes import tentacles.Trading as tentacles_trading - if tentacle_class := tentacles_management.get_class_from_string( + if tentacle_class := _get_tentacle_class_from_module( tentacle_name, trading_modes.AbstractTradingMode, tentacles_trading.Mode, tentacles_management.trading_mode_parent_inspection): return tentacle_class import octobot_trading.exchanges as trading_exchanges - if tentacle_class := tentacles_management.get_class_from_string( + if tentacle_class := _get_tentacle_class_from_module( tentacle_name, trading_exchanges.AbstractExchange, tentacles_trading.Exchange, tentacles_management.default_parents_inspection): return tentacle_class try: import octobot.automation as automation import tentacles.Automation - if tentacle_class := tentacles_management.get_class_from_string( + if tentacle_class := _get_tentacle_class_from_module( tentacle_name, automation.Automation, tentacles.Automation, tentacles_management.default_parents_inspection): return tentacle_class @@ -140,16 +152,16 @@ def _load_tentacle_class(tentacle_name): try: import octobot_services.services as services import tentacles.Services as tentacles_services - if tentacle_class := tentacles_management.get_class_from_string( + if tentacle_class := _get_tentacle_class_from_module( tentacle_name, services.AbstractAIService, tentacles_services.Services_bases, tentacles_management.default_parents_inspection): return tentacle_class - if tentacle_class := tentacles_management.get_class_from_string( + if tentacle_class := _get_tentacle_class_from_module( tentacle_name, services.AbstractWebSearchService, tentacles_services.Services_bases, tentacles_management.default_parents_inspection): return tentacle_class import tentacles.Services.Interfaces.web_interface.plugins as web_plugins - if tentacle_class := tentacles_management.get_class_from_string( + if tentacle_class := _get_tentacle_class_from_module( tentacle_name, web_plugins.AbstractWebInterfacePlugin, tentacles_services.Interfaces, tentacles_management.default_parents_inspection): return tentacle_class diff --git a/packages/tentacles_manager/octobot_tentacles_manager/configuration/__init__.py b/packages/tentacles_manager/octobot_tentacles_manager/configuration/__init__.py index 3c1793a59d..28c093f952 100644 --- a/packages/tentacles_manager/octobot_tentacles_manager/configuration/__init__.py +++ b/packages/tentacles_manager/octobot_tentacles_manager/configuration/__init__.py @@ -23,7 +23,6 @@ ) from octobot_tentacles_manager.configuration.tentacle_configuration import ( get_config, - set_get_config_proxy, local_get_config_proxy, update_config, factory_reset_config, @@ -39,7 +38,6 @@ __all__ = [ "TentaclesSetupConfiguration", - "set_get_config_proxy", "local_get_config_proxy", "get_config", "update_config", diff --git a/packages/tentacles_manager/octobot_tentacles_manager/configuration/profile_tentacles_util.py b/packages/tentacles_manager/octobot_tentacles_manager/configuration/profile_tentacles_util.py new file mode 100644 index 0000000000..cf801cdef8 --- /dev/null +++ b/packages/tentacles_manager/octobot_tentacles_manager/configuration/profile_tentacles_util.py @@ -0,0 +1,191 @@ +# Drakkar-Software OctoBot-Tentacles-Manager +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import os +import typing + +import octobot_commons.constants as commons_constants +import octobot_commons.json_util as json_util +import octobot_commons.profiles.profile_types.profile as profile_module +import octobot_commons.profiles.profile_data as profile_data_module + +import octobot_tentacles_manager.api as tentacles_manager_api +import octobot_tentacles_manager.constants as tentacles_manager_constants +import octobot_tentacles_manager.configuration.tentacle_configuration as tentacle_configuration + + +def build_setup_config_from_profile_data( + profile_data: profile_data_module.ProfileData, + output_path: typing.Optional[str] = None, + import_registered_tentacles: bool = False, +): + tentacle_classes = [] + for tentacle_data in profile_data.tentacles: + tentacle_name = tentacle_data.name + if not tentacle_data.activated: + continue + if ( + not tentacle_name + or tentacle_name + in tentacles_manager_constants.IGNORED_TENTACLES_NAMES_IN_TENTACLES_SETUP_CONFIG + ): + continue + try: + tentacle_class = tentacles_manager_api.get_tentacle_class_from_string( + tentacle_name + ) + except RuntimeError: + continue + tentacle_classes.append(tentacle_class.__name__) + config_path = None + if output_path is not None: + config_path = os.path.join(output_path, commons_constants.CONFIG_TENTACLES_FILE) + tentacles_setup_config = ( + tentacles_manager_api.create_tentacles_setup_config_with_tentacles( + *tentacle_classes, config_path=config_path + ) + ) + use_reference_registered_tentacles = not tentacles_setup_config.registered_tentacles + tentacles_manager_api.fill_with_installed_tentacles( + tentacles_setup_config, + import_registered_tentacles=import_registered_tentacles, + use_reference_registered_tentacles=use_reference_registered_tentacles, + ) + return tentacles_setup_config + + +def write_specific_configs_to_profile_folder( + profile_data: profile_data_module.ProfileData, + output_path: str, + is_config_update: bool, +) -> bool: + changed = False + specific_config_dir = os.path.join( + output_path, + tentacles_manager_constants.TENTACLES_SPECIFIC_CONFIG_FOLDER, + ) + if not os.path.exists(specific_config_dir): + os.mkdir(specific_config_dir) + for tentacle_config in profile_data.tentacles: + file_path = os.path.join( + specific_config_dir, + f"{tentacle_config.name}{tentacles_manager_constants.CONFIG_EXT}", + ) + if is_config_update and json_util.has_same_content( + file_path, tentacle_config.config + ): + continue + changed = True + json_util.safe_dump( + tentacle_config.config, + file_path, + ) + return changed + + +def load_setup_config_from_profile_path(tentacles_config_path: str): + return tentacles_manager_api.get_tentacles_setup_config(tentacles_config_path) + + +def read_specific_configs_by_tentacle_name(profile_folder_path: str) -> dict[str, dict]: + specific_config_dir = os.path.join( + profile_folder_path, + tentacles_manager_constants.TENTACLES_SPECIFIC_CONFIG_FOLDER, + ) + config_by_tentacle = {} + if not os.path.isdir(specific_config_dir): + return config_by_tentacle + for config_file_name in os.listdir(specific_config_dir): + if not config_file_name.endswith(tentacles_manager_constants.CONFIG_EXT): + continue + tentacle_name = config_file_name[ + : -len(tentacles_manager_constants.CONFIG_EXT) + ] + config_by_tentacle[tentacle_name] = json_util.read_file( + os.path.join(specific_config_dir, config_file_name) + ) + return config_by_tentacle + + +def collect_tentacles_data_from_setup( + tentacles_setup_config, + specific_configs_by_tentacle_name: typing.Optional[dict[str, dict]] = None, +) -> list[profile_data_module.TentaclesData]: + tentacles_data = [] + preloaded_configs = specific_configs_by_tentacle_name or {} + for ( + tentacle_type, + tentacle_names, + ) in tentacles_setup_config.tentacles_activation.items(): + for tentacle_class_name, is_activated in tentacle_names.items(): + if not is_activated: + continue + try: + tentacle_class = tentacles_manager_api.get_tentacle_class_from_string( + tentacle_class_name + ) + except Exception: + continue + tentacle_name = tentacle_class.get_name() + tentacle_config = preloaded_configs.get(tentacle_name) + if tentacle_config is None: + tentacle_config = tentacle_configuration.get_config( + tentacles_setup_config, tentacle_class + ) + tentacles_data.append( + profile_data_module.TentaclesData( + name=tentacle_name, + config=tentacle_config or {}, + activated=True, + ) + ) + return tentacles_data + + +def merge_inactive_tentacles_data_from_profile( + tentacles_data: list[profile_data_module.TentaclesData], + profile_data: profile_data_module.ProfileData, +) -> list[profile_data_module.TentaclesData]: + collected_names = {tentacle_data.name for tentacle_data in tentacles_data} + merged_tentacles_data = list(tentacles_data) + for tentacle_data in profile_data.tentacles: + if tentacle_data.name in collected_names: + continue + merged_tentacles_data.append( + profile_data_module.TentaclesData( + name=tentacle_data.name, + config=tentacle_data.config or {}, + activated=False, + ) + ) + return merged_tentacles_data + + +def collect_tentacles_data_from_filesystem_profile( + profile: profile_module.Profile, +) -> typing.Optional[list[profile_data_module.TentaclesData]]: + tentacles_config_path = profile.get_tentacles_config_path() + if not os.path.isfile(tentacles_config_path): + return None + tentacles_setup_config = load_setup_config_from_profile_path( + tentacles_config_path + ) + specific_configs_by_tentacle_name = read_specific_configs_by_tentacle_name( + profile.path + ) + return collect_tentacles_data_from_setup( + tentacles_setup_config, + specific_configs_by_tentacle_name=specific_configs_by_tentacle_name, + ) diff --git a/packages/tentacles_manager/octobot_tentacles_manager/configuration/tentacle_configuration.py b/packages/tentacles_manager/octobot_tentacles_manager/configuration/tentacle_configuration.py index 91106c9307..1b0bb638b4 100644 --- a/packages/tentacles_manager/octobot_tentacles_manager/configuration/tentacle_configuration.py +++ b/packages/tentacles_manager/octobot_tentacles_manager/configuration/tentacle_configuration.py @@ -13,15 +13,27 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +from __future__ import annotations + import contextlib +import contextvars import os import os.path as path import shutil +import typing +import octobot_commons.profiles.profile_data as profile_data_module import octobot_tentacles_manager.configuration as configuration import octobot_tentacles_manager.constants as constants import octobot_tentacles_manager.loaders as loaders +if typing.TYPE_CHECKING: + import octobot_tentacles_manager.configuration.tentacles_setup_configuration as tentacles_setup_configuration + + +_LOCAL_GET_CONFIG_OVERRIDE: contextvars.ContextVar = contextvars.ContextVar( + "local_get_config_override", default=None +) def _get_config_from_file_system(tentacles_setup_config, klass): @@ -31,30 +43,7 @@ def _get_config_from_file_system(tentacles_setup_config, klass): return configuration.read_config(config_path) -_GET_CONFIG_PROXY = _get_config_from_file_system - - -def set_get_config_proxy(new_proxy): - # todo handle concurrency - global _GET_CONFIG_PROXY - _GET_CONFIG_PROXY = new_proxy - - -@contextlib.contextmanager -def local_get_config_proxy(new_proxy): - previous_proxy = _GET_CONFIG_PROXY - try: - set_get_config_proxy(new_proxy) - yield - finally: - set_get_config_proxy(previous_proxy) - - -def get_config(tentacles_setup_config, klass) -> dict: - return _GET_CONFIG_PROXY(tentacles_setup_config, klass) - - -def update_config(tentacles_setup_config, klass, config_update, keep_existing=True) -> None: +def _update_config_from_file_system(tentacles_setup_config, klass, config_update, keep_existing=True) -> None: config_file = _get_config_file_path(tentacles_setup_config, klass) current_config = configuration.read_config(config_file) # only update values in config update not to erase values in root config (might not be editable) @@ -66,18 +55,129 @@ def update_config(tentacles_setup_config, klass, config_update, keep_existing=Tr config_file = _get_config_file_path(tentacles_setup_config, klass, updated_config=True) configuration.write_config(config_file, current_config) + def _recursive_config_update(current_config: dict, config_update: dict)-> dict: for key, values in config_update.items(): if isinstance(values, dict) and isinstance(current_config.get(key), dict): current_config[key] = _recursive_config_update(current_config[key], values) continue - current_config[key] = values + current_config[key] = values return current_config -def factory_reset_config(tentacles_setup_config, klass) -> None: + +def _factory_reset_config_from_file_system(tentacles_setup_config, klass) -> None: shutil.copy(_get_reference_config_file_path(klass), _get_config_file_path(tentacles_setup_config, klass)) +def _get_config_for_profile(tentacles_setup_config, klass) -> dict: + profile = tentacles_setup_config.profile + file_or_factory_config = _get_config_from_file_system(tentacles_setup_config, klass) + if profile is None or not profile.is_profile_data_tentacle_backed(): + return file_or_factory_config + tentacle_name = klass.get_name() + for tentacle_data in profile.get_profile_data().tentacles: + if tentacle_data.name == tentacle_name: + if tentacle_data.config: + return _recursive_config_update(dict(file_or_factory_config), tentacle_data.config) + return file_or_factory_config + return file_or_factory_config + + +def _update_config_for_profile( + tentacles_setup_config, klass, config_update, keep_existing=True +) -> None: + profile = tentacles_setup_config.profile + if profile is None or not profile.is_profile_data_tentacle_backed(): + _update_config_from_file_system( + tentacles_setup_config, klass, config_update, keep_existing=keep_existing + ) + return + tentacle_name = klass.get_name() + profile_data = profile.get_profile_data() + updated_tentacle = None + for tentacle_data in profile_data.tentacles: + if tentacle_data.name == tentacle_name: + updated_tentacle = tentacle_data + break + if updated_tentacle is None: + updated_tentacle = profile_data_module.TentaclesData( + name=tentacle_name, config={} + ) + profile_data.tentacles.append(updated_tentacle) + import octobot_tentacles_manager.api as tentacles_manager_api + + updated_tentacle.activated = tentacles_manager_api.is_tentacle_activated_in_tentacles_setup_config( + tentacles_setup_config, + tentacle_name, + default_value=False, + ) + if keep_existing: + merged_config = dict(updated_tentacle.config or {}) + merged_config.update(config_update) + updated_tentacle.config = merged_config + else: + updated_tentacle.config = config_update + + +def _factory_reset_config_for_profile(tentacles_setup_config, klass) -> None: + profile = tentacles_setup_config.profile + if profile is None or not profile.is_profile_data_tentacle_backed(): + _factory_reset_config_from_file_system(tentacles_setup_config, klass) + return + tentacle_name = klass.get_name() + profile_data = profile.get_profile_data() + profile_data.tentacles = [ + tentacle_data + for tentacle_data in profile_data.tentacles + if tentacle_data.name != tentacle_name + ] + + +@contextlib.contextmanager +def local_get_config_proxy(new_proxy): + override_token = _LOCAL_GET_CONFIG_OVERRIDE.set(new_proxy) + try: + yield + finally: + _LOCAL_GET_CONFIG_OVERRIDE.reset(override_token) + + +def get_config( + tentacles_setup_config: tentacles_setup_configuration.TentaclesSetupConfiguration, + klass, +) -> dict: + local_override = _LOCAL_GET_CONFIG_OVERRIDE.get() + if local_override is not None: + return local_override(tentacles_setup_config, klass) + if tentacles_setup_config.profile: + return _get_config_for_profile(tentacles_setup_config, klass) + return _get_config_from_file_system(tentacles_setup_config, klass) + + +def update_config( + tentacles_setup_config: tentacles_setup_configuration.TentaclesSetupConfiguration, + klass, + config_update, + keep_existing=True, +) -> None: + if tentacles_setup_config.profile: + return _update_config_for_profile( + tentacles_setup_config, klass, config_update, keep_existing=keep_existing + ) + return _update_config_from_file_system( + tentacles_setup_config, klass, config_update, keep_existing=keep_existing + ) + + +def factory_reset_config( + tentacles_setup_config: tentacles_setup_configuration.TentaclesSetupConfiguration, + klass, +) -> None: + if tentacles_setup_config.profile: + return _factory_reset_config_for_profile(tentacles_setup_config, klass) + return _factory_reset_config_from_file_system(tentacles_setup_config, klass) + + def get_config_schema_path(klass) -> str: return path.join(_get_reference_config_path(klass), f"{klass.get_name()}{constants.CONFIG_SCHEMA_EXT}") diff --git a/packages/tentacles_manager/octobot_tentacles_manager/configuration/tentacles_setup_configuration.py b/packages/tentacles_manager/octobot_tentacles_manager/configuration/tentacles_setup_configuration.py index a04d4c3646..8f1459eda9 100644 --- a/packages/tentacles_manager/octobot_tentacles_manager/configuration/tentacles_setup_configuration.py +++ b/packages/tentacles_manager/octobot_tentacles_manager/configuration/tentacles_setup_configuration.py @@ -18,8 +18,9 @@ import octobot_commons.logging as logging import octobot_commons.constants as commons_constants +import octobot_commons.errors as commons_errors import octobot_commons.user_root_folder_provider as user_root_folder_provider -import octobot_commons.profiles as commons_profiles +import octobot_commons.profiles.backends as profile_backends_module import octobot_tentacles_manager.constants as constants import octobot_tentacles_manager.configuration as configuration @@ -46,6 +47,7 @@ def __init__(self, bot_installation_path=constants.DEFAULT_BOT_PATH, config_path if config_path is None: config_path = user_root_folder_provider.get_user_reference_tentacle_config_file_path() self.config_path = path.join(bot_installation_path, config_path) + self.profile = None self.tentacles_activation = {} self.registered_tentacles = {} self.installation_context = {} @@ -94,8 +96,10 @@ def get_associated_profile_path(self): @staticmethod def is_imported_profile(profile_folder): try: - return commons_profiles.Profile(profile_folder).read_config().imported - except OSError: + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + profile = filesystem_backend.read_profile_from_path(profile_folder) + return profile.imported + except (OSError, commons_errors.ProfileDataError): return False def refresh_profiles_tentacles_config(self, @@ -208,9 +212,10 @@ def _deactivate_tentacle_if_evaluator(self, element_name, element_type): return False def _apply_default_profile_activation(self): - default_profile = commons_profiles.Profile.load_profile( + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + default_profile = filesystem_backend.load_profile( user_root_folder_provider.get_user_profiles_folder(), - commons_constants.DEFAULT_PROFILE + commons_constants.DEFAULT_PROFILE, ) profile_setup_config = configuration.TentaclesSetupConfiguration( config_path=default_profile.get_tentacles_config_path() diff --git a/packages/tentacles_manager/tests/api/test_inspector.py b/packages/tentacles_manager/tests/api/test_inspector.py new file mode 100644 index 0000000000..6c3ec81d69 --- /dev/null +++ b/packages/tentacles_manager/tests/api/test_inspector.py @@ -0,0 +1,38 @@ +import types + +import octobot_commons.tentacles_management as tentacles_management +import octobot_tentacles_manager.api.inspector as inspector_module + + +class _ParentTentacle: + pass + + +class _BingxTentacle(_ParentTentacle): + pass + + +def _fake_tentacle_module(): + fake_module = types.ModuleType("fake_tentacle_module") + fake_module.Bingx = _BingxTentacle + return fake_module + + +class TestGetTentacleClassFromModule: + def test_resolves_tentacle_name_case_insensitively(self): + tentacle_class = inspector_module._get_tentacle_class_from_module( + "bingx", + _ParentTentacle, + _fake_tentacle_module(), + tentacles_management.default_parents_inspection, + ) + assert tentacle_class is _BingxTentacle + + def test_resolves_tentacle_class_name(self): + tentacle_class = inspector_module._get_tentacle_class_from_module( + "Bingx", + _ParentTentacle, + _fake_tentacle_module(), + tentacles_management.default_parents_inspection, + ) + assert tentacle_class is _BingxTentacle diff --git a/packages/tentacles_manager/tests/configuration/test_profile_tentacles_util.py b/packages/tentacles_manager/tests/configuration/test_profile_tentacles_util.py new file mode 100644 index 0000000000..6fd40a5b30 --- /dev/null +++ b/packages/tentacles_manager/tests/configuration/test_profile_tentacles_util.py @@ -0,0 +1,358 @@ +# Drakkar-Software OctoBot-Tentacles-Manager +# Copyright (c) Drakkar-Software, All rights reserved. + +import json +import os + +import mock + +import octobot_commons.constants as commons_constants +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_tentacles_manager.constants as tentacles_manager_constants + + +class TestBuildSetupConfigFromProfileData: + def test_builds_setup_config_from_profile_data_tentacles(self): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData(name="MyEvaluator", config={}), + ] + tentacle_class = mock.Mock() + tentacle_class.__name__ = "MyEvaluator" + setup_config = mock.Mock() + setup_config.registered_tentacles = {"pkg": "url"} + + with mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "get_tentacle_class_from_string", + mock.Mock(return_value=tentacle_class), + ), mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "create_tentacles_setup_config_with_tentacles", + mock.Mock(return_value=setup_config), + ) as create_setup_mock, mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "fill_with_installed_tentacles", + ) as fill_mock: + result = profile_tentacles_util.build_setup_config_from_profile_data( + profile_data, "/output", import_registered_tentacles=True + ) + + assert result is setup_config + create_setup_mock.assert_called_once_with( + "MyEvaluator", + config_path=os.path.join("/output", commons_constants.CONFIG_TENTACLES_FILE), + ) + fill_mock.assert_called_once_with( + setup_config, + import_registered_tentacles=True, + use_reference_registered_tentacles=False, + ) + + def test_builds_setup_config_without_output_path(self): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData(name="MyEvaluator", config={}), + ] + tentacle_class = mock.Mock() + tentacle_class.__name__ = "MyEvaluator" + setup_config = mock.Mock() + setup_config.registered_tentacles = {"pkg": "url"} + + with mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "get_tentacle_class_from_string", + mock.Mock(return_value=tentacle_class), + ), mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "create_tentacles_setup_config_with_tentacles", + mock.Mock(return_value=setup_config), + ) as create_setup_mock, mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "fill_with_installed_tentacles", + ): + result = profile_tentacles_util.build_setup_config_from_profile_data( + profile_data, import_registered_tentacles=False + ) + + assert result is setup_config + create_setup_mock.assert_called_once_with("MyEvaluator", config_path=None) + + def test_skips_missing_tentacle_names(self): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData(name="MyEvaluator", config={}), + profile_data_module.TentaclesData(name="upbit", config={}), + ] + my_evaluator_class = mock.Mock() + my_evaluator_class.__name__ = "MyEvaluator" + setup_config = mock.Mock() + setup_config.registered_tentacles = {"pkg": "url"} + + def get_tentacle_class_side_effect(tentacle_name): + if tentacle_name == "MyEvaluator": + return my_evaluator_class + raise RuntimeError(f"Can't find tentacle: {tentacle_name}") + + with mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "get_tentacle_class_from_string", + mock.Mock(side_effect=get_tentacle_class_side_effect), + ), mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "create_tentacles_setup_config_with_tentacles", + mock.Mock(return_value=setup_config), + ) as create_setup_mock, mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "fill_with_installed_tentacles", + ): + result = profile_tentacles_util.build_setup_config_from_profile_data( + profile_data, import_registered_tentacles=False + ) + + assert result is setup_config + create_setup_mock.assert_called_once_with("MyEvaluator", config_path=None) + + def test_all_missing_still_returns_setup_config(self): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData(name="upbit", config={}), + ] + setup_config = mock.Mock() + setup_config.registered_tentacles = {"pkg": "url"} + + with mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "get_tentacle_class_from_string", + mock.Mock(side_effect=RuntimeError("Can't find tentacle: upbit")), + ), mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "create_tentacles_setup_config_with_tentacles", + mock.Mock(return_value=setup_config), + ) as create_setup_mock, mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "fill_with_installed_tentacles", + ) as fill_mock: + result = profile_tentacles_util.build_setup_config_from_profile_data( + profile_data, import_registered_tentacles=False + ) + + assert result is setup_config + create_setup_mock.assert_called_once_with(config_path=None) + fill_mock.assert_called_once_with( + setup_config, + import_registered_tentacles=False, + use_reference_registered_tentacles=False, + ) + + +class TestWriteSpecificConfigsToProfileFolder: + def test_skips_unchanged_configs_when_updating(self, tmp_path): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData(name="MyEvaluator", config={"a": 1}), + ] + specific_config_dir = os.path.join( + tmp_path, tentacles_manager_constants.TENTACLES_SPECIFIC_CONFIG_FOLDER + ) + os.makedirs(specific_config_dir) + file_path = os.path.join( + specific_config_dir, + f"MyEvaluator{tentacles_manager_constants.CONFIG_EXT}", + ) + with open(file_path, "w", encoding="utf-8") as config_file: + json.dump({"a": 1}, config_file) + + changed = profile_tentacles_util.write_specific_configs_to_profile_folder( + profile_data, str(tmp_path), is_config_update=True + ) + + assert changed is False + + def test_writes_new_specific_configs(self, tmp_path): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData(name="MyEvaluator", config={"enabled": True}), + ] + + changed = profile_tentacles_util.write_specific_configs_to_profile_folder( + profile_data, str(tmp_path), is_config_update=False + ) + + assert changed is True + file_path = os.path.join( + tmp_path, + tentacles_manager_constants.TENTACLES_SPECIFIC_CONFIG_FOLDER, + f"MyEvaluator{tentacles_manager_constants.CONFIG_EXT}", + ) + assert os.path.isfile(file_path) + with open(file_path, encoding="utf-8") as config_file: + assert json.load(config_file) == {"enabled": True} + + +class TestLoadSetupConfigFromProfilePath: + def test_delegates_to_api(self): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + setup_config = mock.Mock() + with mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "get_tentacles_setup_config", + mock.Mock(return_value=setup_config), + ) as get_setup_mock: + result = profile_tentacles_util.load_setup_config_from_profile_path( + "/profiles/default/tentacles_config.json" + ) + + assert result is setup_config + get_setup_mock.assert_called_once_with( + "/profiles/default/tentacles_config.json" + ) + + +class TestReadSpecificConfigsByTentacleName: + def test_reads_specific_config_json_files(self, tmp_path): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + specific_config_dir = os.path.join( + tmp_path, tentacles_manager_constants.TENTACLES_SPECIFIC_CONFIG_FOLDER + ) + os.makedirs(specific_config_dir) + file_path = os.path.join( + specific_config_dir, + f"MyEvaluator{tentacles_manager_constants.CONFIG_EXT}", + ) + with open(file_path, "w", encoding="utf-8") as config_file: + json.dump({"key": "value"}, config_file) + + configs = profile_tentacles_util.read_specific_configs_by_tentacle_name( + str(tmp_path) + ) + + assert configs == {"MyEvaluator": {"key": "value"}} + + +class TestCollectTentaclesDataFromSetup: + def test_uses_preloaded_specific_config_before_get_config(self): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + tentacles_setup_config = mock.Mock() + tentacles_setup_config.tentacles_activation = { + "evaluators": {"MyEvaluator": True}, + } + tentacle_class = mock.Mock() + tentacle_class.get_name.return_value = "MyEvaluator" + + with mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "get_tentacle_class_from_string", + mock.Mock(return_value=tentacle_class), + ), mock.patch.object( + profile_tentacles_util.tentacle_configuration, + "get_config", + mock.Mock(side_effect=AssertionError("get_config should not be called")), + ): + tentacles_data = profile_tentacles_util.collect_tentacles_data_from_setup( + tentacles_setup_config, + specific_configs_by_tentacle_name={"MyEvaluator": {"from_file": True}}, + ) + + assert len(tentacles_data) == 1 + assert tentacles_data[0].name == "MyEvaluator" + assert tentacles_data[0].config == {"from_file": True} + assert tentacles_data[0].activated is True + + +class TestBuildSetupConfigFromProfileDataInactiveTentacles: + def test_skips_inactive_tentacles_when_building_setup(self): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData(name="IndexTradingMode", config={}, activated=True), + profile_data_module.TentaclesData( + name="GridTradingMode", + config={"flat_spread": 2}, + activated=False, + ), + ] + index_class = mock.Mock() + index_class.__name__ = "IndexTradingMode" + setup_config = mock.Mock() + setup_config.registered_tentacles = {"pkg": "url"} + + with mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "get_tentacle_class_from_string", + mock.Mock(return_value=index_class), + ), mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "create_tentacles_setup_config_with_tentacles", + mock.Mock(return_value=setup_config), + ) as create_setup_mock, mock.patch.object( + profile_tentacles_util.tentacles_manager_api, + "fill_with_installed_tentacles", + ): + profile_tentacles_util.build_setup_config_from_profile_data( + profile_data, import_registered_tentacles=False + ) + + create_setup_mock.assert_called_once_with("IndexTradingMode", config_path=None) + + +class TestMergeInactiveTentaclesDataFromProfile: + def test_preserves_inactive_tentacle_configs(self): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData( + name="GridTradingMode", + config={"flat_spread": 2}, + activated=False, + ), + ] + collected = [ + profile_data_module.TentaclesData( + name="IndexTradingMode", + config={"refresh_interval": 0}, + activated=True, + ), + ] + + merged = profile_tentacles_util.merge_inactive_tentacles_data_from_profile( + collected, profile_data + ) + + assert len(merged) == 2 + assert merged[0].name == "IndexTradingMode" + assert merged[0].activated is True + assert merged[1].name == "GridTradingMode" + assert merged[1].activated is False + assert merged[1].config == {"flat_spread": 2} + + +class TestCollectTentaclesDataFromFilesystemProfile: + def test_returns_none_when_tentacles_config_file_missing(self, tmp_path): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + import octobot_commons.profiles.profile_types.profile as profile_module + + profile = profile_module.Profile(str(tmp_path)) + + result = profile_tentacles_util.collect_tentacles_data_from_filesystem_profile( + profile + ) + + assert result is None diff --git a/packages/tentacles_manager/tests/configuration/test_tentacle_configuration.py b/packages/tentacles_manager/tests/configuration/test_tentacle_configuration.py index 6c7f6162aa..b93f65f0e4 100644 --- a/packages/tentacles_manager/tests/configuration/test_tentacle_configuration.py +++ b/packages/tentacles_manager/tests/configuration/test_tentacle_configuration.py @@ -290,3 +290,4 @@ def _cleanup(): ref_tent = user_root_folder_provider.get_user_reference_tentacle_config_path() if path.exists(ref_tent): rmtree(ref_tent) + diff --git a/packages/tentacles_manager/tests/configuration/test_tentacle_configuration_profile_dispatch.py b/packages/tentacles_manager/tests/configuration/test_tentacle_configuration_profile_dispatch.py new file mode 100644 index 0000000000..e02f0bdec5 --- /dev/null +++ b/packages/tentacles_manager/tests/configuration/test_tentacle_configuration_profile_dispatch.py @@ -0,0 +1,261 @@ +# Drakkar-Software OctoBot-Tentacles-Manager +# Copyright (c) Drakkar-Software, All rights reserved. + + +class TestTentacleConfigurationProfileDispatch: + def test_local_get_config_proxy_overrides_get_config(self): + import octobot_tentacles_manager.configuration.tentacle_configuration as tentacle_configuration + import octobot_tentacles_manager.configuration.tentacles_setup_configuration as tentacles_setup_configuration + + called = {} + setup_config = tentacles_setup_configuration.TentaclesSetupConfiguration() + + def custom_get(tentacles_setup_config, klass): + called["klass"] = klass + return {"custom": True} + + with tentacle_configuration.local_get_config_proxy(custom_get): + result = tentacle_configuration.get_config(setup_config, "TestKlass") + assert result == {"custom": True} + assert called["klass"] == "TestKlass" + + def test_sync_backed_profile_reads_config_from_profile_data(self): + import mock + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_tentacles_manager.configuration.tentacle_configuration as tentacle_configuration + import octobot_tentacles_manager.configuration.tentacles_setup_configuration as tentacles_setup_configuration + + tentacle_klass = mock.Mock() + tentacle_klass.get_name.return_value = "TestKlass" + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData( + name="TestKlass", config={"from_profile_data": True} + ) + ] + profile = mock.Mock() + profile.is_profile_data_tentacle_backed.return_value = True + profile.get_profile_data.return_value = profile_data + setup = tentacles_setup_configuration.TentaclesSetupConfiguration() + setup.profile = profile + factory_config = {"required_strategies": []} + with mock.patch.object( + tentacle_configuration, + "_get_config_from_file_system", + mock.Mock(return_value=factory_config), + ): + result = tentacle_configuration.get_config(setup, tentacle_klass) + assert result == {"required_strategies": [], "from_profile_data": True} + + def test_filesystem_profile_falls_back_to_file_system(self): + import mock + import octobot_tentacles_manager.configuration.tentacle_configuration as tentacle_configuration + import octobot_tentacles_manager.configuration.tentacles_setup_configuration as tentacles_setup_configuration + + profile = mock.Mock() + profile.is_profile_data_tentacle_backed.return_value = False + setup = tentacles_setup_configuration.TentaclesSetupConfiguration() + setup.profile = profile + with mock.patch.object( + tentacle_configuration, + "_get_config_from_file_system", + mock.Mock(return_value={"from_filesystem": True}), + ) as get_from_filesystem_mock: + result = tentacle_configuration.get_config(setup, "TestKlass") + assert result == {"from_filesystem": True} + get_from_filesystem_mock.assert_called_once_with(setup, "TestKlass") + + def test_sync_backed_update_config_does_not_write_to_filesystem(self): + import mock + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_tentacles_manager.configuration.tentacle_configuration as tentacle_configuration + import octobot_tentacles_manager.configuration.tentacles_setup_configuration as tentacles_setup_configuration + + tentacle_klass = mock.Mock() + tentacle_klass.get_name.return_value = "TestKlass" + profile_data = profile_data_module.ProfileData() + profile = mock.Mock() + profile.is_profile_data_tentacle_backed.return_value = True + profile.get_profile_data.return_value = profile_data + setup = tentacles_setup_configuration.TentaclesSetupConfiguration() + setup.profile = profile + with mock.patch.object( + tentacle_configuration, + "_update_config_from_file_system", + mock.Mock(), + ) as update_filesystem_mock: + tentacle_configuration.update_config( + setup, tentacle_klass, {"updated": True} + ) + update_filesystem_mock.assert_not_called() + assert profile_data.tentacles[0].name == "TestKlass" + assert profile_data.tentacles[0].config == {"updated": True} + + def test_sync_backed_update_config_sets_activated_from_setup(self): + import mock + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_tentacles_manager.configuration.tentacle_configuration as tentacle_configuration + import octobot_tentacles_manager.configuration.tentacles_setup_configuration as tentacles_setup_configuration + + tentacle_klass = mock.Mock() + tentacle_klass.get_name.return_value = "GridTradingMode" + profile_data = profile_data_module.ProfileData() + profile = mock.Mock() + profile.is_profile_data_tentacle_backed.return_value = True + profile.get_profile_data.return_value = profile_data + setup = tentacles_setup_configuration.TentaclesSetupConfiguration() + setup.profile = profile + with mock.patch( + "octobot_tentacles_manager.api.is_tentacle_activated_in_tentacles_setup_config", + mock.Mock(return_value=False), + ): + tentacle_configuration.update_config( + setup, tentacle_klass, {"flat_spread": 2} + ) + assert profile_data.tentacles[0].name == "GridTradingMode" + assert profile_data.tentacles[0].config == {"flat_spread": 2} + assert profile_data.tentacles[0].activated is False + + def test_ephemeral_profile_reads_config_from_profile_data(self): + import octobot_commons.profiles.profile_types.ephemeral_profile as ephemeral_profile_module + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_tentacles_manager.configuration.tentacle_configuration as tentacle_configuration + import octobot_tentacles_manager.configuration.tentacles_setup_configuration as tentacles_setup_configuration + + tentacle_klass = type("TestKlass", (), {"get_name": staticmethod(lambda: "TestKlass")})() + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData( + name="TestKlass", config={"from_ephemeral": True} + ) + ] + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + setup = tentacles_setup_configuration.TentaclesSetupConfiguration() + setup.profile = profile + result = tentacle_configuration.get_config(setup, tentacle_klass) + assert result == {"from_ephemeral": True} + + def test_ephemeral_profile_update_config_does_not_write_to_filesystem(self): + import mock + import octobot_commons.profiles.profile_types.ephemeral_profile as ephemeral_profile_module + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_tentacles_manager.configuration.tentacle_configuration as tentacle_configuration + import octobot_tentacles_manager.configuration.tentacles_setup_configuration as tentacles_setup_configuration + + tentacle_klass = mock.Mock() + tentacle_klass.get_name.return_value = "TestKlass" + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + setup = tentacles_setup_configuration.TentaclesSetupConfiguration() + setup.profile = profile + with mock.patch.object( + tentacle_configuration, + "_update_config_from_file_system", + mock.Mock(), + ) as update_filesystem_mock: + tentacle_configuration.update_config( + setup, tentacle_klass, {"updated": True} + ) + update_filesystem_mock.assert_not_called() + assert profile_data.tentacles[0].name == "TestKlass" + assert profile_data.tentacles[0].config == {"updated": True} + + def test_profile_not_persisted_in_setup_config_dict(self): + import mock + import octobot_tentacles_manager.configuration.tentacles_setup_configuration as tentacles_setup_configuration + + setup = tentacles_setup_configuration.TentaclesSetupConfiguration() + setup.profile = mock.Mock() + setup.tentacles_activation = {"t": {"c": True}} + persisted = setup._to_dict() + assert "profile" not in persisted + assert setup.profile is not None + + +class TestSyncBackedProfileConfigFilesystemFallback: + def test_sync_backed_empty_config_falls_back_to_filesystem(self): + import mock + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_tentacles_manager.configuration.tentacle_configuration as tentacle_configuration + import octobot_tentacles_manager.configuration.tentacles_setup_configuration as tentacles_setup_configuration + + tentacle_klass = mock.Mock() + tentacle_klass.get_name.return_value = "TestKlass" + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData(name="TestKlass", config={}), + ] + profile = mock.Mock() + profile.is_profile_data_tentacle_backed.return_value = True + profile.get_profile_data.return_value = profile_data + setup = tentacles_setup_configuration.TentaclesSetupConfiguration() + setup.profile = profile + factory_config = {"required_evaluators": ["*"]} + with mock.patch.object( + tentacle_configuration, + "_get_config_from_file_system", + mock.Mock(return_value=factory_config), + ) as get_from_filesystem_mock: + result = tentacle_configuration.get_config(setup, tentacle_klass) + assert result == factory_config + get_from_filesystem_mock.assert_called_once_with(setup, tentacle_klass) + + def test_sync_backed_partial_inactive_config_merges_factory_defaults(self): + import mock + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_tentacles_manager.configuration.tentacle_configuration as tentacle_configuration + import octobot_tentacles_manager.configuration.tentacles_setup_configuration as tentacles_setup_configuration + + tentacle_klass = mock.Mock() + tentacle_klass.get_name.return_value = "GridTradingMode" + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData( + name="GridTradingMode", + config={"flat_spread": 2}, + activated=False, + ), + ] + profile = mock.Mock() + profile.is_profile_data_tentacle_backed.return_value = True + profile.get_profile_data.return_value = profile_data + setup = tentacles_setup_configuration.TentaclesSetupConfiguration() + setup.profile = profile + factory_config = {"required_strategies": [], "flat_spread": 1} + with mock.patch.object( + tentacle_configuration, + "_get_config_from_file_system", + mock.Mock(return_value=factory_config), + ): + result = tentacle_configuration.get_config(setup, tentacle_klass) + assert result["required_strategies"] == [] + assert result["flat_spread"] == 2 + + def test_sync_backed_missing_tentacle_falls_back_to_filesystem(self): + import mock + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_tentacles_manager.configuration.tentacle_configuration as tentacle_configuration + import octobot_tentacles_manager.configuration.tentacles_setup_configuration as tentacles_setup_configuration + + tentacle_klass = mock.Mock() + tentacle_klass.get_name.return_value = "OtherKlass" + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData( + name="TestKlass", config={"from_profile_data": True} + ), + ] + profile = mock.Mock() + profile.is_profile_data_tentacle_backed.return_value = True + profile.get_profile_data.return_value = profile_data + setup = tentacles_setup_configuration.TentaclesSetupConfiguration() + setup.profile = profile + factory_config = {"required_evaluators": ["TA"]} + with mock.patch.object( + tentacle_configuration, + "_get_config_from_file_system", + mock.Mock(return_value=factory_config), + ) as get_from_filesystem_mock: + result = tentacle_configuration.get_config(setup, tentacle_klass) + assert result == factory_config + get_from_filesystem_mock.assert_called_once_with(setup, tentacle_klass) diff --git a/packages/trading/octobot_trading/util/config_util.py b/packages/trading/octobot_trading/util/config_util.py index 7427e38655..a22ea15ead 100644 --- a/packages/trading/octobot_trading/util/config_util.py +++ b/packages/trading/octobot_trading/util/config_util.py @@ -224,7 +224,7 @@ def get_config( ) # do not allow using backtesting context when using exchange data portfolio profile_data.backtesting_context = None # type: ignore - profile = profile_data.to_profile(None) + profile = commons_profiles.EphemeralProfile.from_profile_data(profile_data) profile_data.backtesting_context = initial_backtesting_context config.profile_by_id[profile.profile_id] = profile config.select_profile(profile.profile_id) diff --git a/tests/unit_tests/community/test_authentication.py b/tests/unit_tests/community/test_authentication.py index 7dc1ca04c7..3f6fe4291b 100644 --- a/tests/unit_tests/community/test_authentication.py +++ b/tests/unit_tests/community/test_authentication.py @@ -516,3 +516,31 @@ def test_is_node_wallet_configured(auth): auth._wallet_backend.list_wallets.return_value = [mock.Mock()] assert auth.is_node_wallet_configured() is True + +class TestGetWalletSyncStorageMasterConfigRead: + def test_reads_master_config_without_activating_profile(self): + auth = community.CommunityAuthentication.__new__(community.CommunityAuthentication) + master_config = mock.Mock() + sync_storage = mock.Mock() + with mock.patch( + "octobot.community.authentication.user_root_folder_provider.get_sync_data_root", + mock.Mock(return_value="/master/user"), + ), mock.patch( + "octobot.community.authentication.user_root_folder_provider.get_user_root_folder", + mock.Mock(return_value="/child/automation"), + ), mock.patch( + "octobot.community.authentication.os.path.isfile", + mock.Mock(return_value=True), + ), mock.patch( + "octobot.community.authentication.commons_configuration.Configuration", + mock.Mock(return_value=master_config), + ) as configuration_cls_mock, mock.patch( + "octobot.community.authentication.supabase_backend.SyncConfigurationStorage", + mock.Mock(return_value=sync_storage), + ) as sync_storage_cls_mock: + result = auth._get_wallet_sync_storage() + configuration_cls_mock.assert_called_once() + master_config.read.assert_called_once_with(should_raise=False, activate_profile=False) + sync_storage_cls_mock.assert_called_once_with(master_config) + assert result is sync_storage + diff --git a/tests/unit_tests/storage/test_process_bot_state_dumper.py b/tests/unit_tests/storage/test_process_bot_state_dumper.py new file mode 100644 index 0000000000..b5511bc1b5 --- /dev/null +++ b/tests/unit_tests/storage/test_process_bot_state_dumper.py @@ -0,0 +1,46 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import decimal +import json + +import mock +import pytest + +import octobot.storage.process_bot_state_dumper as process_bot_state_dumper_import +import octobot_flow.entities.accounts.exchange_account_elements as exchange_account_elements_import +import octobot_trading.exchanges as trading_exchanges + + +class TestWriteStateFileAsync: + @pytest.mark.asyncio + async def test_writes_json_when_orders_contain_decimal(self, tmp_path): + elements = exchange_account_elements_import.ExchangeAccountElements( + name="binance", + orders=trading_exchanges.OrdersDetails( + open_orders=[ + { + "price": decimal.Decimal("1.0001"), + "quantity": decimal.Decimal("50"), + "symbol": "USDC/USDT", + } + ], + ), + ) + state_file = tmp_path / "bot_state.json" + with mock.patch.object( + process_bot_state_dumper_import, + "_synced_exchange_account_elements_for_first_trading_exchange", + mock.Mock(return_value=elements), + ): + await process_bot_state_dumper_import._write_state_file_async( + str(state_file), + 30.0, + mock.Mock(), + ) + parsed = json.loads(state_file.read_text(encoding="utf-8")) + order = parsed["exchange_account_elements"]["orders"]["open_orders"][0] + assert order["price"] == 1.0001 + assert order["quantity"] == 50.0 + assert isinstance(order["price"], float) + assert isinstance(order["quantity"], float) diff --git a/tests/unit_tests/test_cli_process_child_sync_user.py b/tests/unit_tests/test_cli_process_child_sync_user.py new file mode 100644 index 0000000000..c3cbbac893 --- /dev/null +++ b/tests/unit_tests/test_cli_process_child_sync_user.py @@ -0,0 +1,133 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import mock +import pytest + +import octobot.constants as octobot_constants +import octobot_commons.errors as commons_errors +import octobot.cli as octobot_cli + + +class TestConfigureProfileSyncUserProcessChild: + def test_raises_when_process_child_env_missing(self, monkeypatch): + monkeypatch.setattr(octobot_constants, "PROCESS_BOT_SYNC_USER_ID", "") + config = mock.Mock() + with pytest.raises(commons_errors.ConfigError, match=octobot_constants.ENV_PROCESS_BOT_SYNC_USER_ID): + octobot_cli._configure_profile_sync_user( + config, + None, + is_process_child=True, + ) + config.profile_storage.configure_sync_user.assert_not_called() + config.profile_storage.bind_process_child_sync_user_id.assert_not_called() + + def test_configures_sync_user_from_env_for_process_child(self, monkeypatch): + monkeypatch.setattr(octobot_constants, "PROCESS_BOT_SYNC_USER_ID", "child-user") + config = mock.Mock() + octobot_cli._configure_profile_sync_user( + config, + None, + is_process_child=True, + ) + config.profile_storage.configure_sync_user.assert_called_once_with("child-user") + config.profile_storage.bind_process_child_sync_user_id.assert_not_called() + + def test_raises_when_parent_wallet_missing_for_process_child(self, monkeypatch): + monkeypatch.setattr(octobot_constants, "PROCESS_BOT_SYNC_USER_ID", "child-user") + config = mock.Mock() + config.profile_storage.configure_sync_user.side_effect = commons_errors.ProfileDataError( + "Unknown sync user id: child-user" + ) + with pytest.raises(commons_errors.ProfileDataError, match="Unknown sync user id"): + octobot_cli._configure_profile_sync_user( + config, + None, + is_process_child=True, + ) + + +class TestActivateSavedProfileAfterSync: + def test_activate_saved_profile_after_sync(self): + config = mock.Mock() + logger = mock.Mock() + with mock.patch.object(octobot_cli.commands, "ensure_profile", mock.Mock()) as ensure_profile_mock, \ + mock.patch.object(octobot_cli, "_validate_config", mock.Mock()) as validate_config_mock: + octobot_cli._activate_saved_profile_after_sync(config, logger) + config.activate_saved_profile.assert_called_once_with() + ensure_profile_mock.assert_called_once_with(config) + validate_config_mock.assert_called_once_with(config, logger) + + +class TestCreateStartupConfigDeferredProfileActivation: + @pytest.mark.parametrize("is_process_child", [False, True]) + def test_defers_profile_activation_until_after_sync_configure(self, is_process_child): + logger = mock.Mock() + config = mock.Mock() + config.is_config_file_empty_or_missing.return_value = False + with mock.patch.object(octobot_cli, "_create_configuration", mock.Mock(return_value=config)), \ + mock.patch.object(octobot_cli, "_read_config", mock.Mock()) as read_config_mock, \ + mock.patch.object(octobot_cli.commands, "ensure_profile", mock.Mock()) as ensure_profile_mock, \ + mock.patch.object(octobot_cli, "_validate_config", mock.Mock()) as validate_config_mock, \ + mock.patch.object( + octobot_cli.configuration_manager, + "get_distribution", + mock.Mock(return_value=octobot_cli.enums.OctoBotDistribution.DEFAULT), + ): + octobot_cli._create_startup_config( + logger, + "default.json", + is_process_child=is_process_child, + ) + read_config_mock.assert_called_once_with(config, logger, activate_profile=False) + ensure_profile_mock.assert_not_called() + validate_config_mock.assert_not_called() + + +class TestLoadOrCreateTentaclesProfileDataBacked: + def test_uses_active_tentacles_setup_config_for_profile_data_backed(self): + config = mock.Mock() + setup_config = mock.Mock() + config.get_active_tentacles_setup_config.return_value = setup_config + community_auth = mock.Mock() + logger = mock.Mock() + with mock.patch.object( + octobot_cli.user_root_folder_provider, + "get_user_reference_tentacle_config_file_path", + mock.Mock(return_value="/tmp/tentacles.json"), + ), mock.patch("octobot.cli.os.path.isfile", mock.Mock(return_value=True)), mock.patch.object( + octobot_cli.commands, + "run_update_or_repair_tentacles_if_necessary", + mock.Mock(), + ) as repair_mock: + octobot_cli._load_or_create_tentacles(community_auth, config, logger) + config.get_active_tentacles_setup_config.assert_called_once_with() + config.get_tentacles_config_path.assert_not_called() + repair_mock.assert_called_once_with(community_auth, config, setup_config) + + +class TestConfigureProfileSyncUserNormalBot: + def test_uses_auto_init_sync_client(self): + community_auth = mock.Mock() + community_auth.auto_init_sync_client.return_value = True + community_auth.sync_user_id = "normal-user" + config = mock.Mock() + octobot_cli._configure_profile_sync_user( + config, + community_auth, + is_process_child=False, + ) + community_auth.auto_init_sync_client.assert_called_once_with() + config.profile_storage.configure_sync_user.assert_called_once_with("normal-user") + config.profile_storage.bind_process_child_sync_user_id.assert_not_called() + + def test_skips_when_auto_init_sync_client_fails(self): + community_auth = mock.Mock() + community_auth.auto_init_sync_client.return_value = False + config = mock.Mock() + octobot_cli._configure_profile_sync_user( + config, + community_auth, + is_process_child=False, + ) + config.profile_storage.configure_sync_user.assert_not_called() diff --git a/tests/unit_tests/test_configuration_manager.py b/tests/unit_tests/test_configuration_manager.py index 8defd2214c..a66306ff29 100644 --- a/tests/unit_tests/test_configuration_manager.py +++ b/tests/unit_tests/test_configuration_manager.py @@ -15,13 +15,14 @@ # License along with OctoBot. If not, see . import os -from octobot.configuration_manager import init_config -from octobot_commons.constants import CONFIG_FILE -from octobot_commons.tests.test_config import TEST_CONFIG_FOLDER +import octobot.configuration_manager as configuration_manager +import octobot_commons.constants as commons_constants +import octobot_commons.tests.test_config as test_config +import octobot_commons.user_root_folder_provider as user_root_folder_provider def get_fake_config_path(): - return os.path.join(TEST_CONFIG_FOLDER, f"test_{CONFIG_FILE}") + return os.path.join(test_config.TEST_CONFIG_FOLDER, f"test_{commons_constants.CONFIG_FILE}") def test_init_config(): @@ -29,6 +30,34 @@ def test_init_config(): if os.path.isfile(config_path): os.remove(config_path) - init_config(config_file=config_path, from_config_file=os.path.join(TEST_CONFIG_FOLDER, CONFIG_FILE)) + configuration_manager.init_config( + config_file=config_path, + from_config_file=os.path.join(test_config.TEST_CONFIG_FOLDER, commons_constants.CONFIG_FILE), + ) assert os.path.isfile(config_path) os.remove(config_path) + + +def test_init_config_uses_runtime_user_root_not_import_time_default(tmp_path): + automation_user_root = tmp_path / "user" / "automations" / "child_a" + automation_config_path = automation_user_root / commons_constants.CONFIG_FILE + provider = user_root_folder_provider.instance() + previous_root = provider.get_root() + provider.set_root(str(automation_user_root)) + try: + configuration_manager.init_config( + from_config_file=os.path.join( + test_config.TEST_CONFIG_FOLDER, + commons_constants.CONFIG_FILE, + ), + ) + assert automation_config_path.is_file() + master_config_path = tmp_path / commons_constants.USER_FOLDER / commons_constants.CONFIG_FILE + assert not master_config_path.is_file() + finally: + if previous_root == commons_constants.USER_FOLDER: + provider.set_root(previous_root) + else: + provider._root = None + if automation_config_path.is_file(): + os.remove(automation_config_path) diff --git a/tests/unit_tests/test_task_manager_stop.py b/tests/unit_tests/test_task_manager_stop.py new file mode 100644 index 0000000000..bab90a45bc --- /dev/null +++ b/tests/unit_tests/test_task_manager_stop.py @@ -0,0 +1,52 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . +import asyncio + +import mock +import pytest + +import octobot.task_manager as task_manager_module + + +class TestStopTasksForceExit: + def test_force_timeout_calls_os_exit(self): + octobot_mock = mock.Mock() + octobot_mock.community_handler = None + + task_manager = task_manager_module.TaskManager(octobot_mock) + task_manager.async_loop = mock.Mock() + + with mock.patch("octobot_commons.asyncio_tools.run_coroutine_in_asyncio_loop") as run_coroutine_mock: + run_coroutine_mock.side_effect = asyncio.TimeoutError + with mock.patch("os._exit") as os_exit_mock: + with mock.patch("asyncio.run_coroutine_threadsafe"): + task_manager.stop_tasks(force=True) + os_exit_mock.assert_called_once_with(1) + + def test_non_force_timeout_calls_sys_exit(self): + octobot_mock = mock.Mock() + octobot_mock.community_handler = None + + task_manager = task_manager_module.TaskManager(octobot_mock) + task_manager.async_loop = mock.Mock() + + with mock.patch("octobot_commons.asyncio_tools.run_coroutine_in_asyncio_loop") as run_coroutine_mock: + run_coroutine_mock.side_effect = asyncio.TimeoutError + with mock.patch("os._exit") as os_exit_mock: + with pytest.raises(SystemExit) as exit_info: + task_manager.stop_tasks(force=False) + os_exit_mock.assert_not_called() + assert exit_info.value.code == -1