Sync profiles

docs/content/guides/octobot-configuration/profiles.md

@@ -6,10 +6,15 @@ sidebar_position: 1
 
 # Profiles
 
-OctoBot's trading configuration is using profiles (located into
-user/profiles). This allows for quick switches between previously set
+OctoBot's trading configuration is using profiles. This allows for quick switches between previously set
 configurations. Each profile defines a [Trading Mode](/guides/octobot-trading-modes/trading-modes) configuration as well as other settings.
 
+Bundled default profiles (for example `default` and `non-trading`) remain on the filesystem under
+`user/profiles/`. When a wallet is configured, user-created profiles are stored in the sync
+`StrategyProvider` collection as `GenericProcessConfiguration.profile_data` (same profile id as the
+strategy). OctoBot still exposes the same profile API to the web UI and configuration layer; only the
+profile module selects the storage backend.
+
 ![octobot trading mode details from profiles](/images/guides/configuration/octobot-trading-mode-details-from-profiles.png)
 
 Profiles include:

octobot/cli.py

@@ -134,23 +134,30 @@ def _create_configuration():
     return config
 
 
-def _create_startup_config(logger, default_config_file):
+def _create_startup_config(logger, default_config_file, *, is_process_child: bool = False):
     logger.info("Loading config files...")
     config = _create_configuration()
     is_first_startup = config.is_config_file_empty_or_missing()
     if is_first_startup:
-        logger.info("No configuration found creating default configuration...")
-        configuration_manager.init_config(from_config_file=default_config_file)
-        config.read(should_raise=False)
+        user_config_path = configuration.get_user_config()
+        if is_process_child:
+            raise errors.ConfigError(
+                f"Process OctoBot child expected prepared {common_constants.CONFIG_FILE} at "
+                f"{user_config_path!r} (under {common_constants.USER_AUTOMATIONS_FOLDER}/<automation_id>/). "
+                f"The executor must materialize the automation layout before spawn; if this file is missing, "
+                f"check ensure_user_profile_and_layout timing or parallel functional test races."
+            )
+        logger.info(
+            f"No configuration found in {user_config_path}. "
+            f"Creating default configuration..."
+        )
+        configuration_manager.init_config(
+            config_file=user_config_path,
+            from_config_file=default_config_file,
+        )
+        config.read(should_raise=False, activate_profile=False)
     else:
-        _read_config(config, logger)
-        try:
-            commands.ensure_profile(config)
-            _validate_config(config, logger)
-        except (errors.NoProfileError, errors.ConfigError):
-            # real issue if tentacles exist otherwise continue
-            if os.path.isdir(tentacles_manager_constants.TENTACLES_PATH):
-                raise
+        _read_config(config, logger, activate_profile=False)
     distribution = configuration_manager.get_distribution(config.config)
     if distribution is not enums.OctoBotDistribution.DEFAULT:
         logger.info(f"Using {distribution.value} OctoBot distribution.")
@@ -183,7 +190,7 @@ async def _apply_db_bot_config(logger, config, community_auth) -> bool:
             constants.COMMUNITY_BOT_ID,
             constants.USER_AUTH_KEY,
         )
-        profile = await profiles.import_profile_data_as_profile(
+        profile = await config.profile_storage.import_profile_data(
             profile_data,
             constants.PROFILE_FILE_SCHEMA,
             None,
@@ -277,17 +284,31 @@ def _apply_forced_configs(community_auth, logger, config, is_first_startup):
     _apply_env_variables_to_config(logger, config)
 
 
-def _read_config(config, logger):
+def _read_config(config, logger, *, activate_profile=True):
     try:
-        config.read(should_raise=True, fill_missing_fields=True)
+        config.read(
+            should_raise=True,
+            fill_missing_fields=True,
+            activate_profile=activate_profile,
+        )
     except errors.NoProfileError:
         _repair_with_default_profile(config, logger)
         config = _create_configuration()
-        config.read(should_raise=False, fill_missing_fields=True)
+        config.read(
+            should_raise=False,
+            fill_missing_fields=True,
+            activate_profile=activate_profile,
+        )
     except Exception as e:
         raise errors.ConfigError(e)
 
 
+def _activate_saved_profile_after_sync(config, logger):
+    config.activate_saved_profile()
+    commands.ensure_profile(config)
+    _validate_config(config, logger)
+
+
 def _validate_config(config, logger):
     try:
         config.validate()
@@ -314,9 +335,7 @@ def _load_or_create_tentacles(community_auth, config, logger):
     ):
         # when tentacles folder already exists
         config.load_profiles_if_possible_and_necessary()
-        tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config(
-            config.get_tentacles_config_path()
-        )
+        tentacles_setup_config = config.get_active_tentacles_setup_config()
         commands.run_update_or_repair_tentacles_if_necessary(community_auth, config, tentacles_setup_config)
     else:
         # when no tentacles folder has been found
@@ -338,6 +357,55 @@ def _init_cli_overriden_folders(args):
     return overrides, logs_folder
 
 
+def _assert_process_child_folder_overrides(args) -> None:
+    """When ``--dump-state`` is set (DSL process children), require an automation-scoped ``--user-folder``."""
+    if not args.dump_state or not str(args.dump_state).strip():
+        return
+    user_folder = args.user_folder
+    if not user_folder or not str(user_folder).strip():
+        raise errors.ConfigError(
+            "Process OctoBot children require --user-folder under "
+            f"{common_constants.USER_AUTOMATIONS_FOLDER}/<automation_id>/ when --dump-state is set."
+        )
+    path_segments = tuple(
+        segment
+        for segment in str(user_folder).replace("\\", "/").split("/")
+        if segment
+    )
+    expected_prefix = (
+        common_constants.USER_FOLDER,
+        common_constants.AUTOMATIONS_FOLDER,
+    )
+    if len(path_segments) < len(expected_prefix) + 1:
+        raise errors.ConfigError(
+            "Process OctoBot children require --user-folder under "
+            f"{common_constants.USER_AUTOMATIONS_FOLDER}/<automation_id>/, got "
+            f"{user_folder!r}."
+        )
+    if path_segments[: len(expected_prefix)] != expected_prefix:
+        raise errors.ConfigError(
+            "Process OctoBot children require --user-folder to start with "
+            f"{common_constants.USER_AUTOMATIONS_FOLDER}/, got {user_folder!r}."
+        )
+    if ".." in path_segments:
+        raise errors.ConfigError(
+            f"Invalid --user-folder for process child: parent segments are not allowed ({user_folder!r})."
+        )
+
+def _configure_profile_sync_user(config, community_auth, *, is_process_child: bool = False):
+    if is_process_child:
+        sync_user_id = constants.PROCESS_BOT_SYNC_USER_ID
+        if not sync_user_id:
+            raise errors.ConfigError(
+                "Process OctoBot children require "
+                f"{constants.ENV_PROCESS_BOT_SYNC_USER_ID} to be set by the executor."
+            )
+        config.profile_storage.bind_process_child_sync_user_id(sync_user_id)
+        return
+    if community_auth is not None and community_auth.auto_init_sync_client():
+        config.profile_storage.configure_sync_user(community_auth.sync_user_id)
+
+
 def start_octobot(args, default_config_file=None):
     logger = None
     try:
@@ -346,6 +414,7 @@ def start_octobot(args, default_config_file=None):
             return
 
         overrides, logs_folder = _init_cli_overriden_folders(args)
+        _assert_process_child_folder_overrides(args)
         logger = octobot_logger.init_logger(logs_folder=logs_folder)
         startup_messages = []
 
@@ -364,8 +433,11 @@ def start_octobot(args, default_config_file=None):
         octobot_community.init_sentry_tracker()
 
         # load configuration
+        is_process_child = bool(args.dump_state and str(args.dump_state).strip())
         config, is_first_startup = _create_startup_config(
-            logger, default_config_file or constants.DEFAULT_CONFIG_FILE
+            logger,
+            default_config_file or constants.DEFAULT_CONFIG_FILE,
+            is_process_child=is_process_child,
         )
 
         # check config loading
@@ -387,6 +459,14 @@ def start_octobot(args, default_config_file=None):
             _get_authenticated_community_if_possible(config, logger)
         )
 
+        _configure_profile_sync_user(
+            config,
+            community_auth,
+            is_process_child=is_process_child,
+        )
+
+        _activate_saved_profile_after_sync(config, logger)
+
         # tries to load, install or repair tentacles
         _load_or_create_tentacles(community_auth, config, logger)
 

octobot/commands.py

@@ -333,7 +333,7 @@ async def start_bot(bot, logger, catch=False):
 
 
 def stop_bot(bot, force=False):
-    bot.task_manager.stop_tasks()
+    bot.task_manager.stop_tasks(stop_managed_child_processes=True, force=force)
     if force:
         os._exit(0)
 

octobot/community/authentication.py

@@ -44,6 +44,7 @@
 import octobot_commons.authentication as authentication
 import octobot_commons.configuration as commons_configuration
 import octobot_commons.profiles as commons_profiles
+import octobot_commons.user_root_folder_provider as user_root_folder_provider
 import octobot_trading.enums as trading_enums
 import octobot_sync.client as sync_client
 import octobot_sync.chain as sync_chain
@@ -129,10 +130,10 @@ def __init__(self, config=None, backend_url=None, backend_key=None, use_as_singl
 
         self._fetch_account_task: typing.Optional[asyncio.Task] = None
         self._sync_client = None
-        self._sync_user_id: str = ""
+        self.sync_user_id: str = ""
         self._sync_client_lock = threading.Lock()
         self._wallet_backend: wallet_backend.WalletBackend = wallet_backend.WalletBackend(
-            self.configuration_storage.sync_storage, self.logger
+            self._get_wallet_sync_storage(), self.logger
         )
 
     @staticmethod
@@ -144,6 +145,23 @@ def create(configuration: commons_configuration.Configuration, **kwargs):
 
     def update(self, configuration: commons_configuration.Configuration):
         self.configuration_storage.set_configuration(configuration)
+        self._wallet_backend = wallet_backend.WalletBackend(
+            self._get_wallet_sync_storage(), self.logger
+        )
+
+    def _get_wallet_sync_storage(self):
+        sync_data_root = os.path.normpath(user_root_folder_provider.get_sync_data_root())
+        user_root = os.path.normpath(user_root_folder_provider.get_user_root_folder())
+        if sync_data_root != user_root:
+            master_config_path = os.path.join(sync_data_root, commons_constants.CONFIG_FILE)
+            master_config = commons_configuration.Configuration(
+                master_config_path,
+                os.path.join(sync_data_root, commons_constants.PROFILES_FOLDER),
+            )
+            if os.path.isfile(master_config_path):
+                master_config.read(should_raise=False, activate_profile=False)
+            return supabase_backend.SyncConfigurationStorage(master_config)
+        return self.configuration_storage.sync_storage
 
     def get_logged_in_email(self):
         if self.user_account.has_user_data():
@@ -712,7 +730,7 @@ def init_sync_client_for_wallet(self, address: str) -> None:
                     self.logger.debug("No sync server URL configured, skipping sync client init")
                     return
                 wallet = self.get_wallet(address)
-                self._sync_client, self._sync_user_id = sync_client.create_sync_client(
+                self._sync_client, self.sync_user_id = sync_client.create_sync_client(
                     private_key=wallet.private_key,
                     sync_url=sync_url,
                 )
@@ -745,7 +763,7 @@ def auto_init_sync_client(self) -> bool:
                         "No sync server URL configured, skipping auto sync client init"
                     )
                     return False
-                self._sync_client, self._sync_user_id = sync_client.create_sync_client(
+                self._sync_client, self.sync_user_id = sync_client.create_sync_client(
                     private_key=wallet.private_key,
                     sync_url=sync_url,
                 )

octobot/configuration_manager.py

@@ -115,14 +115,16 @@ def config_health_check(config: configuration.Configuration, in_backtesting: boo
 
 
 def init_config(
-    config_file=configuration.get_user_config(),
+    config_file=None,
     from_config_file=constants.DEFAULT_CONFIG_FILE
 ):
     """
     Initialize default config
     :param config_file: the config file path
     :param from_config_file: the default config file path
     """
+    if config_file is None:
+        config_file = configuration.get_user_config()
     try:
         user_root = user_root_folder_provider.get_user_root_folder()
         if not os.path.exists(user_root):

octobot/constants.py

@@ -215,6 +215,9 @@
 # Process bot state JSON next to user config (--dump-state); liveness for run_octobot_process
 PROCESS_BOT_STATE_FILE_NAME = "process_bot_state.json"
 ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS = "OCTOBOT_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS"
+ENV_PROCESS_BOT_SYNC_USER_ID = "OCTOBOT_PROCESS_BOT_SYNC_USER_ID"
+PROCESS_BOT_SYNC_USER_ID = os.environ.get(ENV_PROCESS_BOT_SYNC_USER_ID, "").strip()
+
 PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS = float(
     os.getenv(ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS, "30")
 )
@@ -292,6 +295,15 @@
 ENABLE_RUN_DATABASE_LIMIT = os_util.parse_boolean_environment_var("ENABLE_RUN_DATABASE_LIMIT", "True")
 MAX_TOTAL_RUN_DATABASES_SIZE = int(os.getenv("MAX_TOTAL_RUN_DATABASES_SIZE", DEFAULT_MAX_TOTAL_RUN_DATABASES_SIZE))
 
+# Managed child OctoBot processes (run_octobot_process)
+MANAGED_CHILD_GRACEFUL_STOP_TIMEOUT_SECONDS = float(
+    os.getenv("OCTOBOT_MANAGED_CHILD_STOP_TIMEOUT", "10.0")
+)
+
+OCTOBOT_STOP_TIMEOUT_SECONDS = float(
+    os.getenv("OCTOBOT_STOP_TIMEOUT_SECONDS", "30.0")
+)
+
 # Channel
 OCTOBOT_CHANNEL = "OctoBot"
 

octobot/initializer.py

@@ -13,7 +13,6 @@
 #
 #  You should have received a copy of the GNU General Public
 #  License along with OctoBot. If not, see <https://www.gnu.org/licenses/>.
-import octobot_tentacles_manager.api as tentacles_manager_api
 import octobot.constants as constants
 import octobot_commons.databases as databases
 import octobot_commons.logging as logging
@@ -31,9 +30,8 @@ def __init__(self, octobot):
 
     async def create(self, init_bot_storage):
         # initialize tentacle configuration
-        tentacles_config_path = self.octobot.get_startup_config(constants.CONFIG_KEY, dict_only=False).\
-            get_tentacles_config_path()
-        self.octobot.tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config(tentacles_config_path)
+        startup_config = self.octobot.get_startup_config(constants.CONFIG_KEY, dict_only=False)
+        self.octobot.tentacles_setup_config = startup_config.get_active_tentacles_setup_config()
 
         if init_bot_storage:
             try:

octobot/limits.py

@@ -21,7 +21,6 @@
 import octobot_commons.enums as common_enums
 import octobot_commons.logging as logging
 import octobot_commons.time_frame_manager as time_frame_manager
-import octobot_tentacles_manager.api as tentacles_manager_api
 import octobot_evaluators.api as evaluators_api
 import octobot_trading.api as trading_api
 
@@ -79,7 +78,7 @@ def _apply_symbols_limits(dict_config, logger, limit):
 
 
 def _apply_time_frames_limits(full_config, logger, limit):
-    tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config(full_config.get_tentacles_config_path())
+    tentacles_setup_config = full_config.get_active_tentacles_setup_config()
     has_disabled_time_frames = False
     all_enabled_time_frames = []
     # patch time frames config

octobot/storage/process_bot_state_dumper.py

@@ -82,7 +82,7 @@ async def _write_state_file_async(
             bot,
         ),
     )
-    content = state.to_dict(include_default_values=False)
+    content = json_util.sanitize(state.to_dict(include_default_values=False))
     str_content = json_util.dump_formatted_json(content)
     full_path = os.path.abspath(state_file_path)
     directory = os.path.dirname(full_path)