From 66cf2fcdb8def9526cce717618652eda8ab382f5 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Wed, 24 Jun 2026 22:48:19 +0200 Subject: [PATCH] [Node] handle full bot start --- packages/commons/octobot_commons/os_util.py | 25 +- packages/commons/tests/test_os_util.py | 41 +- .../octobot_process_functional_shared.py | 2 +- .../test_octobot_process_edit_config.py | 4 +- .../test_octobot_process_start.py | 212 +++++++- packages/node/octobot_node/constants.py | 8 + .../node/octobot_node/protocol/automations.py | 10 +- .../automation/create_automation.py | 25 +- .../util/action_details_factory.py | 83 ++- packages/node/tests/conftest.py | 19 + ...default_config_octobot_process_workflow.py | 210 ++++++++ .../util/octobot_process_workflow.py | 220 ++++++++ .../node/tests/protocol/test_automations.py | 44 ++ .../automation/test_create_automation.py | 240 ++++++++- .../util/test_action_details_factory.py | 32 ++ .../docs/GenericProcessConfiguration.md | 2 +- .../protocol/docs/StrategyConfiguration.md | 2 +- .../models/generic_process_configuration.py | 4 +- .../models/strategy_configuration.py | 10 +- .../models/GenericProcessConfiguration.ts | 2 +- .../models/StrategyConfiguration.ts | 10 +- packages/protocol/openapi.json | 13 +- .../test_generic_process_configuration.py | 1 - .../test/test_strategy_configuration.py | 1 - .../octobot_process_ops.py | 176 +++++-- .../tests/test_octobot_process_ops.py | 495 +++++++++++++++++- .../__tests__/user-action-templates.test.ts | 37 ++ .../src/lib/debug/user-action-templates.ts | 25 + ...mple_market_making_profile_data_adapter.py | 3 - .../simple_market_making_trading.py | 13 +- .../test_simple_market_making_trading.py | 46 ++ 31 files changed, 1912 insertions(+), 103 deletions(-) create mode 100644 packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py create mode 100644 packages/node/tests/functional_tests/util/octobot_process_workflow.py diff --git a/packages/commons/octobot_commons/os_util.py b/packages/commons/octobot_commons/os_util.py index cd2fcc8c1c..ecb434b4c2 100644 --- a/packages/commons/octobot_commons/os_util.py +++ b/packages/commons/octobot_commons/os_util.py @@ -194,6 +194,27 @@ def tcp_port_is_free(bind_host: str, port: int) -> bool: return True +_HOST_WIDE_LISTENER_PROBE_HOSTS = ("0.0.0.0", "127.0.0.1") + + +def tcp_port_has_listener_on_host(port: int) -> bool: + """ + :return: True if any TCP listener on this machine uses ``port`` + """ + try: + for connection in psutil.net_connections(kind="tcp"): + if connection.status != psutil.CONN_LISTEN: + continue + local_address = connection.laddr + if local_address is not None and local_address.port == port: + return True + except (psutil.AccessDenied, psutil.Error): + for probe_host in _HOST_WIDE_LISTENER_PROBE_HOSTS: + if not tcp_port_is_free(probe_host, port): + return True + return False + + def find_first_free_listen_port_after_base( bind_host_for_probe: str, listen_port_base: int, @@ -202,13 +223,15 @@ def find_first_free_listen_port_after_base( ) -> int: """ First offset where ``listen_port_base + offset`` is TCP-free on ``bind_host_for_probe`` - (optional: require ``paired_listen_port_base + offset`` free as well, same scan step). + and not in use by any listener on this host. Returns ``listen_port``. """ for offset_from_base in range(max_offset): listen_port = listen_port_base + offset_from_base if blocklist and listen_port in blocklist: continue + if tcp_port_has_listener_on_host(listen_port): + continue if not tcp_port_is_free(bind_host_for_probe, listen_port): continue return listen_port diff --git a/packages/commons/tests/test_os_util.py b/packages/commons/tests/test_os_util.py index 2c48e59e2b..011cab57ee 100644 --- a/packages/commons/tests/test_os_util.py +++ b/packages/commons/tests/test_os_util.py @@ -45,19 +45,40 @@ def test_returns_true_after_listener_released(self): assert os_util.tcp_port_is_free("127.0.0.1", bound_port) is True +class TestTcpPortHasListenerOnHost: + def test_returns_true_when_listener_holds_port(self): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + bound_port = listener.getsockname()[1] + assert os_util.tcp_port_has_listener_on_host(bound_port) is True + + def test_returns_false_after_listener_released(self): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", 0)) + bound_port = listener.getsockname()[1] + assert os_util.tcp_port_has_listener_on_host(bound_port) is False + + class TestFindFirstFreeListenPortAfterBase: def test_returns_base_when_free(self): - with mock.patch.object(os_util, "tcp_port_is_free", return_value=True): + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): listen_port = os_util.find_first_free_listen_port_after_base("127.0.0.1", 50000) assert listen_port == 50000 def test_skips_until_first_free_port(self): - with mock.patch.object(os_util, "tcp_port_is_free", side_effect=[False, False, True]): + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", side_effect=[False, False, True] + ): listen_port = os_util.find_first_free_listen_port_after_base("127.0.0.1", 50100) assert listen_port == 50102 def test_skips_blocklisted_ports(self): - with mock.patch.object(os_util, "tcp_port_is_free", return_value=True): + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): listen_port = os_util.find_first_free_listen_port_after_base( "127.0.0.1", 50200, @@ -65,7 +86,19 @@ def test_skips_blocklisted_ports(self): ) assert listen_port == 50201 + def test_skips_port_with_host_wide_listener(self): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", 0)) + base_port = listener.getsockname()[1] + listener.listen(1) + listen_port = os_util.find_first_free_listen_port_after_base( + "127.0.0.1", base_port, max_offset=5 + ) + assert listen_port == base_port + 1 + def test_raises_when_scan_exhausted(self): - with mock.patch.object(os_util, "tcp_port_is_free", return_value=False): + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=False + ): with pytest.raises(ValueError, match="No free listen port"): os_util.find_first_free_listen_port_after_base("127.0.0.1", 50300, max_offset=2) diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py index 001c20c533..4b309ac27a 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py @@ -38,7 +38,7 @@ EXPECTED_PROCESS_BOT_DUMP_INTERVAL_SEC = 5.0 # Same as `waiting_time=` in run_octobot_process(...) DSL for this file's tests. -WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC = 2.0 +WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC = 2 RECALL_SCHEDULE_TOLERANCE_SEC = 1.5 EXCHANGE_BINANCEUS = "binanceus" diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py index 99b1e91d51..e69c73ff36 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py @@ -82,7 +82,7 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( run_dsl = ( "run_octobot_process(" f"{user_folder!r}, {repr(profile_2x2)}, " - "waiting_time=2.0, ping_timeout=30.0)" + f"waiting_time={octobot_process_functional_shared.WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC}, ping_timeout=30.0)" ) run_action = { "id": octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT, @@ -231,7 +231,7 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( new_run_dsl = ( "run_octobot_process(" f"{user_folder!r}, {repr(profile_3x3)}, " - "waiting_time=2.0, ping_timeout=30.0)" + f"waiting_time={octobot_process_functional_shared.WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC}, ping_timeout=30.0)" ) update_config_priority_action = { "id": ACTION_ID_UPDATE_AUTOMATION_CONFIGURATION, diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py index cbfd3ab2ac..1f1a56be7a 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py @@ -4,6 +4,7 @@ import asyncio import json import os +import pathlib import shutil import time import typing @@ -11,7 +12,9 @@ import mock import octobot.constants as octobot_app_constants +import octobot_commons.configuration as configuration_module import octobot_commons.constants as common_constants +import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.process_util as process_util import octobot_node.constants as octobot_node_constants import pytest @@ -43,7 +46,7 @@ async def test_run_octobot_process_lifecycle_grid_trading( run_dsl = ( "run_octobot_process(" f"{user_folder!r}, {repr(octobot_process_functional_shared.GRID_BINANCEUS_PROFILE_DATA)}, " - "waiting_time=2.0, ping_timeout=30.0)" + f"waiting_time={octobot_process_functional_shared.WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC}, ping_timeout=30.0)" ) run_action = { "id": octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT, @@ -298,3 +301,210 @@ async def test_run_octobot_process_lifecycle_grid_trading( shutil.rmtree(user_root_guess, ignore_errors=True) if os.path.isdir(log_folder_guess): shutil.rmtree(log_folder_guess, ignore_errors=True) + + +async def test_run_octobot_process_lifecycle_default_config_no_profile_data( + init_action: dict, + monkeypatch: pytest.MonkeyPatch, +): + if not os.path.isfile(os.path.join(os.getcwd(), "start.py")): + pytest.skip("start.py missing: run pytest with cwd set to the OctoBot project root") + + non_trading_profile_json = os.path.join( + os.getcwd(), + common_constants.USER_FOLDER, + common_constants.PROFILES_FOLDER, + "non-trading", + common_constants.PROFILE_CONFIG_FILE, + ) + if not os.path.isfile(non_trading_profile_json): + pytest.skip("non-trading profile missing under OctoBot user/profiles") + + monkeypatch.setenv(octobot_app_constants.ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS, "5") + + user_folder = f"functionnal_tests/octlife_default_{uuid.uuid4().hex[:12]}" + exchange_auth = [ + { + "internal_name": octobot_process_functional_shared.EXCHANGE_BINANCEUS, + "api_key": "functional-default-config-key", + "api_secret": "functional-default-config-secret", + "sandboxed": True, + "exchange_type": common_constants.CONFIG_EXCHANGE_SPOT, + } + ] + run_dsl = ( + f"run_octobot_process({user_folder!r}, " + f"exchange_auth_data={dsl_interpreter.format_parameter_value(exchange_auth)}, " + f"waiting_time={octobot_process_functional_shared.WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC}, ping_timeout=30.0)" + ) + run_action = { + "id": octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT, + "dsl_script": run_dsl, + "dependencies": [{"action_id": octobot_process_functional_shared.ACTION_ID_INIT}], + } + stop_automation_action = { + "id": octobot_process_functional_shared.ACTION_ID_STOP_AUTOMATION, + "dsl_script": "stop_automation()", + "dependencies": [{"action_id": octobot_process_functional_shared.ACTION_ID_INIT}], + } + + popen_calls = {"count": 0} + tracked_spawn_managed = ( + octobot_process_functional_shared._make_tracked_spawn_managed_with_forward_terminal_output( + process_util.spawn_managed_subprocess, + popen_calls, + ) + ) + + user_root_guess = os.path.normpath( + os.path.join( + os.getcwd(), + *common_constants.USER_AUTOMATIONS_FOLDER.split("/"), + *user_folder.replace("\\", "/").split("/"), + ) + ) + log_folder_guess = os.path.normpath( + os.path.join( + os.getcwd(), + *octobot_node_constants.AUTOMATION_LOGS_FOLDER.split("/"), + *[segment for segment in user_folder.replace("\\", "/").split("/") if segment], + ) + ) + + try: + with ( + functionnal_tests.mocked_community_authentication(), + functionnal_tests.mocked_community_repository(), + mock.patch.object( + process_util, + "spawn_managed_subprocess", + side_effect=tracked_spawn_managed, + ), + ): + state = functionnal_tests.automation_state_dict( + functionnal_tests.resolved_actions([init_action]) + ) + async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as init_job: + await init_job.run() + state = init_job.dump() + + async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as job: + job.automation_state.upsert_automation_actions( + functionnal_tests.resolved_actions([run_action]) + ) + state = job.dump() + + deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC + inner: typing.Optional[dict] = None + async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as first_poll: + await first_poll.run() + octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( + first_poll.dump() + ) + first_run = octobot_process_functional_shared._get_action_by_id( + first_poll, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT + ) + assert first_run is not None + inner = octobot_process_functional_shared._recall_inner_from_dsl_action(first_run) + state = first_poll.dump() + if not (inner and inner.get("init_state_ok") is True): + while time.monotonic() < deadline: + await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) + async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as poll_job: + await poll_job.run() + octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( + poll_job.dump() + ) + run_details = octobot_process_functional_shared._get_action_by_id( + poll_job, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT + ) + assert run_details is not None + inner = octobot_process_functional_shared._recall_inner_from_dsl_action(run_details) + if inner and inner.get("init_state_ok") is True: + state = poll_job.dump() + break + state = poll_job.dump() + else: + pytest.fail( + f"OctoBot did not become ready (init_state_ok) within " + f"{octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC}s" + ) + + assert inner is not None + assert inner.get("pid"), "expected child pid in ensure state" + assert popen_calls["count"] >= 1 + child_pid = int(inner["pid"]) + assert process_util.pid_is_running(child_pid) + + user_root = pathlib.Path(inner["user_root"]) + assert inner.get("profile_id") == "non-trading" + root_cfg = json.loads((user_root / common_constants.CONFIG_FILE).read_text(encoding="utf-8")) + exchange_cfg = root_cfg[common_constants.CONFIG_EXCHANGES][ + octobot_process_functional_shared.EXCHANGE_BINANCEUS + ] + exchange_auth_entry = exchange_auth[0] + stored_api_key = exchange_cfg[common_constants.CONFIG_EXCHANGE_KEY] + stored_api_secret = exchange_cfg[common_constants.CONFIG_EXCHANGE_SECRET] + assert configuration_module.decrypt(stored_api_key) == exchange_auth_entry["api_key"] + assert configuration_module.decrypt(stored_api_secret) == exchange_auth_entry["api_secret"] + profile_json_path = ( + user_root + / common_constants.PROFILES_FOLDER + / "non-trading" + / common_constants.PROFILE_CONFIG_FILE + ) + assert profile_json_path.is_file() + + state_path = os.path.normpath( + os.path.join( + inner["user_root"], + octobot_app_constants.PROCESS_BOT_STATE_FILE_NAME, + ) + ) + assert os.path.isfile(state_path) + with open(state_path, encoding="utf-8") as process_state_file: + file_metadata_payload = json.load(process_state_file) + process_metadata = process_bot_state_import.Metadata.from_dict( + file_metadata_payload["metadata"] + ) + assert isinstance(process_metadata, process_bot_state_import.Metadata) + assert isinstance(process_metadata.updated_at, (int, float)) + assert isinstance(process_metadata.next_updated_at, (int, float)) + assert process_metadata.updated_at <= time.time() + assert process_metadata.next_updated_at >= process_metadata.updated_at + assert abs( + (process_metadata.next_updated_at - process_metadata.updated_at) + - octobot_process_functional_shared.EXPECTED_PROCESS_BOT_DUMP_INTERVAL_SEC + ) < 1.0 + + priority_actions = functionnal_tests.resolved_actions([stop_automation_action]) + async with octobot_flow.jobs.AutomationJob(state, priority_actions, [], {}) as stop_phase: + await stop_phase.run() + octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( + stop_phase.dump(), + assert_delay_matches_waiting_time=False, + ) + assert stop_phase.automation_state.automation.post_actions.stop_automation is True + run_stopped = octobot_process_functional_shared._get_action_by_id( + stop_phase, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT + ) + assert run_stopped is not None + assert isinstance(run_stopped.result, dict) + assert run_stopped.result.get("status") in ("stopped", "already_stopped") + + process_deadline = time.monotonic() + octobot_process_functional_shared.CHILD_STOP_WAIT_SEC + while time.monotonic() < process_deadline: + if not process_util.pid_is_running(child_pid): + break + await asyncio.sleep(0.5) + else: + pytest.fail( + f"expected child pid {child_pid} to be stopped after stop_automation/execution_stop " + f"within {octobot_process_functional_shared.CHILD_STOP_WAIT_SEC}s" + ) + + finally: + if os.path.isdir(user_root_guess): + shutil.rmtree(user_root_guess, ignore_errors=True) + if os.path.isdir(log_folder_guess): + shutil.rmtree(log_folder_guess, ignore_errors=True) diff --git a/packages/node/octobot_node/constants.py b/packages/node/octobot_node/constants.py index e0b0c871de..f93305b4d8 100644 --- a/packages/node/octobot_node/constants.py +++ b/packages/node/octobot_node/constants.py @@ -37,6 +37,14 @@ USER_ACTION_WORKFLOW_MAX_ITERATION_RETRIES = int(os.getenv("USER_ACTION_WORKFLOW_MAX_ITERATION_RETRIES", 6)) USER_ACTION_WORKFLOW_BACKOFF_RATE = float(os.getenv("USER_ACTION_WORKFLOW_BACKOFF_RATE", 2)) +# run_octobot_process DSL recall interval and init-state timeout for child OctoBot spawns. +RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS = float( + os.getenv("RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS", 60.0) +) +RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS = float( + os.getenv("RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS", 150.0) +) + TASKS_ENCRYPTION_ENV_VARS = [ "TASKS_SERVER_RSA_PRIVATE_KEY", "TASKS_SERVER_ECDSA_PRIVATE_KEY", diff --git a/packages/node/octobot_node/protocol/automations.py b/packages/node/octobot_node/protocol/automations.py index 3143c43499..90a21f5389 100644 --- a/packages/node/octobot_node/protocol/automations.py +++ b/packages/node/octobot_node/protocol/automations.py @@ -236,6 +236,14 @@ def _flow_result_to_protocol_str(result: typing.Any) -> typing.Optional[str]: return json.dumps(result, default=str) +def _flow_action_result_for_protocol( + flow_action: flow_entities.AbstractActionDetails, +) -> typing.Any: + if flow_action.result is not None: + return flow_action.result + return flow_action.previous_execution_result + + def _metadata_updated_at_from_execution( execution: flow_entities.ExecutionDetails, ) -> typing.Optional[datetime.datetime]: @@ -312,7 +320,7 @@ def _protocol_action_from_flow( status=action_status, dsl=dsl_value, configuration=configuration, - result=_flow_result_to_protocol_str(flow_action.result), + result=_flow_result_to_protocol_str(_flow_action_result_for_protocol(flow_action)), error=_flow_error_status_to_protocol_str(flow_action.error_status), completed_at=completed_at, ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py index 1e5ad14b86..300b77a1ce 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py @@ -193,12 +193,26 @@ def _create_automation_actions(self, user_action: protocol_models.UserAction) -> trading_configuration, ), ] - case protocol_models.GenericProcessConfiguration(): - raise node_errors.UnsupportedAutomationConfigurationTypeError( - f"Unsupported automation configuration type: {protocol_models.ActionConfigurationType.GENERIC_PROCESS.value!r}" - ) + case protocol_models.GenericProcessConfiguration() as generic_process_configuration: + return [ + init_action, + action_details_factory.generic_process_action_factory( + init_action, + generic_process_configuration, + protocol_account, + self._user_id, + automation_id=automation_id, + ), + ] case protocol_models.CopyConfiguration() as copy_configuration: - return [init_action, action_details_factory.copy_action_factory(init_action, copy_configuration)] + return [ + init_action, + action_details_factory.copy_action_factory( + init_action, + copy_configuration, + reference_market=stored_strategy.reference_market, + ), + ] case protocol_models.GenericWorkflowConfiguration() as generic_workflow_configuration: workflow_actions = action_details_factory.generic_workflow_actions_factory( init_action, @@ -215,6 +229,7 @@ def _create_automation_actions(self, user_action: protocol_models.UserAction) -> self._user_id, stored_strategy.reference_market, stored_strategy, + automation_id=automation_id, ), ] case _: diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/action_details_factory.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/action_details_factory.py index fdf41dc2c2..4f8099b94c 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/action_details_factory.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/action_details_factory.py @@ -14,6 +14,7 @@ import octobot_trading.exchanges.util.exchange_data as exchange_data_module import octobot_node.errors as node_errors +import octobot_node.constants as node_constants import octobot_node.scheduler.user_actions.user_actions_executor.util.account_authentication_resolver as account_authentication_resolver import octobot_node.scheduler.user_actions.user_actions_executor.util.exchange_account_resolver as exchange_account_resolver import octobot_node.scheduler.user_actions.user_actions_executor.util.trading_tentacles_config as trading_tentacles_config @@ -22,6 +23,13 @@ _ACTION_ID_INIT = "action_init" +def _run_octobot_process_recall_kwarg_segments() -> list[str]: + return [ + f"waiting_time={node_constants.RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS}", + f"ping_timeout={node_constants.RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS}", + ] + + def _protocol_account_updated_at_unix_seconds(protocol_account: protocol_models.Account) -> float: moment = protocol_account.updated_at if moment.tzinfo is None: @@ -192,10 +200,12 @@ def init_action_factory( def copy_action_factory( init_action: flow_entities.AbstractActionDetails, copy_configuration: protocol_models.CopyConfiguration, + *, + reference_market: str, ) -> flow_entities.AbstractActionDetails: dsl_script = ( f"copy_exchange_account(strategy_id={json.dumps(copy_configuration.strategy_id)}, " - "reference_market='', reference_account='', account_copy_settings='{}')" + f"reference_market={json.dumps(reference_market)}, reference_account='', account_copy_settings='{{}}')" ) return flow_entities.DSLScriptActionDetails( id=_action_id_from_configuration(copy_configuration), @@ -403,6 +413,8 @@ def market_making_action_factory( user_id: str, reference_market: str, stored_strategy: protocol_models.Strategy, + *, + automation_id: str, ) -> flow_entities.AbstractActionDetails: profile_data = market_making_profile_data_factory( protocol_account=protocol_account, @@ -410,6 +422,7 @@ def market_making_action_factory( reference_market=reference_market, user_id=user_id, stored_strategy=stored_strategy, + automation_id=automation_id, ) profile_data_dict = profile_data.to_dict(include_default_values=False) exchange_auth_data = _exchange_auth_data_list_from_protocol_account( @@ -419,9 +432,9 @@ def market_making_action_factory( exchange_auth_segment = dsl_interpreter.format_parameter_value(exchange_auth_data) run_dsl = ( "run_octobot_process(" - f"{protocol_account.id!r}, {dsl_interpreter.format_parameter_value(profile_data_dict)}, " + f"{automation_id!r}, {dsl_interpreter.format_parameter_value(profile_data_dict)}, " f"{exchange_auth_segment}, " - "waiting_time=2.0, ping_timeout=30.0)" + f"{', '.join(_run_octobot_process_recall_kwarg_segments())})" ) return flow_entities.DSLScriptActionDetails( id=_action_id_from_configuration(market_making_configuration), @@ -430,6 +443,36 @@ def market_making_action_factory( ) +def generic_process_action_factory( + init_action: flow_entities.AbstractActionDetails, + generic_process_configuration: protocol_models.GenericProcessConfiguration, + protocol_account: protocol_models.Account, + user_id: str, + *, + automation_id: str, +) -> flow_entities.AbstractActionDetails: + exchange_auth_data = _exchange_auth_data_list_from_protocol_account( + protocol_account, + user_id, + ) + dsl_arguments = [f"{automation_id!r}"] + if generic_process_configuration.profile_data is not None: + dsl_arguments.append( + dsl_interpreter.format_parameter_value(generic_process_configuration.profile_data) + ) + if exchange_auth_data is not None: + dsl_arguments.append( + f"exchange_auth_data={dsl_interpreter.format_parameter_value(exchange_auth_data)}" + ) + dsl_arguments.extend(_run_octobot_process_recall_kwarg_segments()) + run_dsl = "run_octobot_process(" + ", ".join(dsl_arguments) + ")" + return flow_entities.DSLScriptActionDetails( + id=_action_id_from_configuration(generic_process_configuration), + dsl_script=run_dsl, + dependencies=[_action_dependency(init_action.id)], + ) + + def exchange_protocol_account_to_apply_configuration_dict( protocol_account: protocol_models.Account, *, @@ -499,6 +542,7 @@ def market_making_profile_data_factory( reference_market: str, user_id: str, stored_strategy: protocol_models.Strategy, + automation_id: str, ) -> commons_profile_data.ProfileData: if protocol_account.specifics is None or protocol_account.specifics.actual_instance is None: raise node_errors.InvalidAutomationConfigurationError( @@ -524,16 +568,11 @@ def market_making_profile_data_factory( ) starting_portfolio = {asset.symbol: float(asset.total) for asset in portfolio_assets} - profile_identifier = f"market_making_{protocol_account.id}" + profile_identifier = f"market_making_{automation_id}" profile_details = commons_profile_data.ProfileDetailsData( name=profile_identifier, id=profile_identifier, ) - crypto_currency = commons_profile_data.CryptoCurrencyData( - trading_pairs=list(symbols), - name=symbols[0].split("/")[0], - enabled=True, - ) exchange_entry = commons_profile_data.ExchangeData( internal_name=exchange_account_resolver.get_exchange_config( user_id, @@ -555,11 +594,11 @@ def market_making_profile_data_factory( ) tentacle = commons_profile_data.TentaclesData( name="SimpleMarketMakingTradingMode", - config=market_making_configuration.to_dict(), + config=market_making_configuration.model_dump(mode='json'), ) return commons_profile_data.ProfileData( profile_details=profile_details, - crypto_currencies=[crypto_currency], + crypto_currencies=[], exchanges=[exchange_entry], trader=trader, trader_simulator=trader_simulator, @@ -622,7 +661,27 @@ def _exchange_auth_data_list_from_protocol_account( and AccountAuthenticationProvider credentials. """ if protocol_account.is_simulated: - return None + account_specifics = protocol_account.specifics + if account_specifics is None or account_specifics.actual_instance is None: + raise node_errors.InvalidAutomationConfigurationError( + "Account.specifics.actual_instance is required to build exchange_auth_data for a simulated account." + ) + specifics_instance = account_specifics.actual_instance + if not isinstance(specifics_instance, protocol_models.ExchangeAccount): + raise node_errors.InvalidAutomationConfigurationError( + f"exchange_auth_data requires an exchange account; got {type(specifics_instance).__name__}." + ) + exchange_config = exchange_account_resolver.get_exchange_config( + user_id, + specifics_instance, + ) + return [ + { + "internal_name": exchange_config.exchange, + "sandboxed": exchange_config.sandboxed, + "exchange_type": commons_constants.DEFAULT_EXCHANGE_TYPE, + } + ] account_specifics = protocol_account.specifics if account_specifics is None or account_specifics.actual_instance is None: raise node_errors.InvalidAutomationConfigurationError( diff --git a/packages/node/tests/conftest.py b/packages/node/tests/conftest.py index 946491ea7c..a39916a6a0 100644 --- a/packages/node/tests/conftest.py +++ b/packages/node/tests/conftest.py @@ -1,8 +1,16 @@ import contextlib +import os import mock import pytest +_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS = 2 + +os.environ.setdefault( + "RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS", + str(_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS), +) + import octobot.community.local_authenticator as local_community_auth @@ -20,3 +28,14 @@ def mocked_local_user_configuration(): def _mock_local_user_configuration(): with mocked_local_user_configuration(): yield + + +@pytest.fixture(autouse=True) +def _fast_run_octobot_process_recall(monkeypatch): + import octobot_node.constants as node_constants + + monkeypatch.setattr( + node_constants, + "RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS", + _TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS, + ) diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py b/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py new file mode 100644 index 0000000000..1f5bfb3e64 --- /dev/null +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py @@ -0,0 +1,210 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import asyncio +import os +import shutil +import time +import typing + +import mock +import pytest + +import octobot.constants as octobot_constants_module +import octobot_protocol.models as octobot_protocol_models +import octobot_commons.process_util as process_util_module + +from .util import authenticator_mocks as authenticator_mocks_module +from .util import octobot_process_workflow as octobot_process_workflow_module +from .util import user_action_assertions as user_action_assertions_module +from .util import workflow_common as workflow_common_module + +import octobot.community.authentication as community_authentication_module +import octobot_node.config +import octobot_node.scheduler +import octobot_node.scheduler.workflows_util as workflows_util_module + +from tests.scheduler import temp_dbos_scheduler + +_T_ENQUEUE_SECONDS = 30.0 +_T_INIT_SECONDS = 60.0 +_T_STOP_SEND_SECONDS = 30.0 +_T_STOP_COMPLETE_SECONDS = 45.0 + +_GENERIC_PROCESS_ACCOUNT_ID = "functional_generic_process_account" +_GENERIC_PROCESS_AUTOMATION_NAME = "test_generic_process_default_config_automation" + + +class TestStartCheckAndStopDefaultConfigOctobotProcessWorkflow: + @pytest.mark.asyncio + async def test_generic_process_default_config_lifecycle(self, temp_dbos_scheduler, monkeypatch): + if not os.path.isfile(os.path.join(os.getcwd(), "start.py")): + pytest.skip("start.py missing: run pytest with cwd set to the OctoBot project root") + + non_trading_profile_json = os.path.join( + os.getcwd(), + "user", + "profiles", + "non-trading", + "profile.json", + ) + if not os.path.isfile(non_trading_profile_json): + pytest.skip("non-trading profile missing under OctoBot user/profiles") + + monkeypatch.setenv(octobot_constants_module.ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS, "5") + + user_id = workflow_common_module.SIMULATOR_GRID_TEST_COMMUNITY_USER_ID + protocol_account = workflow_common_module.protocol_account_for_functional( + account_id=_GENERIC_PROCESS_ACCOUNT_ID, + usdc_total=1000.0, + account_name="Generic process functional account", + ) + create_user_action = octobot_process_workflow_module.build_create_generic_process_user_action( + account_id=_GENERIC_PROCESS_ACCOUNT_ID, + name=_GENERIC_PROCESS_AUTOMATION_NAME, + ) + authentication_instance = authenticator_mocks_module.build_community_authentication( + workflow_common_module.SIMULATOR_GRID_TEST_PRIVATE_KEY, + workflow_common_module.SIMULATOR_GRID_TEST_WALLET_PASSPHRASE, + ) + + child_user_root: str | None = None + child_log_folder: str | None = None + child_pid: int | None = None + + with ( + mock.patch.object( + community_authentication_module.CommunityAuthentication, + "instance", + return_value=authentication_instance, + ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=mock.Mock( + get_item=mock.Mock(return_value=protocol_account), + get_exchange_config=mock.Mock( + return_value=workflow_common_module.protocol_exchange_config_for_grid_functional(), + ), + ), + ), + mock.patch( + "octobot_sync.sync.collection_providers.StrategyProvider.instance", + return_value=mock.Mock( + get_item=mock.Mock( + return_value=octobot_process_workflow_module.seeded_generic_process_strategy_for_functional_wallet(), + ), + ), + ), + mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_RSA_PRIVATE_KEY", None), + mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_ECDSA_PRIVATE_KEY", None), + ): + try: + await asyncio.wait_for( + workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( + temp_dbos_scheduler, + create_user_action, + user_id, + ), + timeout=_T_ENQUEUE_SECONDS, + ) + except TimeoutError as exc: + raise AssertionError("execute_user_action timed out enqueueing automation workflow") from exc + + await user_action_assertions_module.assert_user_action_selector_completed_automation_create( + user_id=user_id, + user_action_id=create_user_action.id, + expected_workflow_id=None, + ) + metadata_automation_id = user_action_assertions_module.resolve_create_automation_metadata_id( + create_user_action, + ) + parent_automation_id = await user_action_assertions_module.get_created_automation_id_from_user_action( + user_id=user_id, + user_action_id=create_user_action.id, + ) + + inner_state = await octobot_process_workflow_module.wait_for_init_state_ok( + temp_dbos_scheduler, + metadata_automation_id, + timeout_sec=_T_INIT_SECONDS, + ) + assert inner_state.get("pid") + child_pid = int(inner_state["pid"]) + assert process_util_module.pid_is_running(child_pid) + child_user_root = inner_state.get("user_root") + child_log_folder = inner_state.get("log_folder") + + workflow_rows = await temp_dbos_scheduler.INSTANCE.list_workflows_async() + workflow_row_matching: typing.Any = None + state_reader_matching: typing.Any = None + for workflow_row in workflow_rows: + if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + continue + state_reader = workflows_util_module.get_automation_state_reader(workflow_row) + if state_reader is None: + continue + workflow_row_matching = workflow_row + state_reader_matching = state_reader + break + assert state_reader_matching is not None + + protocol_automation = await workflow_common_module.load_protocol_automation_state_for_workflow( + user_id, + workflow_row_matching, + ) + assert protocol_automation.status == octobot_protocol_models.WorkflowStatus.RUNNING + + stop_user_action = workflow_common_module.build_stop_user_action( + automation_id=parent_automation_id, + user_action_id=f"ua-stop-{create_user_action.id}", + ) + try: + await asyncio.wait_for( + workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( + temp_dbos_scheduler, + stop_user_action, + user_id, + ), + timeout=_T_STOP_SEND_SECONDS, + ) + except TimeoutError as exc: + raise AssertionError("execute_user_action timed out enqueueing automation stop") from exc + + stop_assert_deadline = time.monotonic() + _T_STOP_COMPLETE_SECONDS + while time.monotonic() < stop_assert_deadline: + listed_user_actions = await octobot_node.scheduler.SCHEDULER.list_user_actions(user_id) + latest_by_id = user_action_assertions_module.merge_user_actions_latest_per_id(listed_user_actions) + stop_row = latest_by_id.get(stop_user_action.id) + if stop_row is not None and stop_row.status == octobot_protocol_models.UserActionStatus.COMPLETED: + break + await asyncio.sleep(workflow_common_module.DEFAULT_WORKFLOW_POLL_INTERVAL_SECONDS) + else: + await user_action_assertions_module.assert_user_action_selector_completed_automation_stop( + user_id=user_id, + user_action_id=stop_user_action.id, + ) + + await workflow_common_module.wait_for_stop_success_output( + temp_dbos_scheduler, + metadata_automation_id, + _T_STOP_COMPLETE_SECONDS, + ) + + protocol_automation_after_stop = await workflow_common_module.load_protocol_automation_state_for_workflow( + user_id, + workflow_row_matching, + ) + assert protocol_automation_after_stop.status == octobot_protocol_models.WorkflowStatus.COMPLETED + + stop_deadline = time.monotonic() + octobot_process_workflow_module.CHILD_STOP_WAIT_SEC + while time.monotonic() < stop_deadline: + if child_pid is not None and not process_util_module.pid_is_running(child_pid): + break + await asyncio.sleep(0.5) + else: + pytest.fail(f"expected child pid {child_pid} to exit after AUTOMATION_STOP") + + if child_user_root and os.path.isdir(child_user_root): + shutil.rmtree(child_user_root, ignore_errors=True) + if child_log_folder and os.path.isdir(child_log_folder): + shutil.rmtree(child_log_folder, ignore_errors=True) diff --git a/packages/node/tests/functional_tests/util/octobot_process_workflow.py b/packages/node/tests/functional_tests/util/octobot_process_workflow.py new file mode 100644 index 0000000000..3e71ba4166 --- /dev/null +++ b/packages/node/tests/functional_tests/util/octobot_process_workflow.py @@ -0,0 +1,220 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +"""Generic-process OctoBot automation helpers for DBOS functional tests.""" + +from __future__ import annotations + +import asyncio +import datetime +import time +import typing +import uuid + +import octobot_commons.dsl_interpreter as dsl_interpreter +import octobot_flow.entities as flow_entities +import octobot_protocol.models as protocol_models_module +import pytest + +from . import workflow_common as workflow_common_module + +GENERIC_PROCESS_DEFAULT_STRATEGY_ID = "functional-generic-process-default-strategy" +GENERIC_PROCESS_ACTION_ID = f"{protocol_models_module.ActionConfigurationType.GENERIC_PROCESS.value}_1" +GLOBAL_INIT_TIMEOUT_SEC = 60.0 +INIT_POLL_INTERVAL_SEC = 2.0 +CHILD_STOP_WAIT_SEC = 20.0 + + +def build_generic_process_configuration( + *, + profile_data: dict[str, typing.Any] | None = None, +) -> protocol_models_module.GenericProcessConfiguration: + configuration_fields: dict[str, typing.Any] = { + "configuration_type": protocol_models_module.ActionConfigurationType.GENERIC_PROCESS, + } + if profile_data is not None: + configuration_fields["profile_data"] = profile_data + return protocol_models_module.GenericProcessConfiguration(**configuration_fields) + + +def seeded_generic_process_strategy_for_functional_wallet( + *, + stored_strategy_id: str = GENERIC_PROCESS_DEFAULT_STRATEGY_ID, + profile_data: dict[str, typing.Any] | None = None, +) -> protocol_models_module.Strategy: + return protocol_models_module.Strategy( + id=stored_strategy_id, + version=workflow_common_module.SIMULATOR_FUNCTIONAL_STRATEGY_VERSION, + name="Generic process automation strategy", + reference_market="USDC", + configuration=protocol_models_module.StrategyConfiguration( + build_generic_process_configuration(profile_data=profile_data), + ), + ) + + +def build_create_strategy_user_action( + *, + strategy_id: str = GENERIC_PROCESS_DEFAULT_STRATEGY_ID, + profile_data: dict[str, typing.Any] | None = None, +) -> protocol_models_module.UserAction: + strategy = seeded_generic_process_strategy_for_functional_wallet( + stored_strategy_id=strategy_id, + profile_data=profile_data, + ) + strategy_payload = protocol_models_module.CreateStrategyConfiguration( + action_type=protocol_models_module.UserActionType.STRATEGY_CREATE, + configuration=strategy, + ) + return protocol_models_module.UserAction( + id=f"ua-strategy-create-{uuid.uuid4()}", + configuration=workflow_common_module.wrap_user_action_configuration(strategy_payload), + ) + + +def build_create_exchange_config_user_action( + *, + exchange_config_id: str = "functional-generic-process-exchange-config", +) -> protocol_models_module.UserAction: + payload = protocol_models_module.CreateExchangeConfigConfiguration( + action_type=protocol_models_module.UserActionType.EXCHANGE_CONFIG_CREATE, + configuration=protocol_models_module.ExchangeConfig( + id=exchange_config_id, + name="binance-main", + exchange=workflow_common_module.exchange_internal_name(), + sandboxed=False, + ), + ) + return protocol_models_module.UserAction( + id=f"ua-exchange-config-{uuid.uuid4()}", + configuration=workflow_common_module.wrap_user_action_configuration(payload), + ) + + +def build_create_account_user_action( + *, + account_id: str, + exchange_config_id: str = "functional-generic-process-exchange-config", +) -> protocol_models_module.UserAction: + payload = protocol_models_module.CreateAccountConfiguration( + action_type=protocol_models_module.UserActionType.ACCOUNT_CREATE, + configuration=protocol_models_module.Account( + id=account_id, + name="Functional generic process account", + is_simulated=True, + created_at=datetime.datetime(2026, 6, 1, 10, 0, 0, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 6, 1, 10, 0, 0, tzinfo=datetime.UTC), + assets=[ + protocol_models_module.DetailedAssetsForTradingType( + trading_type=protocol_models_module.TradingType.SPOT, + assets=[ + protocol_models_module.DetailedAsset( + symbol="USDC", + total=1000.0, + available=1000.0, + ) + ], + ) + ], + specifics=protocol_models_module.AccountSpecifics( + actual_instance=protocol_models_module.ExchangeAccount( + account_type=protocol_models_module.AccountType.EXCHANGE, + remote_account_id=account_id, + exchange_config_ids=[exchange_config_id], + ), + ), + ), + ) + return protocol_models_module.UserAction( + id=f"ua-account-create-{uuid.uuid4()}", + configuration=workflow_common_module.wrap_user_action_configuration(payload), + ) + + +def build_create_generic_process_user_action( + *, + account_id: str, + name: str, + strategy_id: str = GENERIC_PROCESS_DEFAULT_STRATEGY_ID, + automation_id: str | None = None, +) -> protocol_models_module.UserAction: + strategy_reference = protocol_models_module.StrategyReference( + id=strategy_id, + version=workflow_common_module.SIMULATOR_FUNCTIONAL_STRATEGY_VERSION, + ) + automation_configuration_fields: dict[str, typing.Any] = { + "name": name, + "created_at": datetime.datetime(2026, 6, 1, 11, 0, 0, tzinfo=datetime.UTC), + "strategy": strategy_reference, + "accounts": [protocol_models_module.AccountReference(id=account_id)], + } + if automation_id is not None: + automation_configuration_fields["id"] = automation_id + automation_configuration = protocol_models_module.AutomationConfiguration( + **automation_configuration_fields, + ) + payload = protocol_models_module.CreateAutomationConfiguration( + action_type=protocol_models_module.UserActionType.AUTOMATION_CREATE, + configuration=automation_configuration, + ) + return protocol_models_module.UserAction( + id=f"ua-generic-process-{uuid.uuid4()}", + configuration=workflow_common_module.wrap_user_action_configuration(payload), + ) + + +def _recall_inner_state(run_result: typing.Optional[dict]) -> typing.Optional[dict]: + if not isinstance(run_result, dict): + return None + recalling_payload = run_result.get(dsl_interpreter.ReCallingOperatorResult.__name__) + if not isinstance(recalling_payload, dict): + return None + inner_state = recalling_payload.get("last_execution_result") + return inner_state if isinstance(inner_state, dict) else None + + +def recall_inner_from_run_octobot_action( + action: flow_entities.AbstractActionDetails, +) -> typing.Optional[dict]: + for run_result in (action.result, action.previous_execution_result): + inner_state = _recall_inner_state(run_result) if run_result is not None else None + if inner_state is not None: + return inner_state + return None + + +def get_action_by_id( + automation_state: flow_entities.AutomationState, + action_id: str, +) -> typing.Optional[flow_entities.AbstractActionDetails]: + for action in automation_state.automation.actions_dag.actions: + if action.id == action_id: + return action + return None + + +async def wait_for_init_state_ok( + scheduler: typing.Any, + automation_id: str, + *, + timeout_sec: float = GLOBAL_INIT_TIMEOUT_SEC, + poll_interval_sec: float = INIT_POLL_INTERVAL_SEC, +) -> dict: + deadline = time.monotonic() + timeout_sec + while time.monotonic() < deadline: + workflow_rows = await scheduler.INSTANCE.list_workflows_async() + for workflow_row in workflow_rows: + import octobot_node.scheduler.workflows_util as workflows_util_module + + if workflows_util_module.get_automation_id(workflow_row) != automation_id: + continue + state_reader = workflows_util_module.get_automation_state_reader(workflow_row) + if state_reader is None: + continue + run_action = get_action_by_id(state_reader.state, GENERIC_PROCESS_ACTION_ID) + if run_action is None: + continue + inner_state = recall_inner_from_run_octobot_action(run_action) + if inner_state and inner_state.get("init_state_ok") is True: + return inner_state + await asyncio.sleep(poll_interval_sec) + pytest.fail(f"Timed out waiting for init_state_ok on {GENERIC_PROCESS_ACTION_ID!r} within {timeout_sec}s") diff --git a/packages/node/tests/protocol/test_automations.py b/packages/node/tests/protocol/test_automations.py index 46abd8208d..dd9164b25b 100644 --- a/packages/node/tests/protocol/test_automations.py +++ b/packages/node/tests/protocol/test_automations.py @@ -501,6 +501,50 @@ def test_dsl_and_configured_action_mapping(self): assert by_id["c1"].configuration is not None +class TestProtocolActionFromFlowResult: + def _protocol_action_from_priority_lane_dsl_action( + self, + dsl_action: flow_entities.DSLScriptActionDetails, + ) -> protocol_models.Action: + flow_state = flow_entities.AutomationState( + automation=flow_entities.AutomationDetails( + metadata=flow_entities.AutomationMetadata(automation_id="automation_1"), + actions_dag=flow_entities.ActionsDAG(actions=[]), + ), + priority_actions=[dsl_action], + ) + filled = automations_protocol._fill_protocol_automation_state( + _minimal_protocol_base(), + flow_state, + ) + assert filled.priority_actions is not None + return filled.priority_actions[0] + + def test_maps_current_result_when_set(self): + dsl_action = flow_entities.DSLScriptActionDetails(id="recall_action", dsl_script="True") + dsl_action.result = {"pid": 1} + protocol_action = self._protocol_action_from_priority_lane_dsl_action(dsl_action) + assert protocol_action.result == json.dumps({"pid": 1}) + + def test_falls_back_to_previous_execution_result_when_result_unset(self): + dsl_action = flow_entities.DSLScriptActionDetails(id="recall_action", dsl_script="True") + dsl_action.previous_execution_result = {"pid": 1} + protocol_action = self._protocol_action_from_priority_lane_dsl_action(dsl_action) + assert protocol_action.result == json.dumps({"pid": 1}) + + def test_result_none_when_both_unset(self): + dsl_action = flow_entities.DSLScriptActionDetails(id="recall_action", dsl_script="True") + protocol_action = self._protocol_action_from_priority_lane_dsl_action(dsl_action) + assert protocol_action.result is None + + def test_prefers_current_result_over_previous_execution_result(self): + dsl_action = flow_entities.DSLScriptActionDetails(id="recall_action", dsl_script="True") + dsl_action.result = {"pid": 2} + dsl_action.previous_execution_result = {"pid": 1} + protocol_action = self._protocol_action_from_priority_lane_dsl_action(dsl_action) + assert protocol_action.result == json.dumps({"pid": 2}) + + class TestFillProtocolAutomationStatePriorityActions: def test_priority_actions_separate_and_running_when_incomplete(self): dag_action = flow_entities.DSLScriptActionDetails(id="dag_action", dsl_script="True") diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py index 1ae7797fef..fdf5c6efc8 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py @@ -155,10 +155,10 @@ def _expected_trading_tentacles_dsl_script( ).dsl_script -def _expected_copy_dsl_script(*, strategy_id: str) -> str: +def _expected_copy_dsl_script(*, strategy_id: str, reference_market: str = "") -> str: return ( f"copy_exchange_account(strategy_id={json.dumps(strategy_id)}, " - "reference_market='', reference_account='', account_copy_settings='{}')" + f"reference_market={json.dumps(reference_market)}, reference_account='', account_copy_settings='{{}}')" ) @@ -306,10 +306,53 @@ def test_copy_returns_init_and_copy_action(self): protocol_account=account, strategy_reference=strat_ref, ) - assert actions[1].dsl_script == _expected_copy_dsl_script(strategy_id=copy_strategy_id) + assert actions[1].dsl_script == _expected_copy_dsl_script( + strategy_id=copy_strategy_id, + reference_market="USDT", + ) assert len(actions[1].dependencies) == 1 assert actions[1].dependencies[0].action_id == "action_init" + def test_copy_uses_strategy_reference_market(self): + copy_strategy_id = "copy-strategy" + copy_configuration = protocol_models.CopyConfiguration( + configuration_type=protocol_models.ActionConfigurationType.COPY, + strategy_id=copy_strategy_id, + ) + strat_ref = _default_strategy_reference() + create_payload = protocol_models.CreateAutomationConfiguration( + action_type=protocol_models.UserActionType.AUTOMATION_CREATE, + configuration=_automation_configuration( + name="copy-automation-with-ref-market", + strategy_reference=strat_ref, + account_id="acc-1", + ), + ) + user_action = _user_action_with_context(action_id="ua-copy-ref-market", payload=create_payload) + + executor = create_automation_executor.CreateAutomationActionExecutor(_TEST_WALLET_ADDRESS) + account = _minimal_exchange_account(account_id="acc-1") + stored = protocol_models.Strategy( + id=strat_ref.id, + version=strat_ref.version, + name="Seeded automation strategy", + reference_market="USDC", + configuration=protocol_models.StrategyConfiguration(copy_configuration), + ) + with mock.patch(_ACCOUNT_PROVIDER_INSTANCE_PATCH) as account_mock, mock.patch( + _STRATEGY_PROVIDER_INSTANCE_PATCH, + ) as strategy_mock: + _stub_account_provider(account_mock, account) + strategy_mock.return_value.get_item.return_value = stored + actions = executor._create_automation_actions(user_action) + + assert len(actions) == 2 + assert isinstance(actions[1], flow_entities.DSLScriptActionDetails) + assert actions[1].dsl_script == _expected_copy_dsl_script( + strategy_id=copy_strategy_id, + reference_market="USDC", + ) + def test_grid_returns_init_and_grid_action(self): grid_configuration = trading_tentacles_test_utils.grid_trading_configuration() strat_ref = _default_strategy_reference() @@ -472,14 +515,19 @@ def test_market_making_returns_init_and_run_octobot_process(self): reference_market=stored.reference_market, user_id=_TEST_WALLET_ADDRESS, stored_strategy=stored, + automation_id="ua-mm", + ) + expected_exchange_auth = action_details_factory._exchange_auth_data_list_from_protocol_account( + account, + _TEST_WALLET_ADDRESS, ) expected_profile_dict = expected_profile.to_dict(include_default_values=False) - expected_exchange_auth_segment = dsl_interpreter.format_parameter_value(None) + expected_exchange_auth_segment = dsl_interpreter.format_parameter_value(expected_exchange_auth) expected_dsl = ( "run_octobot_process(" - f"{account.id!r}, {dsl_interpreter.format_parameter_value(expected_profile_dict)}, " + f"{'ua-mm'!r}, {dsl_interpreter.format_parameter_value(expected_profile_dict)}, " f"{expected_exchange_auth_segment}, " - "waiting_time=2.0, ping_timeout=30.0)" + f"{', '.join(action_details_factory._run_octobot_process_recall_kwarg_segments())})" ) _assert_init_action_matches_minimal_account( init_action_details=actions[0], @@ -492,7 +540,8 @@ def test_market_making_returns_init_and_run_octobot_process(self): assert len(main_action.dependencies) == 1 assert main_action.dependencies[0].action_id == "action_init" assert main_action.dsl_script == expected_dsl - assert expected_profile_dict["crypto_currencies"][0]["trading_pairs"] == ["BTC/USDT"] + assert expected_profile.crypto_currencies == [] + assert expected_profile_dict.get("crypto_currencies", []) == [] assert ( expected_profile_dict["tentacles"][0]["config"]["pair_settings"][0]["trading_pair"] == "BTC/USDT" @@ -500,6 +549,55 @@ def test_market_making_returns_init_and_run_octobot_process(self): assert expected_profile_dict["tentacles"][0]["config"]["pair_settings"][0]["min_spread"] == 0.5 assert expected_profile_dict["tentacles"][0]["config"]["pair_settings"][0]["max_spread"] == 1.0 + def test_market_making_run_octobot_process_uses_configuration_id_as_user_folder(self): + configuration_automation_id = _DEFAULT_AUTOMATION_CONFIGURATION_ID + mf = protocol_models.MarketMakingConfiguration( + configuration_type=protocol_models.ActionConfigurationType.MARKET_MAKING, + pair_settings=[ + protocol_models.MarketMakingSymbolConfiguration( + trading_pair="BTC/USDT", + exchange="binanceus", + reference_price=[ + protocol_models.MarketMakingReferencePair( + exchange="binanceus", + pair="BTC/USDT", + ) + ], + min_spread=0.5, + max_spread=1.0, + bids_count=1, + asks_count=1, + orders_distribution=protocol_models.MarketMakingOrdersDistribution.LINEAR, + funds_distribution=protocol_models.MarketMakingFundsDistribution.FLAT, + ) + ], + ) + strat_ref = _default_strategy_reference() + create_payload = protocol_models.CreateAutomationConfiguration( + action_type=protocol_models.UserActionType.AUTOMATION_CREATE, + configuration=_automation_configuration( + name="mm-config-id-automation", + strategy_reference=strat_ref, + account_id="acc-1", + automation_id=configuration_automation_id, + ), + ) + user_action = _user_action_with_context(action_id="ua-different-mm", payload=create_payload) + executor = create_automation_executor.CreateAutomationActionExecutor(_TEST_WALLET_ADDRESS) + account = _minimal_exchange_account(account_id="acc-1") + stored = _stored_strategy_matching_reference(strat_ref, mf) + with mock.patch(_ACCOUNT_PROVIDER_INSTANCE_PATCH) as account_mock, mock.patch( + _STRATEGY_PROVIDER_INSTANCE_PATCH, + ) as strategy_mock: + _stub_account_provider(account_mock, account) + strategy_mock.return_value.get_item.return_value = stored + actions = executor._create_automation_actions(user_action) + + main_action = actions[1] + assert isinstance(main_action, flow_entities.DSLScriptActionDetails) + assert main_action.dsl_script.startswith(f"run_octobot_process({configuration_automation_id!r},") + assert account.id not in main_action.dsl_script.split(",", 1)[0] + def test_dca_returns_init_and_dca_dsl(self): dca_configuration = trading_tentacles_test_utils.functional_dca_trading_configuration() strat_ref = _default_strategy_reference() @@ -721,11 +819,120 @@ def test_trading_tentacles_raises_when_strategies_without_evaluators(self): with pytest.raises(node_errors.InvalidTradingTentaclesConfigurationError): executor._create_automation_actions(user_action) - def test_unsupported_types_raise_dedicated_errors(self): - generic_process = protocol_models.GenericProcessConfiguration( + def test_generic_process_returns_init_and_run_octobot_process(self): + generic_process_configuration = protocol_models.GenericProcessConfiguration( configuration_type=protocol_models.ActionConfigurationType.GENERIC_PROCESS, - profile_data={}, ) + strat_ref = _default_strategy_reference() + create_payload = protocol_models.CreateAutomationConfiguration( + action_type=protocol_models.UserActionType.AUTOMATION_CREATE, + configuration=_automation_configuration( + name="generic-process-automation", + strategy_reference=strat_ref, + account_id="acc-1", + ), + ) + user_action = _user_action_with_context(action_id="ua-generic-process", payload=create_payload) + executor = create_automation_executor.CreateAutomationActionExecutor(_TEST_WALLET_ADDRESS) + account = _minimal_exchange_account(account_id="acc-1") + stored = _stored_strategy_matching_reference(strat_ref, generic_process_configuration) + with mock.patch(_ACCOUNT_PROVIDER_INSTANCE_PATCH) as account_mock, mock.patch( + _STRATEGY_PROVIDER_INSTANCE_PATCH, + ) as strategy_mock: + _stub_account_provider(account_mock, account) + strategy_mock.return_value.get_item.return_value = stored + actions = executor._create_automation_actions(user_action) + + assert len(actions) == 2 + main_action = actions[1] + assert isinstance(main_action, flow_entities.DSLScriptActionDetails) + with mock.patch.object( + action_details_factory.exchange_account_resolver, + "get_exchange_config", + return_value=account_executor_test_utils.exchange_config_payload(), + ): + expected_exchange_auth = action_details_factory._exchange_auth_data_list_from_protocol_account( + account, + _TEST_WALLET_ADDRESS, + ) + expected_dsl = ( + "run_octobot_process(" + f"{'ua-generic-process'!r}, exchange_auth_data={dsl_interpreter.format_parameter_value(expected_exchange_auth)}, " + f"{', '.join(action_details_factory._run_octobot_process_recall_kwarg_segments())})" + ) + _assert_init_action_matches_minimal_account( + init_action_details=actions[0], + expected_automation_id="ua-generic-process", + account_id="acc-1", + protocol_account=account, + strategy_reference=strat_ref, + ) + assert main_action.id == f"{protocol_models.ActionConfigurationType.GENERIC_PROCESS.value}_1" + assert main_action.dsl_script == expected_dsl + assert "profile_data" not in main_action.dsl_script + + def test_generic_process_run_octobot_process_uses_configuration_id_as_user_folder(self): + configuration_automation_id = _DEFAULT_AUTOMATION_CONFIGURATION_ID + generic_process_configuration = protocol_models.GenericProcessConfiguration( + configuration_type=protocol_models.ActionConfigurationType.GENERIC_PROCESS, + ) + strat_ref = _default_strategy_reference() + create_payload = protocol_models.CreateAutomationConfiguration( + action_type=protocol_models.UserActionType.AUTOMATION_CREATE, + configuration=_automation_configuration( + name="generic-process-config-id-automation", + strategy_reference=strat_ref, + account_id="acc-1", + automation_id=configuration_automation_id, + ), + ) + user_action = _user_action_with_context(action_id="ua-different-generic", payload=create_payload) + executor = create_automation_executor.CreateAutomationActionExecutor(_TEST_WALLET_ADDRESS) + stored = _stored_strategy_matching_reference(strat_ref, generic_process_configuration) + with mock.patch(_ACCOUNT_PROVIDER_INSTANCE_PATCH) as account_mock, mock.patch( + _STRATEGY_PROVIDER_INSTANCE_PATCH, + ) as strategy_mock: + _stub_account_provider(account_mock, _minimal_exchange_account(account_id="acc-1")) + strategy_mock.return_value.get_item.return_value = stored + actions = executor._create_automation_actions(user_action) + + main_action = actions[1] + assert isinstance(main_action, flow_entities.DSLScriptActionDetails) + assert main_action.dsl_script.startswith(f"run_octobot_process({configuration_automation_id!r},") + assert "acc-1" not in main_action.dsl_script.split(",", 1)[0] + + def test_generic_process_simulated_account_emits_config_only_exchange_auth_data(self): + generic_process_configuration = protocol_models.GenericProcessConfiguration( + configuration_type=protocol_models.ActionConfigurationType.GENERIC_PROCESS, + ) + strat_ref = _default_strategy_reference() + create_payload = protocol_models.CreateAutomationConfiguration( + action_type=protocol_models.UserActionType.AUTOMATION_CREATE, + configuration=_automation_configuration( + name="generic-process-simulated", + strategy_reference=strat_ref, + account_id="acc-sim", + ), + ) + user_action = _user_action_with_context(action_id="ua-generic-process-sim", payload=create_payload) + executor = create_automation_executor.CreateAutomationActionExecutor(_TEST_WALLET_ADDRESS) + account = _minimal_exchange_account(account_id="acc-sim") + stored = _stored_strategy_matching_reference(strat_ref, generic_process_configuration) + with mock.patch(_ACCOUNT_PROVIDER_INSTANCE_PATCH) as account_mock, mock.patch( + _STRATEGY_PROVIDER_INSTANCE_PATCH, + ) as strategy_mock: + _stub_account_provider(account_mock, account) + strategy_mock.return_value.get_item.return_value = stored + actions = executor._create_automation_actions(user_action) + + main_action = actions[1] + assert isinstance(main_action, flow_entities.DSLScriptActionDetails) + assert "api_key" not in main_action.dsl_script + assert "api_secret" not in main_action.dsl_script + assert "binanceus" in main_action.dsl_script + assert "sandboxed" in main_action.dsl_script + + def test_unknown_configuration_instance_type_raises(self): strat_ref = _default_strategy_reference() create_payload = protocol_models.CreateAutomationConfiguration( action_type=protocol_models.UserActionType.AUTOMATION_CREATE, @@ -738,10 +945,19 @@ def test_unsupported_types_raise_dedicated_errors(self): user_action = _user_action_with_context(action_id="ua-unsupported", payload=create_payload) executor = create_automation_executor.CreateAutomationActionExecutor(_TEST_WALLET_ADDRESS) - stored = _stored_strategy_matching_reference(strat_ref, generic_process) + stored = _stored_strategy_matching_reference( + strat_ref, + protocol_models.GenericProcessConfiguration( + configuration_type=protocol_models.ActionConfigurationType.GENERIC_PROCESS, + ), + ) with mock.patch(_ACCOUNT_PROVIDER_INSTANCE_PATCH) as account_mock, mock.patch( _STRATEGY_PROVIDER_INSTANCE_PATCH, - ) as strategy_mock: + ) as strategy_mock, mock.patch.object( + create_automation_executor, + "_get_strategy_configuration_instance", + return_value=object(), + ): _stub_account_provider(account_mock, _minimal_exchange_account(account_id="acc-1")) strategy_mock.return_value.get_item.return_value = stored with pytest.raises(node_errors.UnsupportedAutomationConfigurationTypeError): diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_action_details_factory.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_action_details_factory.py index 1047c78b2e..88d65ae8f3 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_action_details_factory.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_action_details_factory.py @@ -258,3 +258,35 @@ def test_same_action_ids_for_camel_case_and_snake_case_names(self): assert trading_tentacles_test_utils.tentacle_action_id( dca_trading.DCATradingMode.get_name() ) in {action.id for action in camel_case_actions} + + +class TestCopyActionFactory: + def test_passes_reference_market_argument_to_dsl(self): + copy_configuration = protocol_models.CopyConfiguration( + configuration_type=protocol_models.ActionConfigurationType.COPY, + strategy_id="copied-strategy", + ) + copy_action = action_details_factory_module.copy_action_factory( + _init_action(), + copy_configuration, + reference_market="USDT", + ) + assert copy_action.dsl_script == ( + 'copy_exchange_account(strategy_id="copied-strategy", ' + 'reference_market="USDT", reference_account=\'\', account_copy_settings=\'{}\')' + ) + + def test_passes_different_reference_market_to_dsl(self): + copy_configuration = protocol_models.CopyConfiguration( + configuration_type=protocol_models.ActionConfigurationType.COPY, + strategy_id="copied-strategy", + ) + copy_action = action_details_factory_module.copy_action_factory( + _init_action(), + copy_configuration, + reference_market="USDC", + ) + assert copy_action.dsl_script == ( + 'copy_exchange_account(strategy_id="copied-strategy", ' + 'reference_market="USDC", reference_account=\'\', account_copy_settings=\'{}\')' + ) diff --git a/packages/protocol/docs/GenericProcessConfiguration.md b/packages/protocol/docs/GenericProcessConfiguration.md index 8354396995..48dc51cc99 100644 --- a/packages/protocol/docs/GenericProcessConfiguration.md +++ b/packages/protocol/docs/GenericProcessConfiguration.md @@ -7,7 +7,7 @@ GenericProcessConfiguration Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **configuration_type** | [**ActionConfigurationType**](ActionConfigurationType.md) | generic_process | -**profile_data** | **object** | | +**profile_data** | **object** | | [optional] ## Example diff --git a/packages/protocol/docs/StrategyConfiguration.md b/packages/protocol/docs/StrategyConfiguration.md index 7894b7cac9..b436758141 100644 --- a/packages/protocol/docs/StrategyConfiguration.md +++ b/packages/protocol/docs/StrategyConfiguration.md @@ -12,7 +12,7 @@ Name | Type | Description | Notes **strategies** | [**List[StrategyEvaluatorConfiguration]**](StrategyEvaluatorConfiguration.md) | | [optional] **evaluators** | [**List[EvaluatorConfiguration]**](EvaluatorConfiguration.md) | | [optional] **strategy_id** | **str** | | -**profile_data** | **object** | | +**profile_data** | **object** | | [optional] **actions** | [**List[Action]**](Action.md) | | ## Example diff --git a/packages/protocol/octobot_protocol/models/generic_process_configuration.py b/packages/protocol/octobot_protocol/models/generic_process_configuration.py index d00208b894..886499d383 100644 --- a/packages/protocol/octobot_protocol/models/generic_process_configuration.py +++ b/packages/protocol/octobot_protocol/models/generic_process_configuration.py @@ -18,7 +18,7 @@ import json from pydantic import BaseModel, ConfigDict, Field -from typing import Any, ClassVar, Dict, List +from typing import Any, ClassVar, Dict, List, Optional from octobot_protocol.models.action_configuration_type import ActionConfigurationType from typing import Optional, Set from typing_extensions import Self @@ -29,7 +29,7 @@ class GenericProcessConfiguration(BaseModel): GenericProcessConfiguration """ # noqa: E501 configuration_type: ActionConfigurationType = Field(description="generic_process") - profile_data: Dict[str, Any] + profile_data: Optional[Dict[str, Any]] = None __properties: ClassVar[List[str]] = ["configuration_type", "profile_data"] model_config = ConfigDict( diff --git a/packages/protocol/octobot_protocol/models/strategy_configuration.py b/packages/protocol/octobot_protocol/models/strategy_configuration.py index 8c5419eebd..78c8b7fac5 100644 --- a/packages/protocol/octobot_protocol/models/strategy_configuration.py +++ b/packages/protocol/octobot_protocol/models/strategy_configuration.py @@ -120,27 +120,27 @@ def from_json(cls, json_str: str) -> Self: raise ValueError("Failed to lookup data type from the field `configuration_type` in the input.") # check if data type is `CopyConfiguration` - if _data_type == "CopyConfiguration": + if _data_type == "copy": instance.actual_instance = CopyConfiguration.from_json(json_str) return instance # check if data type is `GenericProcessConfiguration` - if _data_type == "GenericProcessConfiguration": + if _data_type == "generic_process": instance.actual_instance = GenericProcessConfiguration.from_json(json_str) return instance # check if data type is `GenericWorkflowConfiguration` - if _data_type == "GenericWorkflowConfiguration": + if _data_type == "generic_workflow": instance.actual_instance = GenericWorkflowConfiguration.from_json(json_str) return instance # check if data type is `MarketMakingConfiguration` - if _data_type == "MarketMakingConfiguration": + if _data_type == "market_making": instance.actual_instance = MarketMakingConfiguration.from_json(json_str) return instance # check if data type is `TradingTentaclesConfiguration` - if _data_type == "TradingTentaclesConfiguration": + if _data_type == "trading_tentacles": instance.actual_instance = TradingTentaclesConfiguration.from_json(json_str) return instance diff --git a/packages/protocol/octobot_protocol_ts/models/GenericProcessConfiguration.ts b/packages/protocol/octobot_protocol_ts/models/GenericProcessConfiguration.ts index c8dd13a720..04d637ceaa 100644 --- a/packages/protocol/octobot_protocol_ts/models/GenericProcessConfiguration.ts +++ b/packages/protocol/octobot_protocol_ts/models/GenericProcessConfiguration.ts @@ -20,7 +20,7 @@ export class GenericProcessConfiguration { * generic_process */ 'configuration_type': 'generic_process'; - 'profile_data': any; + 'profile_data'?: any; static readonly discriminator: string | undefined = undefined; diff --git a/packages/protocol/octobot_protocol_ts/models/StrategyConfiguration.ts b/packages/protocol/octobot_protocol_ts/models/StrategyConfiguration.ts index 2704b1306a..eb7f2ad9c7 100644 --- a/packages/protocol/octobot_protocol_ts/models/StrategyConfiguration.ts +++ b/packages/protocol/octobot_protocol_ts/models/StrategyConfiguration.ts @@ -31,11 +31,11 @@ export class StrategyConfigurationClass { static readonly discriminator: string | undefined = "configuration_type"; static readonly mapping: {[index: string]: string} | undefined = { - "CopyConfiguration": "CopyConfiguration", - "GenericProcessConfiguration": "GenericProcessConfiguration", - "GenericWorkflowConfiguration": "GenericWorkflowConfiguration", - "MarketMakingConfiguration": "MarketMakingConfiguration", - "TradingTentaclesConfiguration": "TradingTentaclesConfiguration", + "copy": "CopyConfiguration", + "generic_process": "GenericProcessConfiguration", + "generic_workflow": "GenericWorkflowConfiguration", + "market_making": "MarketMakingConfiguration", + "trading_tentacles": "TradingTentaclesConfiguration", }; } diff --git a/packages/protocol/openapi.json b/packages/protocol/openapi.json index 6f2c0bcba5..41b3451ab9 100644 --- a/packages/protocol/openapi.json +++ b/packages/protocol/openapi.json @@ -1536,8 +1536,7 @@ "description": "GenericProcessConfiguration", "type": "object", "required": [ - "configuration_type", - "profile_data" + "configuration_type" ], "properties": { "configuration_type": { @@ -2380,11 +2379,11 @@ "discriminator": { "propertyName": "configuration_type", "mapping": { - "MarketMakingConfiguration": "#/components/schemas/MarketMakingConfiguration", - "TradingTentaclesConfiguration": "#/components/schemas/TradingTentaclesConfiguration", - "CopyConfiguration": "#/components/schemas/CopyConfiguration", - "GenericProcessConfiguration": "#/components/schemas/GenericProcessConfiguration", - "GenericWorkflowConfiguration": "#/components/schemas/GenericWorkflowConfiguration" + "market_making": "#/components/schemas/MarketMakingConfiguration", + "trading_tentacles": "#/components/schemas/TradingTentaclesConfiguration", + "copy": "#/components/schemas/CopyConfiguration", + "generic_process": "#/components/schemas/GenericProcessConfiguration", + "generic_workflow": "#/components/schemas/GenericWorkflowConfiguration" } } } diff --git a/packages/protocol/test/test_generic_process_configuration.py b/packages/protocol/test/test_generic_process_configuration.py index d6bbfad945..188a1f7ad5 100644 --- a/packages/protocol/test/test_generic_process_configuration.py +++ b/packages/protocol/test/test_generic_process_configuration.py @@ -41,7 +41,6 @@ def make_instance(self, include_optional) -> GenericProcessConfiguration: else: return GenericProcessConfiguration( configuration_type = 'market_making', - profile_data = None, ) """ diff --git a/packages/protocol/test/test_strategy_configuration.py b/packages/protocol/test/test_strategy_configuration.py index d25171a7cb..c588ba3bdc 100644 --- a/packages/protocol/test/test_strategy_configuration.py +++ b/packages/protocol/test/test_strategy_configuration.py @@ -165,7 +165,6 @@ def make_instance(self, include_optional) -> StrategyConfiguration: name = '', config = { }, strategy_id = '', - profile_data = octobot_protocol.models.profile_data.profile_data(), actions = [ octobot_protocol.models.action.Action( id = '', diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py index 558e1bc4b7..d24f384fa0 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py @@ -34,11 +34,13 @@ import octobot_commons.profiles.profile_data as profile_data_module import octobot_commons.profiles.profile_data_import as profile_data_import import octobot_commons.profiles.exchange_auth_data as exchange_auth_data_module +import octobot_commons.profiles.profile as profiles_profile_module import octobot_commons.profiles.tentacles_profile_data_translator as tentacles_profile_data_translator import octobot_commons.enums as commons_enums import octobot_commons.configuration import octobot.constants as octobot_constants +import octobot.community.supabase_backend.enums as community_enums import octobot_flow.entities as octobot_flow_entities import octobot_flow.entities.accounts.process_bot_state as process_bot_state_import import octobot_node.constants as octobot_node_constants @@ -48,6 +50,7 @@ DSL_PREPARED_MARKER = ".octobot_dsl_prepared" DEFAULT_PING_WAITING_TIME = 2.0 DEFAULT_ENSURE_TIMEOUT = 120.0 +DEFAULT_DSL_PROFILE_ID = "non-trading" class EnsureOctobotProcessState(pydantic.BaseModel): @@ -132,6 +135,16 @@ def _remove_path_for_fresh_start(path: str, *, logger: typing.Any) -> None: shutil.rmtree(path, ignore_errors=True) +def _profile_translator_additional_data( + profile_data: profile_data_module.ProfileData, +) -> dict: + return { + community_enums.BotConfigKeys.IS_SIMULATED.value: bool( + profile_data.trader_simulator.enabled + ), + } + + async def _convert_profile_data_to_profile_directory( profile_data: profile_data_module.ProfileData, temp_profile_path: str, @@ -144,7 +157,12 @@ async def _convert_profile_data_to_profile_directory( # apply it to the profile data await tentacles_profile_data_translator.TentaclesProfileDataTranslator( profile_data, [] - ).translate(tentacles_snapshot, {}, None, None) + ).translate( + tentacles_snapshot, + _profile_translator_additional_data(profile_data), + None, + None, + ) except KeyError: # no translator found, restore tentacles profile_data.tentacles = tentacles_snapshot @@ -206,10 +224,84 @@ def _write_user_root_config_json( json_util.safe_dump(default_cfg, config_path) +def _executor_non_trading_profile_source(working_directory: str) -> str: + return os.path.normpath( + os.path.join( + working_directory, + commons_constants.USER_FOLDER, + commons_constants.PROFILES_FOLDER, + DEFAULT_DSL_PROFILE_ID, + ) + ) + + +def _executor_profiles_directory(working_directory: str) -> str: + return os.path.normpath( + os.path.join( + working_directory, + commons_constants.USER_FOLDER, + commons_constants.PROFILES_FOLDER, + ) + ) + + +async def _copy_read_only_profiles_to_user_root( + working_directory: str, + user_root: str, + *, + active_profile_id: str, +) -> None: + """ + Copy read-only profiles from the master OctoBot into a generic process child layout. + + Generic process bots start on the default non-trading profile but should still see + the same read-only strategy profiles as the master (community/imported templates). + Editable profiles are intentionally omitted so each child keeps its own user edits. + """ + profiles_src = _executor_profiles_directory(working_directory) + if not os.path.isdir(profiles_src): + return + for profile in profiles_profile_module.Profile.get_all_profiles(profiles_src): + if not profile.read_only: + continue + # Active profile was already copied by _copy_non_trading_profile_to_user_root. + if profile.profile_id == active_profile_id: + continue + destination_profile_path = os.path.join( + user_root, + commons_constants.PROFILES_FOLDER, + profile.profile_id, + ) + if os.path.exists(destination_profile_path): + shutil.rmtree(destination_profile_path) + shutil.copytree(profile.path, destination_profile_path) + + +async def _copy_non_trading_profile_to_user_root( + working_directory: str, + user_root: str, +) -> str: + source_profile_path = _executor_non_trading_profile_source(working_directory) + if not os.path.isdir(source_profile_path): + raise commons_errors.DSLInterpreterError( + f"Default profile not found at {source_profile_path!r}; expected " + f"{DEFAULT_DSL_PROFILE_ID!r} under the OctoBot user profiles folder." + ) + destination_profile_path = os.path.join( + user_root, + commons_constants.PROFILES_FOLDER, + DEFAULT_DSL_PROFILE_ID, + ) + if os.path.exists(destination_profile_path): + shutil.rmtree(destination_profile_path) + shutil.copytree(source_profile_path, destination_profile_path) + return DEFAULT_DSL_PROFILE_ID + + async def ensure_user_profile_and_layout( user_folder: str, working_directory: str, - profile_data_dict: dict, + profile_data_dict: dict | None, source_reference_tentacles_config: str | None, exchange_auth_data: typing.Optional[ list[exchange_auth_data_module.ExchangeAuthData] @@ -243,32 +335,51 @@ async def ensure_user_profile_and_layout( } os.makedirs(user_root, exist_ok=True) - # Import writes to a throwaway folder first: the real profile id is assigned during import (see rename below). - temp_profile_path = os.path.join( - user_root, - commons_constants.PROFILES_FOLDER, - f"_dsl_tmp_{uuid.uuid4().hex}", - ) - os.makedirs(os.path.dirname(temp_profile_path), exist_ok=True) - profile_data = profile_data_module.ProfileData.from_dict(profile_data_dict) - await _convert_profile_data_to_profile_directory( - profile_data, temp_profile_path - ) + if profile_data_dict is None: + # Generic process: default non-trading profile plus master's read-only profiles. + profile_id = await _copy_non_trading_profile_to_user_root( + working_directory, + user_root, + ) + await _copy_read_only_profiles_to_user_root( + working_directory, + user_root, + active_profile_id=profile_id, + ) + _write_user_root_config_json( + config_path, + profile_id, + None, + exchange_auth_data, + ) + else: + # Import writes to a throwaway folder first: the real profile id is assigned during import (see rename below). + temp_profile_path = os.path.join( + user_root, + commons_constants.PROFILES_FOLDER, + f"_dsl_tmp_{uuid.uuid4().hex}", + ) + os.makedirs(os.path.dirname(temp_profile_path), exist_ok=True) - profile_file = os.path.join(temp_profile_path, commons_constants.PROFILE_CONFIG_FILE) - profile_on_disk = json_util.read_file(profile_file) - profile_id = profile_on_disk[commons_constants.CONFIG_PROFILE][commons_constants.CONFIG_ID] - # OctoBot expects each profile under profiles//; move the temp tree to that name. - final_profile_path = os.path.join( - user_root, commons_constants.PROFILES_FOLDER, profile_id - ) - if os.path.normpath(temp_profile_path) != os.path.normpath(final_profile_path): - if os.path.exists(final_profile_path): - shutil.rmtree(final_profile_path) - os.replace(temp_profile_path, final_profile_path) + profile_data = profile_data_module.ProfileData.from_dict(profile_data_dict) + await _convert_profile_data_to_profile_directory( + profile_data, temp_profile_path + ) - _write_user_root_config_json(config_path, profile_id, profile_data, exchange_auth_data) + profile_file = os.path.join(temp_profile_path, commons_constants.PROFILE_CONFIG_FILE) + profile_on_disk = json_util.read_file(profile_file) + profile_id = profile_on_disk[commons_constants.CONFIG_PROFILE][commons_constants.CONFIG_ID] + # OctoBot expects each profile under profiles//; move the temp tree to that name. + final_profile_path = os.path.join( + user_root, commons_constants.PROFILES_FOLDER, profile_id + ) + if os.path.normpath(temp_profile_path) != os.path.normpath(final_profile_path): + if os.path.exists(final_profile_path): + shutil.rmtree(final_profile_path) + os.replace(temp_profile_path, final_profile_path) + + _write_user_root_config_json(config_path, profile_id, profile_data, exchange_auth_data) # Mirror default reference tentacles layout expected by the child. ref_src = source_reference_tentacles_config or os.path.join( @@ -279,7 +390,7 @@ async def ensure_user_profile_and_layout( if os.path.isdir(ref_src): if os.path.exists(ref_dst): shutil.rmtree(ref_dst) - await asyncio.to_thread(shutil.copytree, ref_src, ref_dst) + shutil.copytree(ref_src, ref_dst) else: os.makedirs(ref_dst, exist_ok=True) @@ -389,7 +500,7 @@ class EnsureOctobotProcessOperator( "If the state file never becomes live before ping_timeout from the first spawn, the keyword fails and the child is killed." ) EXAMPLE = ( - "run_octobot_process(user_folder='bots/b1', profile_data={...}, " + "run_octobot_process(user_folder='bots/b1', " "exchange_auth_data=[{'internal_name': 'binance', 'api_key': '...', 'api_secret': '...'}], " "last_execution_result=None)" ) @@ -420,9 +531,14 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: ), dsl_interpreter.OperatorParameter( name="profile_data", - description="Object compatible with octobot_commons.profiles.profile_data.ProfileData.", - required=True, + description=( + "Optional object compatible with octobot_commons.profiles.profile_data.ProfileData. " + "When omitted, the child uses the packaged default config and copies the " + f"{DEFAULT_DSL_PROFILE_ID!r} profile from the executor user profiles folder." + ), + required=False, type=dict, + default=None, ), dsl_interpreter.OperatorParameter( name="exchange_auth_data", @@ -640,7 +756,7 @@ async def _pre_compute_first_spawn( init_info = await ensure_user_profile_and_layout( user_folder, working_directory, - params["profile_data"], + params.get("profile_data"), None, exchange_auth, ) diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py index 5a1f25c29b..88c9aaa693 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py @@ -17,7 +17,11 @@ import os import pathlib import shutil +import socket import sys +import asyncio +import time +import uuid import mock import pytest @@ -44,6 +48,7 @@ # Nested class from factory (not exposed on ``octobot_process_ops``). EnsureOctobotProcessOperator = octobot_process_ops.create_octobot_process_operators(None)[0] +_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SEC = 2 pytestmark = pytest.mark.asyncio @@ -133,6 +138,130 @@ def _re_calling_ensure_value(last_execution_result: dict) -> dict: } +def _octobot_project_root_from_test_file() -> pathlib.Path: + return pathlib.Path(__file__).resolve().parents[6] + + +def _require_octobot_project_root_for_subprocess_tests() -> str: + project_root = str(_octobot_project_root_from_test_file()) + start_script = os.path.join(project_root, "start.py") + if not os.path.isfile(start_script): + pytest.skip("start.py missing: run pytest with cwd set to the OctoBot project root") + non_trading_profile_json = os.path.join( + project_root, + commons_constants.USER_FOLDER, + commons_constants.PROFILES_FOLDER, + octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + commons_constants.PROFILE_CONFIG_FILE, + ) + if not os.path.isfile(non_trading_profile_json): + pytest.skip( + f"{octobot_process_ops.DEFAULT_DSL_PROFILE_ID!r} profile missing under OctoBot user/profiles" + ) + return project_root + + +def _seed_executor_non_trading_profile(working_directory: pathlib.Path) -> None: + source_profile_path = _octobot_project_root_from_test_file().joinpath( + commons_constants.USER_FOLDER, + commons_constants.PROFILES_FOLDER, + octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + ) + if source_profile_path.is_dir(): + destination_profile_path = working_directory.joinpath( + commons_constants.USER_FOLDER, + commons_constants.PROFILES_FOLDER, + octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + ) + destination_profile_path.parent.mkdir(parents=True, exist_ok=True) + if destination_profile_path.exists(): + shutil.rmtree(destination_profile_path) + shutil.copytree(source_profile_path, destination_profile_path) + return + minimal_profile_path = working_directory.joinpath( + commons_constants.USER_FOLDER, + commons_constants.PROFILES_FOLDER, + octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + ) + minimal_profile_path.mkdir(parents=True, exist_ok=True) + profile_payload = { + commons_constants.CONFIG_PROFILE: { + commons_constants.CONFIG_ID: octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + commons_constants.CONFIG_NAME: octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + } + } + (minimal_profile_path / commons_constants.PROFILE_CONFIG_FILE).write_text( + json.dumps(profile_payload), + encoding="utf-8", + ) + + +def _seed_executor_profile( + working_directory: pathlib.Path, + profile_id: str, + *, + read_only: bool, +) -> None: + profile_path = working_directory.joinpath( + commons_constants.USER_FOLDER, + commons_constants.PROFILES_FOLDER, + profile_id, + ) + profile_path.mkdir(parents=True, exist_ok=True) + profile_config = { + commons_constants.CONFIG_ID: profile_id, + commons_constants.CONFIG_NAME: profile_id, + } + if read_only: + profile_config[commons_constants.CONFIG_READ_ONLY] = True + profile_payload = { + commons_constants.CONFIG_PROFILE: profile_config, + commons_constants.PROFILE_CONFIG: {}, + } + (profile_path / commons_constants.PROFILE_CONFIG_FILE).write_text( + json.dumps(profile_payload), + encoding="utf-8", + ) + + +def _recall_inner_from_interpreter_result(result: dict) -> dict | None: + rec = result.get(dsl_interpreter.ReCallingOperatorResult.__name__) + if not isinstance(rec, dict): + return None + inner = rec.get("last_execution_result") + return inner if isinstance(inner, dict) else None + + +async def _poll_dsl_until_init_state_ok( + interpreter: dsl_interpreter.Interpreter, + user_folder: str, + exchange_auth_list: list[dict], + *, + timeout_sec: float = 60.0, +) -> dict: + base_arguments = ( + f"{user_folder!r}, exchange_auth_data={repr(exchange_auth_list)}, " + f"waiting_time={_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SEC}, ping_timeout=30.0" + ) + deadline = time.monotonic() + timeout_sec + last_full_result: dict | None = None + while time.monotonic() < deadline: + if last_full_result is None: + expression = f"run_octobot_process({base_arguments})" + else: + expression = ( + f"run_octobot_process({base_arguments}, " + f"last_execution_result={repr(last_full_result)})" + ) + last_full_result = await interpreter.interprete(expression) + assert isinstance(last_full_result, dict) + inner = _recall_inner_from_interpreter_result(last_full_result) + if inner and inner.get("init_state_ok") is True: + return inner + await asyncio.sleep(2.0) + pytest.fail(f"OctoBot did not become ready (init_state_ok) within {timeout_sec}s") + + def _fresh_default_like_cfg_template(): """Minimal dict shaped like packaged ``default_config.json`` for isolated ``read_file`` mocks.""" return { @@ -293,6 +422,163 @@ async def test_marked_prepared_is_skipped(self, tmp_path): assert res["profile_id"] == "p1" +class TestEnsureOctobotProcessOperatorProfileDataOptional: + def test_declares_optional_profile_data_parameter(self): + params = EnsureOctobotProcessOperator.get_parameters() + profile_parameter = next( + (parameter for parameter in params if parameter.name == "profile_data"), + None, + ) + assert profile_parameter is not None + assert profile_parameter.required is False + assert profile_parameter.default is None + + +class TestCopyReadOnlyProfilesToUserRoot: + async def test_copies_read_only_profiles_and_skips_editable(self, tmp_path): + _seed_executor_non_trading_profile(tmp_path) + readonly_profile_id = "readonly_strategy" + editable_profile_id = "editable_strategy" + _seed_executor_profile(tmp_path, readonly_profile_id, read_only=True) + _seed_executor_profile(tmp_path, editable_profile_id, read_only=False) + user_root = tmp_path / "child_user_root" + user_root.mkdir() + await octobot_process_ops._copy_read_only_profiles_to_user_root( + str(tmp_path), + str(user_root), + active_profile_id=octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + ) + profiles_root = user_root / commons_constants.PROFILES_FOLDER + readonly_profile_json = ( + profiles_root / readonly_profile_id / commons_constants.PROFILE_CONFIG_FILE + ) + editable_profile_json = ( + profiles_root / editable_profile_id / commons_constants.PROFILE_CONFIG_FILE + ) + non_trading_profile_json = ( + profiles_root + / octobot_process_ops.DEFAULT_DSL_PROFILE_ID + / commons_constants.PROFILE_CONFIG_FILE + ) + assert readonly_profile_json.is_file() + assert not editable_profile_json.exists() + assert not non_trading_profile_json.exists() + + async def test_skips_active_profile_id(self, tmp_path): + _seed_executor_profile( + tmp_path, + octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + read_only=True, + ) + user_root = tmp_path / "child_user_root" + user_root.mkdir() + destination_profile_path = ( + user_root + / commons_constants.PROFILES_FOLDER + / octobot_process_ops.DEFAULT_DSL_PROFILE_ID + ) + destination_profile_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree( + tmp_path.joinpath( + commons_constants.USER_FOLDER, + commons_constants.PROFILES_FOLDER, + octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + ), + destination_profile_path, + ) + profile_json_path = destination_profile_path / commons_constants.PROFILE_CONFIG_FILE + original_mtime = profile_json_path.stat().st_mtime + await octobot_process_ops._copy_read_only_profiles_to_user_root( + str(tmp_path), + str(user_root), + active_profile_id=octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + ) + assert profile_json_path.stat().st_mtime == original_mtime + + +class TestEnsureUserProfileAndLayoutDefaultProfile: + async def test_copies_non_trading_profile_and_writes_default_config(self, tmp_path): + _seed_executor_non_trading_profile(tmp_path) + user_leaf = "default_profile_layout_user" + result = await octobot_process_ops.ensure_user_profile_and_layout( + user_leaf, + str(tmp_path), + None, + None, + None, + ) + assert result["already_prepared"] is False + assert result["profile_id"] == octobot_process_ops.DEFAULT_DSL_PROFILE_ID + user_root = pathlib.Path(result["user_root"]) + profile_json_path = ( + user_root + / commons_constants.PROFILES_FOLDER + / octobot_process_ops.DEFAULT_DSL_PROFILE_ID + / commons_constants.PROFILE_CONFIG_FILE + ) + assert profile_json_path.is_file() + root_config_path = user_root / commons_constants.CONFIG_FILE + root_cfg = json.loads(root_config_path.read_text(encoding="utf-8")) + assert root_cfg[commons_constants.CONFIG_PROFILE] == octobot_process_ops.DEFAULT_DSL_PROFILE_ID + assert ( + root_cfg[services_constants.CONFIG_CATEGORY_SERVICES][services_constants.CONFIG_WEB][ + services_constants.CONFIG_AUTO_OPEN_IN_WEB_BROWSER + ] + is False + ) + assert root_cfg[commons_constants.CONFIG_ACCEPTED_TERMS] is True + + async def test_copies_read_only_profiles_on_default_layout(self, tmp_path): + _seed_executor_non_trading_profile(tmp_path) + readonly_profile_id = "readonly_strategy" + _seed_executor_profile(tmp_path, readonly_profile_id, read_only=True) + user_leaf = "default_layout_with_readonly_profiles" + result = await octobot_process_ops.ensure_user_profile_and_layout( + user_leaf, + str(tmp_path), + None, + None, + None, + ) + user_root = pathlib.Path(result["user_root"]) + profiles_root = user_root / commons_constants.PROFILES_FOLDER + assert ( + profiles_root + / octobot_process_ops.DEFAULT_DSL_PROFILE_ID + / commons_constants.PROFILE_CONFIG_FILE + ).is_file() + assert ( + profiles_root / readonly_profile_id / commons_constants.PROFILE_CONFIG_FILE + ).is_file() + + async def test_applies_exchange_auth_without_profile_data(self, tmp_path): + _seed_executor_non_trading_profile(tmp_path) + exchange_internal_name = "default_layout_exchange" + exchange_auth_list = [ + exchange_auth_data_module.ExchangeAuthData( + internal_name=exchange_internal_name, + api_key="layout-key", + api_secret="layout-secret", + exchange_type=commons_constants.CONFIG_EXCHANGE_SPOT, + sandboxed=True, + ) + ] + result = await octobot_process_ops.ensure_user_profile_and_layout( + "default_exchange_user", + str(tmp_path), + None, + None, + exchange_auth_list, + ) + user_root = pathlib.Path(result["user_root"]) + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + exchange_cfg = root_cfg[commons_constants.CONFIG_EXCHANGES][exchange_internal_name] + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_KEY] == "layout-key" + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_SECRET] == "layout-secret" + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_TYPE] == commons_constants.CONFIG_EXCHANGE_SPOT + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_SANDBOXED] is True + + class TestEnsureUserProfileAndLayoutFunctional: async def test_writes_profile_tree_top_level_config_and_exchange_credentials(self, tmp_path): exchange_internal_name = "functional_exchange_okx" @@ -452,7 +738,7 @@ async def test_calls_translator_then_convert_when_tentacles_present(self, tmp_pa ) translator_class_mock.assert_called_once_with(profile_data, []) mock_translator.translate.assert_awaited_once_with( - expected_snapshot, {}, None, None + expected_snapshot, {"is_simulated": True}, None, None ) convert_mock.assert_awaited_once() @@ -533,6 +819,17 @@ def test_finds_sequential_ports(self): assert os_util.tcp_port_is_free("127.0.0.1", web_port) assert os_util.tcp_port_is_free("127.0.0.1", node_port) + def test_skips_port_occupied_on_host(self): + node_port_base = 35000 + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", 0)) + occupied_port = listener.getsockname()[1] + listener.listen(1) + web_port, _node_port = octobot_process_ops._listen_port_pair_with_shared_scan_offset( + "127.0.0.1", occupied_port, node_port_base, max_offset=10 + ) + assert web_port != occupied_port + class TestEnsureOctobotProcessOperatorExchangeAuthData: def test_declares_optional_exchange_auth_parameter(self): @@ -1096,6 +1393,202 @@ async def test_run_octobot_process_via_dsl_writes_exchange_auth_into_user_config if (tmp_path / "logs").exists(): shutil.rmtree(tmp_path / "logs", ignore_errors=True) + async def test_run_octobot_process_via_dsl_without_profile_data_accepts_exchange_auth( + self, tmp_path, monkeypatch + ): + monkeypatch.chdir(tmp_path) + (tmp_path / "start.py").write_text("#", encoding="utf-8") + _seed_executor_non_trading_profile(tmp_path) + user_folder = "integration_dsl_no_profile_bot" + exchange_internal_name = "dsl_no_profile_exchange" + exchange_auth_list = [ + { + "internal_name": exchange_internal_name, + "api_key": "no-profile-key", + "api_secret": "no-profile-secret", + "exchange_type": commons_constants.CONFIG_EXCHANGE_SPOT, + "sandboxed": True, + } + ] + expression = ( + f"run_octobot_process({user_folder!r}, exchange_auth_data={repr(exchange_auth_list)}, " + f"waiting_time={_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SEC}, ping_timeout=30.0)" + ) + interpreter = dsl_interpreter.Interpreter( + dsl_interpreter.get_all_operators() + + [EnsureOctobotProcessOperator], + ) + try: + with mock.patch.object( + octobot_process_ops, + "_load_process_bot_state", + new=mock.AsyncMock(side_effect=_async_return_none_mock), + ), mock.patch.object( + process_util, + "spawn_managed_subprocess", + ) as spawn_mock: + spawn_mock.return_value = mock.Mock(spec=["pid"], pid=54321) + result = await interpreter.interprete(expression) + assert isinstance(result, dict) + assert dsl_interpreter.ReCallingOperatorResult.__name__ in result + user_data_root = ( + tmp_path + / commons_constants.USER_FOLDER + / commons_constants.AUTOMATIONS_FOLDER + / user_folder + ) + root_config_path = user_data_root / commons_constants.CONFIG_FILE + assert root_config_path.is_file() + written_root_cfg = json.loads(root_config_path.read_text(encoding="utf-8")) + assert written_root_cfg[commons_constants.CONFIG_PROFILE] == octobot_process_ops.DEFAULT_DSL_PROFILE_ID + profile_json_path = ( + user_data_root + / commons_constants.PROFILES_FOLDER + / octobot_process_ops.DEFAULT_DSL_PROFILE_ID + / commons_constants.PROFILE_CONFIG_FILE + ) + assert profile_json_path.is_file() + exchange_cfg = written_root_cfg[commons_constants.CONFIG_EXCHANGES][exchange_internal_name] + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_KEY] == "no-profile-key" + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_SECRET] == "no-profile-secret" + finally: + shutil.rmtree(tmp_path / commons_constants.USER_FOLDER, ignore_errors=True) + if (tmp_path / "logs").exists(): + shutil.rmtree(tmp_path / "logs", ignore_errors=True) + + +class TestRunOctobotProcessDefaultConfigSubprocess: + async def _run_default_config_lifecycle( + self, + *, + project_root: str, + exchange_auth_list: list[dict], + user_folder_suffix: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.chdir(project_root) + monkeypatch.setenv(octobot_constants.ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS, "5") + user_folder = f"unit_tests/default_cfg_{user_folder_suffix}_{uuid.uuid4().hex[:10]}" + interpreter = dsl_interpreter.Interpreter( + dsl_interpreter.get_all_operators() + + [EnsureOctobotProcessOperator], + ) + user_root_guess = os.path.normpath( + os.path.join( + project_root, + *commons_constants.USER_AUTOMATIONS_FOLDER.split("/"), + *user_folder.replace("\\", "/").split("/"), + ) + ) + log_folder_guess = os.path.normpath( + os.path.join( + project_root, + *octobot_node_constants.AUTOMATION_LOGS_FOLDER.split("/"), + *[segment for segment in user_folder.replace("\\", "/").split("/") if segment], + ) + ) + child_pid: int | None = None + try: + inner = await _poll_dsl_until_init_state_ok( + interpreter, + user_folder, + exchange_auth_list, + timeout_sec=90.0, + ) + assert inner.get("pid") + child_pid = int(inner["pid"]) + assert process_util.pid_is_running(child_pid) + user_root = pathlib.Path(inner["user_root"]) + assert inner.get("profile_id") == octobot_process_ops.DEFAULT_DSL_PROFILE_ID + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + exchange_internal_name = exchange_auth_list[0]["internal_name"] + assert exchange_internal_name in root_cfg[commons_constants.CONFIG_EXCHANGES] + exchange_cfg = root_cfg[commons_constants.CONFIG_EXCHANGES][exchange_internal_name] + if exchange_auth_list[0].get("api_key"): + stored_api_key = exchange_cfg[commons_constants.CONFIG_EXCHANGE_KEY] + stored_api_secret = exchange_cfg[commons_constants.CONFIG_EXCHANGE_SECRET] + assert stored_api_key != exchange_auth_list[0]["api_key"] + assert stored_api_secret != exchange_auth_list[0]["api_secret"] + assert isinstance(stored_api_key, str) and stored_api_key.startswith("gAAAAA") + assert isinstance(stored_api_secret, str) and stored_api_secret.startswith("gAAAAA") + else: + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_SANDBOXED] is exchange_auth_list[0]["sandboxed"] + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_TYPE] == exchange_auth_list[0]["exchange_type"] + profile_json_path = ( + user_root + / commons_constants.PROFILES_FOLDER + / octobot_process_ops.DEFAULT_DSL_PROFILE_ID + / commons_constants.PROFILE_CONFIG_FILE + ) + assert profile_json_path.is_file() + stop_expression = ( + f"run_octobot_process({user_folder!r}, exchange_auth_data={repr(exchange_auth_list)}, " + f"waiting_time={_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SEC}, ping_timeout=30.0, " + f"last_execution_result={repr(_re_calling_ensure_value(inner))})" + ) + operator_signals_holder = dsl_interpreter.OperatorSignals() + stop_operator_cls = octobot_process_ops.create_octobot_process_operators( + operator_signals_holder + )[0] + operator_signals_holder.sync({ + stop_operator_cls.get_name(): dsl_interpreter.OperatorSignal.STOP.value, + }) + stop_interpreter = dsl_interpreter.Interpreter( + dsl_interpreter.get_all_operators() + + [stop_operator_cls], + ) + stop_result = await stop_interpreter.interprete(stop_expression) + assert isinstance(stop_result, dict) + assert stop_result.get("status") in ("stopped", "already_stopped") + process_deadline = time.monotonic() + 30.0 + while time.monotonic() < process_deadline: + if not process_util.pid_is_running(child_pid): + break + await asyncio.sleep(0.5) + else: + pytest.fail(f"expected child pid {child_pid} to exit after STOP within 30s") + finally: + if child_pid is not None and process_util.pid_is_running(child_pid): + process_util.request_graceful_stop_via_sigterm(child_pid) + if os.path.isdir(user_root_guess): + shutil.rmtree(user_root_guess, ignore_errors=True) + if os.path.isdir(log_folder_guess): + shutil.rmtree(log_folder_guess, ignore_errors=True) + + async def test_simulated_bot_without_profile_data(self, monkeypatch): + project_root = _require_octobot_project_root_for_subprocess_tests() + exchange_auth_list = [ + { + "internal_name": "binanceus", + "sandboxed": True, + "exchange_type": commons_constants.CONFIG_EXCHANGE_SPOT, + } + ] + await self._run_default_config_lifecycle( + project_root=project_root, + exchange_auth_list=exchange_auth_list, + user_folder_suffix="simulated", + monkeypatch=monkeypatch, + ) + + async def test_real_bot_without_profile_data(self, monkeypatch): + project_root = _require_octobot_project_root_for_subprocess_tests() + exchange_auth_list = [ + { + "internal_name": "binanceus", + "api_key": "functional-test-api-key", + "api_secret": "functional-test-api-secret", + "sandboxed": True, + "exchange_type": commons_constants.CONFIG_EXCHANGE_SPOT, + } + ] + await self._run_default_config_lifecycle( + project_root=project_root, + exchange_auth_list=exchange_auth_list, + user_folder_suffix="real_creds", + monkeypatch=monkeypatch, + ) + class TestEnsureOctobotProcessOperatorExecutionStop: async def test_execution_stop_dead_child_is_already_stopped(self): diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/user-action-templates.test.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/user-action-templates.test.ts index e89680b062..79d083bd7d 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/user-action-templates.test.ts +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/user-action-templates.test.ts @@ -138,6 +138,22 @@ describe("buildUserActionTemplate", () => { expect(config.rebalance_trigger_min_percent).toBe(5.0) }) + it("builds a copy strategy create template", () => { + const action = buildUserActionTemplate("strategy_create_copy") + expect(action.configuration).toMatchObject({ + action_type: "strategy_create", + }) + + const strategy = ( + action.configuration as { configuration: Record } + ).configuration + expect(strategy.reference_market).toBe("USDT") + + const copyConfiguration = strategy.configuration as Record + expect(copyConfiguration.configuration_type).toBe("copy") + expect(copyConfiguration.strategy_id).toBe("") + }) + it("builds a DCA strategy create template with two evaluators", () => { const action = buildUserActionTemplate("strategy_create_dca") expect(action.id).toBe("ua-manual-strategy_create_dca") @@ -232,6 +248,27 @@ describe("buildUserActionTemplate", () => { expect(pairSettings[0].min_spread).toBe(5) expect(pairSettings[0].max_spread).toBe(20) }) + + it("builds a generic process OctoBot strategy create template", () => { + const action = buildUserActionTemplate("strategy_create_generic_process") + expect(action.id).toBe("ua-manual-strategy_create_generic_process") + expect(action.configuration).toMatchObject({ + action_type: "strategy_create", + }) + + const strategy = ( + action.configuration as { configuration: Record } + ).configuration + expect(strategy.reference_market).toBe("USDC") + expect(strategy.id).toMatch(CANONICAL_UUID_V4_PATTERN) + + const genericProcessConfiguration = strategy.configuration as Record< + string, + unknown + > + expect(genericProcessConfiguration.configuration_type).toBe("generic_process") + expect(genericProcessConfiguration).not.toHaveProperty("profile_data") + }) }) describe("buildUserActionTemplateJson", () => { diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/user-action-templates.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/user-action-templates.ts index c363742faf..a2d5339e4b 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/user-action-templates.ts +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/user-action-templates.ts @@ -50,6 +50,7 @@ export type UserActionTemplateKey = | "strategy_create_dca" | "strategy_create_dca_always_long" | "strategy_create_market_making" + | "strategy_create_generic_process" export const DEFAULT_USER_ACTION_TEMPLATE_KEY: UserActionTemplateKey = DEFAULT_USER_ACTION_TYPE @@ -88,6 +89,10 @@ export const USER_ACTION_TEMPLATE_OPTIONS: { value: "strategy_create_market_making", label: "Strategy create (market making)", }, + { + value: "strategy_create_generic_process", + label: "Strategy create (generic process OctoBot)", + }, { value: "strategy_edit", label: "Strategy edit" }, { value: "strategy_delete", label: "Strategy delete" }, ] @@ -318,6 +323,19 @@ function sampleGenericProcessStrategyConfiguration( } satisfies GenericProcessConfiguration) } +function sampleGenericProcessOctobotStrategyConfiguration( + id = "", +): Strategy { + return sampleStrategyShell( + id, + "Generic process OctoBot strategy", + { + configuration_type: "generic_process", + } satisfies GenericProcessConfiguration, + "USDC", + ) +} + function sampleGridPairSettings( symbol: string, flatSpread: number, @@ -570,6 +588,13 @@ export function buildUserActionTemplate( } satisfies CreateStrategyConfiguration) } + if (templateKey === "strategy_create_generic_process") { + return userAction("ua-manual-strategy_create_generic_process", { + action_type: "strategy_create", + configuration: sampleGenericProcessOctobotStrategyConfiguration(newResourceId()), + } satisfies CreateStrategyConfiguration) + } + const actionType: UserActionType = templateKey const id = `ua-manual-${actionType}` diff --git a/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/simple_market_making_profile_data_adapter.py b/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/simple_market_making_profile_data_adapter.py index 4a1fc4bf5a..8e562731c7 100644 --- a/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/simple_market_making_profile_data_adapter.py +++ b/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/simple_market_making_profile_data_adapter.py @@ -173,10 +173,7 @@ async def adapt( profile_data.trader.enabled = not is_simulated profile_data.trader_simulator.enabled = is_simulated - # should not happen in real environment if is_simulated: - if not octobot_commons.constants.IS_DEV_MODE_ENABLED: - raise ValueError("Simulator configuration is not supported") max_funds_by_symbol = {} for pair_config in pair_configs: parsed_pair = symbols_util.parse_symbol( diff --git a/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/simple_market_making_trading.py b/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/simple_market_making_trading.py index 8b5a82ae8d..48dfae9b90 100644 --- a/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/simple_market_making_trading.py +++ b/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/simple_market_making_trading.py @@ -411,14 +411,19 @@ def get_scheduled_volume_config( cls, symbol_trading_config: dict ): try: - return symbol_trading_config[cls.SCHEDULED_VOLUME] + raw_config = symbol_trading_config[cls.SCHEDULED_VOLUME] + if not raw_config: + return {} except KeyError: return {} + return raw_config @classmethod def get_hedging_engine_config(cls, symbol_trading_config: dict) -> dict[str, typing.Any]: try: raw_config = symbol_trading_config[cls.HEDGING_ENGINE] + if not raw_config: + return {} except KeyError: return {} hedging_config = dict(raw_config) @@ -787,11 +792,7 @@ async def _initialize_hedging_engine(self): self.logger.info("Disabled hedging engine: no hedging engine config") async def _initialize_scheduled_volume(self): - try: - schedule_config = self.trading_mode.get_scheduled_volume_config(self.symbol_trading_config) - except KeyError: - self.logger.error("Skipped scheduled volume: no scheduled volume config") - return + schedule_config = self.trading_mode.get_scheduled_volume_config(self.symbol_trading_config) if max_amount := schedule_config.get(self.trading_mode.MAX_AMOUNT, 0): if self._hedging_engine is not None: self.logger.error( diff --git a/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/tests/test_simple_market_making_trading.py b/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/tests/test_simple_market_making_trading.py index 7d3371abf1..2bdb78f65e 100644 --- a/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/tests/test_simple_market_making_trading.py +++ b/packages/tentacles/Trading/Mode/simple_market_making_trading_mode/tests/test_simple_market_making_trading.py @@ -245,6 +245,33 @@ def test_does_not_mutate_original_hedging_dict(self): assert cls.AVERAGE_PRICE_COUNTED_MINUTES not in inner +class TestGetScheduledVolumeConfig: + def test_returns_empty_dict_when_key_missing(self): + cls = simple_market_making_trading.SimpleMarketMakingTradingMode + assert cls.get_scheduled_volume_config({}) == {} + + def test_returns_empty_dict_when_value_is_none(self): + cls = simple_market_making_trading.SimpleMarketMakingTradingMode + symbol_cfg = {cls.SCHEDULED_VOLUME: None} + assert cls.get_scheduled_volume_config(symbol_cfg) == {} + + def test_returns_empty_dict_when_value_is_empty_object(self): + cls = simple_market_making_trading.SimpleMarketMakingTradingMode + symbol_cfg = {cls.SCHEDULED_VOLUME: {}} + assert cls.get_scheduled_volume_config(symbol_cfg) == {} + + def test_returns_config_when_scheduled_volume_is_set(self): + cls = simple_market_making_trading.SimpleMarketMakingTradingMode + scheduled_volume_cfg = { + cls.MIN_INTERVAL_SECONDS: 1, + cls.MAX_INTERVAL_SECONDS: 2, + cls.MIN_AMOUNT: 3, + cls.MAX_AMOUNT: 4, + } + symbol_cfg = {cls.SCHEDULED_VOLUME: scheduled_volume_cfg} + assert cls.get_scheduled_volume_config(symbol_cfg) == scheduled_volume_cfg + + @pytest.mark.parametrize( "hedging_exchange_case", ( @@ -604,6 +631,25 @@ async def test_start(): assert producer.healthy is True assert producer.should_stop is False + producer.healthy = True + pair_config[simple_market_making_trading.SimpleMarketMakingTradingMode.SCHEDULED_VOLUME] = None + await producer.start() + mm_start_mock.assert_not_called() + producer_start_mock.assert_called_once() + scheduled_volume_start_mock.assert_not_called() + advanced_reference_price_initialize_mock.assert_awaited_once() + _ensure_market_making_orders_and_reschedule_mock.assert_called_once() + schedule_bot_stop_mock.assert_not_called() + assert producer._scheduled_volume is None + mm_start_mock.reset_mock() + producer_start_mock.reset_mock() + scheduled_volume_start_mock.reset_mock() + advanced_reference_price_initialize_mock.reset_mock() + _ensure_market_making_orders_and_reschedule_mock.reset_mock() + schedule_bot_stop_mock.reset_mock() + assert producer.healthy is True + assert producer.should_stop is False + producer.healthy = True # disabled config pair_config[simple_market_making_trading.SimpleMarketMakingTradingMode.SCHEDULED_VOLUME] = {