From 770513e299cee27e80f4d4ca8045c300228ad509 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Thu, 28 May 2026 15:22:50 +0100 Subject: [PATCH 01/11] Initial implementation of debugging --- pyproject.toml | 1 + src/mx_bluesky/common/utils/debug.py | 15 +++++++ src/mx_bluesky/hyperion/__main__.py | 3 ++ src/mx_bluesky/hyperion/parameters/cli.py | 29 +++++++++--- tests/unit_tests/common/utils/test_debug.py | 25 +++++++++++ tests/unit_tests/hyperion/test_main_system.py | 45 +++++++++++++++++++ 6 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 src/mx_bluesky/common/utils/debug.py create mode 100644 tests/unit_tests/common/utils/test_debug.py diff --git a/pyproject.toml b/pyproject.toml index 18c13a1a66..ba4e0bb72b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dependencies = [ "deepdiff", "matplotlib", "cachetools", + "debugpy", "bluesky-stomp >= 0.2.0", "mysql-connector-python == 9.5.0", # Can unpin once https://github.com/DiamondLightSource/ispyb-api/pull/244 is merged and released diff --git a/src/mx_bluesky/common/utils/debug.py b/src/mx_bluesky/common/utils/debug.py new file mode 100644 index 0000000000..fbd394f158 --- /dev/null +++ b/src/mx_bluesky/common/utils/debug.py @@ -0,0 +1,15 @@ +from debugpy import listen, wait_for_client + +from mx_bluesky.common.utils.log import LOGGER + + +def enable_debugging(wait_for_attach: bool, port: int): + """Enable debugging via debugpy + Args: + wait_for_attach (bool): True to wait for the debugger to attach + port (int): Port on which to listen.""" + listen(("0.0.0.0", port)) + LOGGER.info(f"Listening for debugger connections on {port}") + if wait_for_attach: + LOGGER.info("Waiting for debugger to attach...") + wait_for_client() diff --git a/src/mx_bluesky/hyperion/__main__.py b/src/mx_bluesky/hyperion/__main__.py index 8ce2c9db82..f20f049f71 100755 --- a/src/mx_bluesky/hyperion/__main__.py +++ b/src/mx_bluesky/hyperion/__main__.py @@ -9,6 +9,7 @@ from mx_bluesky.common.external_interaction.alerting.log_based_service import ( LoggingAlertService, ) +from mx_bluesky.common.utils.debug import enable_debugging from mx_bluesky.common.utils.log import ( LOGGER, do_default_logging_setup, @@ -37,6 +38,8 @@ def initialise_globals(args: HyperionArgs): dev_mode=args.dev_mode, ) LOGGER.info(f"Hyperion launched with args:{argv}") + if args.debug_port: + enable_debugging(args.wait_for_debug_attach, args.debug_port) alerting.set_alerting_service(LoggingAlertService(CONST.GRAYLOG_STREAM_ID)) diff --git a/src/mx_bluesky/hyperion/parameters/cli.py b/src/mx_bluesky/hyperion/parameters/cli.py index b8038992f5..f5bcfcb6fb 100644 --- a/src/mx_bluesky/hyperion/parameters/cli.py +++ b/src/mx_bluesky/hyperion/parameters/cli.py @@ -14,33 +14,46 @@ class HyperionMode(StrEnum): @dataclass -class HyperionArgs: - mode: HyperionMode +class CommonArgs: dev_mode: bool = False + debug_port: int | None = None + wait_for_debug_attach: bool = False + + +@dataclass +class HyperionArgs(CommonArgs): + mode: HyperionMode = HyperionMode.UDC client_config: str | None = None supervisor_config: str | None = None @dataclass -class CallbackArgs: - dev_mode: bool = False +class CallbackArgs(CommonArgs): watchdog_port: int = HyperionConstants.HYPERION_PORT stomp_config: Path | None = None -def _add_callback_relevant_args(parser: argparse.ArgumentParser) -> None: +def _add_common_args(parser: argparse.ArgumentParser) -> None: """adds arguments relevant to hyperion-callbacks.""" parser.add_argument( "--dev", action="store_true", help="Use dev options, such as local graylog instances", ) + parser.add_argument( + "--debug-port", + help="Enable remote debugging with the specified port", + type=int, + ) + parser.add_argument( + "--wait-for-attach", action="store_true", help="Wait for the debugger to attach" + ) def parse_callback_args() -> CallbackArgs: """Parse the CLI arguments for the watchdog port and dev mode into a CallbackArgs instance.""" parser = argparse.ArgumentParser() - _add_callback_relevant_args(parser) + _add_common_args(parser) parser.add_argument( "--watchdog-port", type=int, @@ -65,7 +78,7 @@ def parse_cli_args() -> HyperionArgs: Returns: an HyperionArgs dataclass with the fields: (dev_mode: bool)""" parser = argparse.ArgumentParser() - _add_callback_relevant_args(parser) + _add_common_args(parser) parser.add_argument( "--version", help="Print hyperion version string", @@ -92,4 +105,6 @@ def parse_cli_args() -> HyperionArgs: mode=args.mode, supervisor_config=args.supervisor_config, client_config=args.client_config, + debug_port=args.debug_port, + wait_for_debug_attach=args.wait_for_attach, ) diff --git a/tests/unit_tests/common/utils/test_debug.py b/tests/unit_tests/common/utils/test_debug.py new file mode 100644 index 0000000000..456b3025c7 --- /dev/null +++ b/tests/unit_tests/common/utils/test_debug.py @@ -0,0 +1,25 @@ +from unittest.mock import MagicMock, patch + +from mx_bluesky.common.utils.debug import enable_debugging + + +@patch("mx_bluesky.common.utils.debug.listen") +@patch("mx_bluesky.common.utils.debug.wait_for_client") +def test_enable_debugging_no_wait( + mock_wait_for_client: MagicMock, + mock_listen: MagicMock, +): + enable_debugging(False, 1234) + mock_listen.assert_called_once_with(("0.0.0.0", 1234)) + mock_wait_for_client.assert_not_called() + + +@patch("mx_bluesky.common.utils.debug.listen") +@patch("mx_bluesky.common.utils.debug.wait_for_client") +def test_enable_debugging_wait( + mock_wait_for_client: MagicMock, + mock_listen: MagicMock, +): + enable_debugging(True, 1234) + mock_listen.assert_called_once_with(("0.0.0.0", 1234)) + mock_wait_for_client.assert_called_once() diff --git a/tests/unit_tests/hyperion/test_main_system.py b/tests/unit_tests/hyperion/test_main_system.py index 020c52337d..f859a91a1b 100644 --- a/tests/unit_tests/hyperion/test_main_system.py +++ b/tests/unit_tests/hyperion/test_main_system.py @@ -135,6 +135,51 @@ def test_hyperion_in_udc_mode_starts_logging( ) +@patch("sys.argv", new=["hyperion", "--mode", "udc", "--debug-port", "1234"]) +@patch("mx_bluesky.hyperion.__main__.do_default_logging_setup", MagicMock()) +@patch("mx_bluesky.hyperion.__main__.create_server_for_udc", MagicMock()) +@patch("mx_bluesky.hyperion.__main__.run_forever", MagicMock()) +@patch("mx_bluesky.hyperion.__main__.enable_debugging") +def test_hyperion_can_enable_debugging( + mock_enable_debugging: MagicMock, + mock_setup_context: MagicMock, +): + main() + + mock_enable_debugging.assert_called_once_with(False, 1234) + + +@patch( + "sys.argv", + new=["hyperion", "--mode", "udc", "--debug-port", "1234", "--wait-for-attach"], +) +@patch("mx_bluesky.hyperion.__main__.do_default_logging_setup", MagicMock()) +@patch("mx_bluesky.hyperion.__main__.create_server_for_udc", MagicMock()) +@patch("mx_bluesky.hyperion.__main__.run_forever", MagicMock()) +@patch("mx_bluesky.hyperion.__main__.enable_debugging") +def test_hyperion_can_wait_for_debug_attach( + mock_enable_debugging: MagicMock, + mock_setup_context: MagicMock, +): + main() + + mock_enable_debugging.assert_called_once_with(True, 1234) + + +@patch("sys.argv", new=["hyperion", "--mode", "udc"]) +@patch("mx_bluesky.hyperion.__main__.do_default_logging_setup", MagicMock()) +@patch("mx_bluesky.hyperion.__main__.create_server_for_udc", MagicMock()) +@patch("mx_bluesky.hyperion.__main__.run_forever", MagicMock()) +@patch("mx_bluesky.hyperion.__main__.enable_debugging") +def test_hyperion_does_not_enable_debugging_if_not_specified( + mock_enable_debugging: MagicMock, + mock_setup_context: MagicMock, +): + main() + + mock_enable_debugging.assert_not_called() + + @patch("sys.argv", new=["hyperion", "--mode", "udc"]) @patch("mx_bluesky.hyperion.__main__.do_default_logging_setup", MagicMock()) @patch("mx_bluesky.hyperion.__main__.run_forever", MagicMock()) From 66c946276fcc5a5185e6328238137ddf116fd679 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Thu, 28 May 2026 17:20:06 +0100 Subject: [PATCH 02/11] Add debugging option --- helm/hyperion/templates/deployment.yaml | 6 +++ helm/hyperion/values.yaml | 4 ++ .../callbacks/__main__.py | 6 +++ src/mx_bluesky/hyperion/parameters/cli.py | 6 +-- .../callbacks/test_external_callbacks.py | 53 +++++++++++++++++-- 5 files changed, 69 insertions(+), 6 deletions(-) diff --git a/helm/hyperion/templates/deployment.yaml b/helm/hyperion/templates/deployment.yaml index b0ddf12391..60ef5def97 100644 --- a/helm/hyperion/templates/deployment.yaml +++ b/helm/hyperion/templates/deployment.yaml @@ -60,6 +60,9 @@ spec: hyperion --mode supervisor \ --client-config /etc/hyperion/client_config.yaml \ --supervisor-config /etc/hyperion/supervisor_config.yaml + {{- if .Values.supervisor.enableDebugging }} \ + --debug-port {{ .Values.supervisor.debugPort }} + {{- end }} {{- if .Values.application.dev }} \ --dev sleep 600 @@ -107,6 +110,9 @@ spec: hyperion-callbacks \ --watchdog-port {{ .Values.supervisor.containerPort }} \ --stomp-config /etc/hyperion/blueapi_callbacks.yml + {{- if .Values.callbacks.enableDebugging }} \ + --debug-port {{ .Values.callbacks.debugPort }} + {{- end }} {{- if .Values.application.dev }} \ --dev sleep 600 diff --git a/helm/hyperion/values.yaml b/helm/hyperion/values.yaml index 31d1aca02e..d4f6fedabf 100644 --- a/helm/hyperion/values.yaml +++ b/helm/hyperion/values.yaml @@ -10,9 +10,13 @@ supervisor: containerPort: 5006 runAsUser: 1000 runAsGroup: 1000 + enableDebugging: false + debugPort: 5050 callbacks: runAsUser: 1000 runAsGroup: 1000 + enableDebugging: false + debugPort: 5051 initContainer: dodalBranch: "main" scratch: diff --git a/src/mx_bluesky/hyperion/external_interaction/callbacks/__main__.py b/src/mx_bluesky/hyperion/external_interaction/callbacks/__main__.py index 729e1a95dc..a8297aee6c 100644 --- a/src/mx_bluesky/hyperion/external_interaction/callbacks/__main__.py +++ b/src/mx_bluesky/hyperion/external_interaction/callbacks/__main__.py @@ -42,6 +42,7 @@ SampleHandlingCallback, ) from mx_bluesky.common.parameters.constants import GridscanParamConstants +from mx_bluesky.common.utils.debug import enable_debugging from mx_bluesky.common.utils.log import ( ISPYB_ZOCALO_CALLBACK_LOGGER, NEXUS_LOGGER, @@ -176,6 +177,11 @@ class HyperionCallbackRunner: def __init__(self, callback_args: CallbackArgs) -> None: setup_logging(callback_args.dev_mode) log_info("Hyperion callback process started.") + if callback_args.debug_port: + enable_debugging( + callback_args.wait_for_debug_attach, callback_args.debug_port + ) + set_config_client(create_config_client()) set_alerting_service(LoggingAlertService(CONST.GRAYLOG_STREAM_ID)) diff --git a/src/mx_bluesky/hyperion/parameters/cli.py b/src/mx_bluesky/hyperion/parameters/cli.py index f5bcfcb6fb..04c7723ced 100644 --- a/src/mx_bluesky/hyperion/parameters/cli.py +++ b/src/mx_bluesky/hyperion/parameters/cli.py @@ -13,21 +13,21 @@ class HyperionMode(StrEnum): SUPERVISOR = "supervisor" -@dataclass +@dataclass(kw_only=True) class CommonArgs: dev_mode: bool = False debug_port: int | None = None wait_for_debug_attach: bool = False -@dataclass +@dataclass(kw_only=True) class HyperionArgs(CommonArgs): mode: HyperionMode = HyperionMode.UDC client_config: str | None = None supervisor_config: str | None = None -@dataclass +@dataclass(kw_only=True) class CallbackArgs(CommonArgs): watchdog_port: int = HyperionConstants.HYPERION_PORT stomp_config: Path | None = None diff --git a/tests/unit_tests/hyperion/external_interaction/callbacks/test_external_callbacks.py b/tests/unit_tests/hyperion/external_interaction/callbacks/test_external_callbacks.py index dda4dc84e5..6d804e44d6 100644 --- a/tests/unit_tests/hyperion/external_interaction/callbacks/test_external_callbacks.py +++ b/tests/unit_tests/hyperion/external_interaction/callbacks/test_external_callbacks.py @@ -17,6 +17,7 @@ from mx_bluesky.common.utils.log import ISPYB_ZOCALO_CALLBACK_LOGGER, NEXUS_LOGGER from mx_bluesky.hyperion.external_interaction.callbacks.__main__ import ( PING_TIMEOUT_S, + HyperionCallbackRunner, main, ping_watchdog_while_alive, run_watchdog, @@ -30,7 +31,9 @@ @patch("mx_bluesky.hyperion.external_interaction.callbacks.__main__.run_watchdog") @patch( "mx_bluesky.hyperion.external_interaction.callbacks.__main__.parse_callback_args", - return_value=CallbackArgs(True, HyperionConstants.SUPERVISOR_PORT), + return_value=CallbackArgs( + dev_mode=True, watchdog_port=HyperionConstants.SUPERVISOR_PORT + ), ) @patch("mx_bluesky.hyperion.external_interaction.callbacks.__main__.setup_callbacks") @patch("mx_bluesky.hyperion.external_interaction.callbacks.__main__.setup_logging") @@ -72,9 +75,51 @@ def test_main_function( assert isinstance(setup_alerting.mock_calls[0].args[0], LoggingAlertService) +@patch( + "mx_bluesky.hyperion.external_interaction.callbacks.__main__.setup_logging", + MagicMock(), +) +@patch( + "mx_bluesky.hyperion.external_interaction.callbacks.__main__.create_config_client", + MagicMock(), +) +@patch("mx_bluesky.hyperion.external_interaction.callbacks.__main__.enable_debugging") +@patch( + "mx_bluesky.hyperion.external_interaction.callbacks.__main__.RemoteDispatcherContextMgr", + MagicMock(), +) +def test_debugging_enabled(mock_enable_debugging: MagicMock): + HyperionCallbackRunner(CallbackArgs(dev_mode=True, debug_port=1234)) + mock_enable_debugging.assert_called_once_with(False, 1234) + + +@patch( + "mx_bluesky.hyperion.external_interaction.callbacks.__main__.setup_logging", + MagicMock(), +) +@patch( + "mx_bluesky.hyperion.external_interaction.callbacks.__main__.create_config_client", + MagicMock(), +) +@patch("mx_bluesky.hyperion.external_interaction.callbacks.__main__.enable_debugging") +@patch( + "mx_bluesky.hyperion.external_interaction.callbacks.__main__.RemoteDispatcherContextMgr", + MagicMock(), +) +def test_debugging_enabled_wait_for_attach(mock_enable_debugging: MagicMock): + HyperionCallbackRunner( + CallbackArgs(dev_mode=True, debug_port=1234, wait_for_debug_attach=True) + ) + mock_enable_debugging.assert_called_once_with(True, 1234) + + @patch( "mx_bluesky.hyperion.external_interaction.callbacks.__main__.parse_callback_args", - MagicMock(return_value=CallbackArgs(True, HyperionConstants.SUPERVISOR_PORT)), + MagicMock( + return_value=CallbackArgs( + dev_mode=True, watchdog_port=HyperionConstants.SUPERVISOR_PORT + ) + ), ) def test_no_config_server_url_raises_exception(): with pytest.raises(ValueError, match="CONFIG_SERVER_URL must be specified"): @@ -91,7 +136,9 @@ def test_setup_callbacks(): @pytest.mark.skip_log_setup @patch( "mx_bluesky.hyperion.external_interaction.callbacks.__main__.parse_callback_args", - return_value=CallbackArgs(True, HyperionConstants.SUPERVISOR_PORT), + return_value=CallbackArgs( + dev_mode=True, watchdog_port=HyperionConstants.SUPERVISOR_PORT + ), ) def test_setup_logging(parse_callback_cli_args): assert DODAL_LOGGER.parent != ISPYB_ZOCALO_CALLBACK_LOGGER From 0b1e272fbbcca23d4d1ceca92d2311b032ae8f42 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Thu, 28 May 2026 17:23:01 +0100 Subject: [PATCH 03/11] Update uv.lock --- uv.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uv.lock b/uv.lock index a54d860057..4946b04a8d 100644 --- a/uv.lock +++ b/uv.lock @@ -2056,6 +2056,7 @@ dependencies = [ { name = "cachetools", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "caproto", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "daq-config-server", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "debugpy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "deepdiff", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "dls-dodal", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "fastapi", extra = ["all"], marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, @@ -2120,6 +2121,7 @@ requires-dist = [ { name = "cachetools" }, { name = "caproto" }, { name = "daq-config-server", specifier = ">=1.3.0" }, + { name = "debugpy" }, { name = "deepdiff" }, { name = "dls-dodal", git = "https://github.com/DiamondLightSource/dodal.git?rev=main" }, { name = "fastapi", extras = ["all"] }, From 96536a8dcfe7b94ee5a3b1914dc5b493144f5f9d Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 29 May 2026 10:36:01 +0100 Subject: [PATCH 04/11] Initial attempt at containerised debugging --- .vscode/launch.json | 38 +++++++++++++++++ docs/developer/hyperion/how-to/debug-k8s.rst | 44 ++++++++++++++++++++ docs/developer/hyperion/index.rst | 2 +- helm/hyperion/templates/deployment.yaml | 9 ++++ helm/hyperion/values.yaml | 2 +- 5 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 docs/developer/hyperion/how-to/debug-k8s.rst diff --git a/.vscode/launch.json b/.vscode/launch.json index a08a5621d7..819e132990 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -19,6 +19,44 @@ "PYTEST_RAISE": "1", "PYTEST_ADDOPTS": "--no-cov" // Needed for debugger - see https://code.visualstudio.com/docs/python/testing#_pytest-configuration-settings }, + }, + { + "name": "Attach to K8s Hyperion Supervisor", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "127.0.0.1", + "port": 5050 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}/mx-bluesky/src", + "remoteRoot": "/scratch/mx-bluesky/src" + }, + { + "localRoot": "${workspaceFolder}/dodal/src", + "remoteRoot": "/scratch/dodal/src" + } + ] + }, + { + "name": "Attach to K8s Hyperion Callbacks", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "127.0.0.1", + "port": 5051 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}/mx-bluesky/src", + "remoteRoot": "/scratch/mx-bluesky/src" + }, + { + "localRoot": "${workspaceFolder}/dodal/src", + "remoteRoot": "/scratch/dodal/src" + } + ] } ] } diff --git a/docs/developer/hyperion/how-to/debug-k8s.rst b/docs/developer/hyperion/how-to/debug-k8s.rst new file mode 100644 index 0000000000..b8b94ea433 --- /dev/null +++ b/docs/developer/hyperion/how-to/debug-k8s.rst @@ -0,0 +1,44 @@ +Debugging Hyperion in Kubernetes +================================ + +Attaching to kubernetes scratch in the PVC +------------------------------------------ + +Loading the scratch folder in visual studio is similar to how it is done for ``blueapi``. + +You will need to have installed the Microsoft Kubernetes extension for VSCode. For details on how that is done, see `Debugging in the Cluster`_ + +Essentially the steps are: + +.. code-block:: bash + + $ module load + $ echo $KUBECONFIG + +In VSCode, run ``Kubernetes: Set Kubeconfig`` and copy the location of your kubeconfig printed from above. +Then from the Kubernetes plugin tray, expand Clusters -> -> Workloads -> Pods -> ``hyperion-scratch``. +Right click -> Attach Visual Studio Code to attach to the scratch pod. +In the Explorer of the new VSCode window that opens, click Open Folder and enter ``/scratch`` as the folder on the pod to open. +You can now edit files on the scratch persistent volume. + +.. _Debugging in the Cluster: https://diamondlightsource.github.io/python-copier-template/main/how-to/debug-in-cluster.html + +Attaching a debugger +-------------------- + +The ``hyperion`` and ``hyperion-callbacks`` commands accept ``--debug-port`` and ``--wait-for-attach`` options. + +``--debug-port`` enables remote debug using ``debugpy``. This is passed to ``hyperion`` and ``hyperion-callbacks`` if ``supervisor.enableDebugging`` is specified in ``values.yaml``, using the ``supervisor.debugPort`` value. Equivalent properties exist in ``values.yaml`` for the callbacks. +The default port numbers are 5050 and 5051. + +If the pod is deployed with this option it will cause the container to listen on the port for VSCode to attach the debugger. +There is also a ``--wait-for-attach`` option available if you need this. + +Having completed the steps above to attach to the PVC, when the pods are running in debug mode you can then run + +.. code-block:: bash + + $ kubectl port-forward pod/hyperion-supervisor 5050:5050 -n + +This will forward the port from the pod to your local machine. +There are launchers configured for the workspace in .vscode/launch.json, select either ``Attach to K8s Hyperion Supervisor`` or ``Attach to K8s Hyperion Callbacks`` to connect on 5050/5051 as appropriate, this will put you into debug mode. diff --git a/docs/developer/hyperion/index.rst b/docs/developer/hyperion/index.rst index a6f542affe..d71d48ace0 100644 --- a/docs/developer/hyperion/index.rst +++ b/docs/developer/hyperion/index.rst @@ -28,7 +28,7 @@ Documentation specific for the Hyperion module within MX-Bluesky .. toctree:: :caption: How-to Guides :maxdepth: 1 - + how-to/debug-k8s how-to/update-panda-ioc +++ diff --git a/helm/hyperion/templates/deployment.yaml b/helm/hyperion/templates/deployment.yaml index 60ef5def97..7d57ca6a18 100644 --- a/helm/hyperion/templates/deployment.yaml +++ b/helm/hyperion/templates/deployment.yaml @@ -75,6 +75,10 @@ spec: - name: rest-api containerPort: {{ .Values.supervisor.containerPort }} protocol: TCP + {{- if .Values.supervisor.enableDebugging }} + - name: debug + containerPort: {{ .Values.supervisor.debugPort }} + {{- end }} env: - name: LOG_DIR value: /var/log/bluesky @@ -125,6 +129,11 @@ spec: limits: cpu: "1" memory: "1Gi" + ports: + {{- if .Values.supervisor.enableDebugging }} + - name: debug + containerPort: {{ .Values.supervisor.debugPort }} + {{- end }} env: - name: LOG_DIR value: /var/log/bluesky diff --git a/helm/hyperion/values.yaml b/helm/hyperion/values.yaml index d4f6fedabf..fa7a2e63ee 100644 --- a/helm/hyperion/values.yaml +++ b/helm/hyperion/values.yaml @@ -10,7 +10,7 @@ supervisor: containerPort: 5006 runAsUser: 1000 runAsGroup: 1000 - enableDebugging: false + enableDebugging: true debugPort: 5050 callbacks: runAsUser: 1000 From 9fa0be291959d4b02111f204f036b6541000b684 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 29 May 2026 11:08:11 +0100 Subject: [PATCH 05/11] Slightly adjust strategy for vscode, add workspace --- .vscode/hyperion-k8s-scratch.code-workspace | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .vscode/hyperion-k8s-scratch.code-workspace diff --git a/.vscode/hyperion-k8s-scratch.code-workspace b/.vscode/hyperion-k8s-scratch.code-workspace new file mode 100644 index 0000000000..8a30283507 --- /dev/null +++ b/.vscode/hyperion-k8s-scratch.code-workspace @@ -0,0 +1,14 @@ +// To use this you should obviously have these all in the parent directory +{ + "folders": [ + { + "path": ".." + }, + { + "path": "../../dodal" + } + ], + "settings": { + "python.defaultInterpreterPath": "../../../app/mx-bluesky/.venv/bin/python", + } +} From 82ff50797cebd09ac42e6b4ee4ddc3f8ebe99780 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 29 May 2026 11:11:42 +0100 Subject: [PATCH 06/11] Tweak launch.json for changed workspace setup --- .vscode/launch.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 819e132990..424ca5a371 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -49,12 +49,12 @@ }, "pathMappings": [ { - "localRoot": "${workspaceFolder}/mx-bluesky/src", - "remoteRoot": "/scratch/mx-bluesky/src" + "localRoot": "${workspaceFolder}/src", + "remoteRoot": "/scratch/src" }, { - "localRoot": "${workspaceFolder}/dodal/src", - "remoteRoot": "/scratch/dodal/src" + "localRoot": "${workspaceFolder}/../dodal/src", + "remoteRoot": "/scratch/../dodal/src" } ] } From e9c11e84e8d6088b8b85224bf92ad0cbb27124d5 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 29 May 2026 12:27:29 +0100 Subject: [PATCH 07/11] Tweaks for vscode niceness --- .vscode/hyperion-k8s-scratch.code-workspace | 2 +- .vscode/launch.json | 8 ++++---- Dockerfile.hyperion | 3 ++- docs/developer/hyperion/how-to/debug-k8s.rst | 11 ++++++++--- helm/hyperion/templates/deployment.yaml | 4 ---- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.vscode/hyperion-k8s-scratch.code-workspace b/.vscode/hyperion-k8s-scratch.code-workspace index 8a30283507..8a87ea2883 100644 --- a/.vscode/hyperion-k8s-scratch.code-workspace +++ b/.vscode/hyperion-k8s-scratch.code-workspace @@ -9,6 +9,6 @@ } ], "settings": { - "python.defaultInterpreterPath": "../../../app/mx-bluesky/.venv/bin/python", + "python.defaultInterpreterPath": "/app/mx-bluesky/.venv/bin/python" } } diff --git a/.vscode/launch.json b/.vscode/launch.json index 424ca5a371..26037fbee1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -30,11 +30,11 @@ }, "pathMappings": [ { - "localRoot": "${workspaceFolder}/mx-bluesky/src", + "localRoot": "${workspaceFolder}/src", "remoteRoot": "/scratch/mx-bluesky/src" }, { - "localRoot": "${workspaceFolder}/dodal/src", + "localRoot": "${workspaceFolder}/../dodal/src", "remoteRoot": "/scratch/dodal/src" } ] @@ -50,11 +50,11 @@ "pathMappings": [ { "localRoot": "${workspaceFolder}/src", - "remoteRoot": "/scratch/src" + "remoteRoot": "/scratch//mx-bluesky/src" }, { "localRoot": "${workspaceFolder}/../dodal/src", - "remoteRoot": "/scratch/../dodal/src" + "remoteRoot": "/scratch/dodal/src" } ] } diff --git a/Dockerfile.hyperion b/Dockerfile.hyperion index 2b2c4f4129..1ece32c2a2 100644 --- a/Dockerfile.hyperion +++ b/Dockerfile.hyperion @@ -11,7 +11,6 @@ FROM python:3.12 AS base-image ARG DEBIAN_FRONTEND=noninteractive -ARG SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MX_BLUESKY # Need: # libgl, libglib2.0 for python cv2 dependency @@ -30,6 +29,8 @@ ADD --chown=hyperion:hyperion .git /app/mx-bluesky/.git RUN mkdir -p /app/scripts && \ chown hyperion:hyperion /app/scripts +ARG SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MX_BLUESKY + ############################################################################# FROM base-image as uv-staging diff --git a/docs/developer/hyperion/how-to/debug-k8s.rst b/docs/developer/hyperion/how-to/debug-k8s.rst index b8b94ea433..71654adc2d 100644 --- a/docs/developer/hyperion/how-to/debug-k8s.rst +++ b/docs/developer/hyperion/how-to/debug-k8s.rst @@ -18,8 +18,13 @@ Essentially the steps are: In VSCode, run ``Kubernetes: Set Kubeconfig`` and copy the location of your kubeconfig printed from above. Then from the Kubernetes plugin tray, expand Clusters -> -> Workloads -> Pods -> ``hyperion-scratch``. Right click -> Attach Visual Studio Code to attach to the scratch pod. -In the Explorer of the new VSCode window that opens, click Open Folder and enter ``/scratch`` as the folder on the pod to open. -You can now edit files on the scratch persistent volume. +In the Explorer of the new VSCode window that opens, click Open Folder and enter ``/`` as the folder on the pod to open. + +Select File->Open Workspace from File... and select ``/scratch/mx-bluesky/.vscode/hyperion-k8s-scratch.code-workspace`` +You can now edit files on the scratch persistent volume located in /scratch. + +When debugging inside the k8s pod, VSCode python extensions run remotely, so you will need to install them into the pod - open the extensions sidebar and install the various python extensions - these install a memory-hungry node server on the ``hyperion-scratch`` container. + .. _Debugging in the Cluster: https://diamondlightsource.github.io/python-copier-template/main/how-to/debug-in-cluster.html @@ -38,7 +43,7 @@ Having completed the steps above to attach to the PVC, when the pods are running .. code-block:: bash - $ kubectl port-forward pod/hyperion-supervisor 5050:5050 -n + $ kubectl port-forward pod/ 5050:5050 -n This will forward the port from the pod to your local machine. There are launchers configured for the workspace in .vscode/launch.json, select either ``Attach to K8s Hyperion Supervisor`` or ``Attach to K8s Hyperion Callbacks`` to connect on 5050/5051 as appropriate, this will put you into debug mode. diff --git a/helm/hyperion/templates/deployment.yaml b/helm/hyperion/templates/deployment.yaml index 7d57ca6a18..9ce64f3f45 100644 --- a/helm/hyperion/templates/deployment.yaml +++ b/helm/hyperion/templates/deployment.yaml @@ -172,10 +172,6 @@ spec: volumeMounts: - mountPath: "/scratch" name: scratch - resources: - limits: - cpu: "1" - memory: "1Gi" command: ['/bin/bash', '-c'] args: - | From 80cb20ee81da6fdb065334e80e41ade801fc6c0d Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 29 May 2026 13:26:00 +0100 Subject: [PATCH 08/11] Ensure manual helm upgrade retains previous pvc --- helm/hyperion/templates/volumes.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/hyperion/templates/volumes.yaml b/helm/hyperion/templates/volumes.yaml index 5cd83135ef..4ab9e0e42d 100644 --- a/helm/hyperion/templates/volumes.yaml +++ b/helm/hyperion/templates/volumes.yaml @@ -3,6 +3,7 @@ kind: PersistentVolumeClaim metadata: name: hyperion-supervisor-scratch-{{ .Values.application.imageVersion }} annotations: + helm.sh/resource-policy: keep argocd.argoproj.io/sync-options: Prune=false,Delete=false argocd.argoproj.io/compare-options: IgnoreExtraneous spec: From 60b99d631fa4f33dd5702e509e2965b78ab49662 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 29 May 2026 13:54:33 +0100 Subject: [PATCH 09/11] Disable debugging by default --- helm/hyperion/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/hyperion/values.yaml b/helm/hyperion/values.yaml index fa7a2e63ee..d4f6fedabf 100644 --- a/helm/hyperion/values.yaml +++ b/helm/hyperion/values.yaml @@ -10,7 +10,7 @@ supervisor: containerPort: 5006 runAsUser: 1000 runAsGroup: 1000 - enableDebugging: true + enableDebugging: false debugPort: 5050 callbacks: runAsUser: 1000 From e63b0a0dd57aa124fcdbc1ecd3301fca3531b0fb Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 29 May 2026 14:07:07 +0100 Subject: [PATCH 10/11] Fix doc index.rst --- docs/developer/hyperion/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/developer/hyperion/index.rst b/docs/developer/hyperion/index.rst index d71d48ace0..94859846e8 100644 --- a/docs/developer/hyperion/index.rst +++ b/docs/developer/hyperion/index.rst @@ -28,6 +28,7 @@ Documentation specific for the Hyperion module within MX-Bluesky .. toctree:: :caption: How-to Guides :maxdepth: 1 + how-to/debug-k8s how-to/update-panda-ioc From 1aafaf9ed5e58d77490ac28ec68dad6fabc99c0a Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 29 May 2026 15:18:24 +0100 Subject: [PATCH 11/11] Update uv.lock --- uv.lock | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/uv.lock b/uv.lock index 4946b04a8d..d7f5a4f84b 100644 --- a/uv.lock +++ b/uv.lock @@ -692,8 +692,8 @@ wheels = [ [[package]] name = "daq-config-server" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } +version = "1.3.3.dev9+gce91043bb" +source = { git = "https://github.com/DiamondLightSource/daq-config-server.git?rev=add_i15-1_crystal_lut#ce91043bbe1c7666527c0ff0592cda4977fbcc3e" } dependencies = [ { name = "cachetools", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "fastapi", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, @@ -707,10 +707,6 @@ dependencies = [ { name = "uvicorn", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "xmltodict", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c3/18/282dfb53acf238416b1c6031dc6530a87a3ef75ce298a6d21db7d661ad3b/daq_config_server-1.3.1.tar.gz", hash = "sha256:0fd447ad2902ee789c459bdcb39fa7902b5529dd2102ab48c29aaaa8ae84ae9d", size = 226549, upload-time = "2026-03-26T13:31:08.725Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/32/1f70fbc87f440b5cd9bdb61e9b7a713d5bd802275a66491a130e10814a79/daq_config_server-1.3.1-py3-none-any.whl", hash = "sha256:af01dc381492b0bffb42f71daf238252710347f4aba5105e0dba5945806b2320", size = 31579, upload-time = "2026-03-26T13:31:07.621Z" }, -] [[package]] name = "dask" @@ -810,8 +806,8 @@ wheels = [ [[package]] name = "dls-dodal" -version = "2.4.1.dev8+gbde3b691f" -source = { git = "https://github.com/DiamondLightSource/dodal.git?rev=main#bde3b691faff4afb4f3fd7540b6bd6fcbead2b45" } +version = "2.4.1.dev10+gf787f8379" +source = { git = "https://github.com/DiamondLightSource/dodal.git?rev=main#f787f83795b4c41605980c648bf88315fc3db1ad" } dependencies = [ { name = "aiofiles", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "aiohttp", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, @@ -2687,7 +2683,7 @@ wheels = [ [[package]] name = "ophyd-async" -version = "0.17a4" +version = "0.17a5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "bluesky", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, @@ -2701,9 +2697,9 @@ dependencies = [ { name = "stamina", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "velocity-profile", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/99/e1/2f0de5ded648fe9ca2e6f7415b9229ecb53226a56e775028410fddf0e7c7/ophyd_async-0.17a4.tar.gz", hash = "sha256:0ac0a890499700f8d6024a272ee616ddf278c42c2a28c3bf2d8ee80b5313a94d", size = 554974, upload-time = "2026-04-28T16:34:04.143Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/3c/127aeba71467a5dab69d21cbdcc7a7dfe294e826f6883af0b741ace01dbe/ophyd_async-0.17a5.tar.gz", hash = "sha256:f0eb30de946148479a1e33822e535542f47da0d9854da7d5ffa579990f650508", size = 557024, upload-time = "2026-05-20T08:21:08.682Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/71/59/a4ca5bd2ef8baa92733260634ba9fa6947fd88c6e7b909511964ac6406e1/ophyd_async-0.17a4-py3-none-any.whl", hash = "sha256:8dcaad6a851883c6906dda998c159a366b50b38310e8de87f385a5794c482792", size = 207790, upload-time = "2026-04-28T16:34:02.379Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a7/a87c4f373082674d2429af6912d2b0c113269063ee089eeb400effcc6bb6/ophyd_async-0.17a5-py3-none-any.whl", hash = "sha256:8175483b6ce5823cc3aff358afdd370bc08e1b2caf9f9e0cc4af9dc5b1802d8b", size = 208907, upload-time = "2026-05-20T08:21:06.899Z" }, ] [package.optional-dependencies]