K8s debugging

.vscode/hyperion-k8s-scratch.code-workspace

@@ -0,0 +1,14 @@
+// To use this you should obviously have these all in the parent directory
+{
+  "folders": [
+    {
+      "path": ".."
+    },
+    {
+      "path": "../../dodal"
+    }
+  ],
+  "settings": {
+    "python.defaultInterpreterPath": "/app/mx-bluesky/.venv/bin/python"
+  }
+}

.vscode/launch.json

@@ -19,6 +19,44 @@
                 "PYTEST_RAISE": "1",
                 "PYTEST_ADDOPTS": "--no-cov" // Needed for debugger - see https://code.visualstudio.com/docs/python/testing#_pytest-configuration-settings
             },
+        },
+        {
+            "name": "Attach to K8s Hyperion Supervisor",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "127.0.0.1",
+                "port": 5050
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}/src",
+                    "remoteRoot": "/scratch/mx-bluesky/src"
+                },
+                {
+                    "localRoot": "${workspaceFolder}/../dodal/src",
+                    "remoteRoot": "/scratch/dodal/src"
+                }
+            ]
+        },
+        {
+            "name": "Attach to K8s Hyperion Callbacks",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "127.0.0.1",
+                "port": 5051
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}/src",
+                    "remoteRoot": "/scratch//mx-bluesky/src"
+                },
+                {
+                    "localRoot": "${workspaceFolder}/../dodal/src",
+                    "remoteRoot": "/scratch/dodal/src"
+                }
+            ]
         }
     ]
 }

Dockerfile.hyperion

@@ -11,7 +11,6 @@
 FROM python:3.12 AS base-image
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MX_BLUESKY
 
 # Need:
 # libgl, libglib2.0 for python cv2 dependency
@@ -30,8 +29,10 @@
 RUN mkdir -p /app/scripts && \
     chown hyperion:hyperion /app/scripts
 
+ARG SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MX_BLUESKY
+
 #############################################################################
 FROM base-image as uv-staging
 
 # Copy the pyproject.toml and install dependencies for better caching when developing
 # & rerunning deployment scripts
@@ -43,7 +44,7 @@
     uv sync --no-cache --locked
 
 #############################################################################
 FROM base-image as build
 
 WORKDIR "/app/mx-bluesky"
 RUN mkdir /scratch && \

docs/developer/hyperion/how-to/debug-k8s.rst

@@ -0,0 +1,49 @@
+Debugging Hyperion in Kubernetes
+================================
+
+Attaching to kubernetes scratch in the PVC
+------------------------------------------
+
+Loading the scratch folder in visual studio is similar to how it is done for ``blueapi``.
+
+You will need to have installed the Microsoft Kubernetes extension for VSCode. For details on how that is done, see `Debugging in the Cluster`_
+
+Essentially the steps are:
+
+.. code-block:: bash
+
+    $ module load <cluster>
+    $ echo $KUBECONFIG
+
+In VSCode, run ``Kubernetes: Set Kubeconfig`` and copy the location of your kubeconfig printed from above.
+Then from the Kubernetes plugin tray, expand Clusters -> <cluster> -> Workloads -> Pods -> ``hyperion-scratch``.
+Right click -> Attach Visual Studio Code to attach to the scratch pod.
+In the Explorer of the new VSCode window that opens, click Open Folder and enter ``/`` as the folder on the pod to open.
+
+Select File->Open Workspace from File... and select ``/scratch/mx-bluesky/.vscode/hyperion-k8s-scratch.code-workspace``
+You can now edit files on the scratch persistent volume located in /scratch.
+
+When debugging inside the k8s pod, VSCode python extensions run remotely, so you will need to install them into the pod - open the extensions sidebar and install the various python extensions - these install a memory-hungry node server on the ``hyperion-scratch`` container.
+
+
+.. _Debugging in the Cluster: https://diamondlightsource.github.io/python-copier-template/main/how-to/debug-in-cluster.html
+
+Attaching a debugger
+--------------------
+
+The ``hyperion`` and ``hyperion-callbacks`` commands accept ``--debug-port`` and ``--wait-for-attach`` options.
+
+``--debug-port`` enables remote debug using ``debugpy``. This is passed to ``hyperion`` and ``hyperion-callbacks`` if ``supervisor.enableDebugging`` is specified in ``values.yaml``, using the ``supervisor.debugPort`` value. Equivalent properties exist in ``values.yaml`` for the callbacks.
+The default port numbers are 5050 and 5051.
+
+If the pod is deployed with this option it will cause the container to listen on the port for VSCode to attach the debugger.
+There is also a ``--wait-for-attach`` option available if you need this.
+
+Having completed the steps above to attach to the PVC, when the pods are running in debug mode you can then run
+
+.. code-block:: bash
+
+   $ kubectl port-forward pod/<pod name> 5050:5050 -n <k8s namespace>
+
+This will forward the port from the pod to your local machine.
+There are launchers configured for the workspace in .vscode/launch.json, select either ``Attach to K8s Hyperion Supervisor`` or ``Attach to K8s Hyperion Callbacks`` to connect on 5050/5051 as appropriate, this will put you into debug mode.

docs/developer/hyperion/index.rst

@@ -29,6 +29,7 @@ Documentation specific for the Hyperion module within MX-Bluesky
             :caption: How-to Guides
             :maxdepth: 1
 
+            how-to/debug-k8s
             how-to/update-panda-ioc
 
         +++

helm/hyperion/templates/deployment.yaml

@@ -60,6 +60,9 @@ spec:
               hyperion --mode supervisor \
                 --client-config /etc/hyperion/client_config.yaml \
                 --supervisor-config /etc/hyperion/supervisor_config.yaml
+              {{- if .Values.supervisor.enableDebugging }} \
+                --debug-port {{ .Values.supervisor.debugPort }}
+              {{- end }}
               {{- if .Values.application.dev }} \
                 --dev
               sleep 600
@@ -72,6 +75,10 @@ spec:
             - name: rest-api
               containerPort: {{ .Values.supervisor.containerPort }}
               protocol: TCP
+            {{- if .Values.supervisor.enableDebugging }}
+            - name: debug
+              containerPort: {{ .Values.supervisor.debugPort }}
+            {{- end }}
           env:
             - name: LOG_DIR
               value: /var/log/bluesky
@@ -107,6 +114,9 @@ spec:
               hyperion-callbacks \
                 --watchdog-port {{ .Values.supervisor.containerPort }} \
                 --stomp-config /etc/hyperion/blueapi_callbacks.yml
+              {{- if .Values.callbacks.enableDebugging }} \
+                --debug-port {{ .Values.callbacks.debugPort }}
+              {{- end }}
               {{- if .Values.application.dev }} \
                 --dev
               sleep 600
@@ -119,6 +129,11 @@ spec:
             limits:
               cpu: "1"
               memory: "1Gi"
+          ports:
+            {{- if .Values.supervisor.enableDebugging }}
+            - name: debug
+              containerPort: {{ .Values.supervisor.debugPort }}
+            {{- end }}
           env:
             - name: LOG_DIR
               value: /var/log/bluesky
@@ -157,10 +172,6 @@ spec:
           volumeMounts:
             - mountPath: "/scratch"
               name: scratch
-          resources:
-            limits:
-              cpu: "1"
-              memory: "1Gi"
           command: ['/bin/bash', '-c']
           args:
             - |

helm/hyperion/templates/volumes.yaml

@@ -3,6 +3,7 @@ kind: PersistentVolumeClaim
 metadata:
   name: hyperion-supervisor-scratch-{{ .Values.application.imageVersion }}
   annotations:
+    helm.sh/resource-policy: keep
     argocd.argoproj.io/sync-options: Prune=false,Delete=false
     argocd.argoproj.io/compare-options: IgnoreExtraneous
 spec:

helm/hyperion/values.yaml

@@ -10,9 +10,13 @@ supervisor:
   containerPort: 5006
   runAsUser: 1000
   runAsGroup: 1000
+  enableDebugging: false
+  debugPort: 5050
 callbacks:
   runAsUser: 1000
   runAsGroup: 1000
+  enableDebugging: false
+  debugPort: 5051
 initContainer:
   dodalBranch: "main"
 scratch:

pyproject.toml

@@ -46,6 +46,7 @@ dependencies = [
     "deepdiff",
     "matplotlib",
     "cachetools",
+    "debugpy",
     "bluesky-stomp >= 0.2.0",
     "mysql-connector-python == 9.5.0", # Can unpin once https://github.com/DiamondLightSource/ispyb-api/pull/244 is merged and released
 

src/mx_bluesky/common/utils/debug.py

@@ -0,0 +1,15 @@
+from debugpy import listen, wait_for_client
+
+from mx_bluesky.common.utils.log import LOGGER
+
+
+def enable_debugging(wait_for_attach: bool, port: int):
+    """Enable debugging via debugpy
+    Args:
+        wait_for_attach (bool): True to wait for the debugger to attach
+        port (int): Port on which to listen."""
+    listen(("0.0.0.0", port))
+    LOGGER.info(f"Listening for debugger connections on {port}")
+    if wait_for_attach:
+        LOGGER.info("Waiting for debugger to attach...")
+        wait_for_client()

src/mx_bluesky/hyperion/__main__.py

@@ -9,6 +9,7 @@
 from mx_bluesky.common.external_interaction.alerting.log_based_service import (
     LoggingAlertService,
 )
+from mx_bluesky.common.utils.debug import enable_debugging
 from mx_bluesky.common.utils.log import (
     LOGGER,
     do_default_logging_setup,
@@ -37,6 +38,8 @@ def initialise_globals(args: HyperionArgs):
         dev_mode=args.dev_mode,
     )
     LOGGER.info(f"Hyperion launched with args:{argv}")
+    if args.debug_port:
+        enable_debugging(args.wait_for_debug_attach, args.debug_port)
     alerting.set_alerting_service(LoggingAlertService(CONST.GRAYLOG_STREAM_ID))
 
 

src/mx_bluesky/hyperion/external_interaction/callbacks/__main__.py

@@ -42,6 +42,7 @@
     SampleHandlingCallback,
 )
 from mx_bluesky.common.parameters.constants import GridscanParamConstants
+from mx_bluesky.common.utils.debug import enable_debugging
 from mx_bluesky.common.utils.log import (
     ISPYB_ZOCALO_CALLBACK_LOGGER,
     NEXUS_LOGGER,
@@ -176,6 +177,11 @@ class HyperionCallbackRunner:
     def __init__(self, callback_args: CallbackArgs) -> None:
         setup_logging(callback_args.dev_mode)
         log_info("Hyperion callback process started.")
+        if callback_args.debug_port:
+            enable_debugging(
+                callback_args.wait_for_debug_attach, callback_args.debug_port
+            )
+
         set_config_client(create_config_client())
         set_alerting_service(LoggingAlertService(CONST.GRAYLOG_STREAM_ID))
 

src/mx_bluesky/hyperion/parameters/cli.py

@@ -13,34 +13,47 @@ class HyperionMode(StrEnum):
     SUPERVISOR = "supervisor"
 
 
-@dataclass
-class HyperionArgs:
-    mode: HyperionMode
+@dataclass(kw_only=True)
+class CommonArgs:
     dev_mode: bool = False
+    debug_port: int | None = None
+    wait_for_debug_attach: bool = False
+
+
+@dataclass(kw_only=True)
+class HyperionArgs(CommonArgs):
+    mode: HyperionMode = HyperionMode.UDC
     client_config: str | None = None
     supervisor_config: str | None = None
 
 
-@dataclass
-class CallbackArgs:
-    dev_mode: bool = False
+@dataclass(kw_only=True)
+class CallbackArgs(CommonArgs):
     watchdog_port: int = HyperionConstants.HYPERION_PORT
     stomp_config: Path | None = None
 
 
-def _add_callback_relevant_args(parser: argparse.ArgumentParser) -> None:
+def _add_common_args(parser: argparse.ArgumentParser) -> None:
     """adds arguments relevant to hyperion-callbacks."""
     parser.add_argument(
         "--dev",
         action="store_true",
         help="Use dev options, such as local graylog instances",
     )
+    parser.add_argument(
+        "--debug-port",
+        help="Enable remote debugging with the specified port",
+        type=int,
+    )
+    parser.add_argument(
+        "--wait-for-attach", action="store_true", help="Wait for the debugger to attach"
+    )
 
 
 def parse_callback_args() -> CallbackArgs:
     """Parse the CLI arguments for the watchdog port and dev mode into a CallbackArgs instance."""
     parser = argparse.ArgumentParser()
-    _add_callback_relevant_args(parser)
+    _add_common_args(parser)
     parser.add_argument(
         "--watchdog-port",
         type=int,
@@ -65,7 +78,7 @@ def parse_cli_args() -> HyperionArgs:
     Returns:
          an HyperionArgs dataclass with the fields: (dev_mode: bool)"""
     parser = argparse.ArgumentParser()
-    _add_callback_relevant_args(parser)
+    _add_common_args(parser)
     parser.add_argument(
         "--version",
         help="Print hyperion version string",
@@ -92,4 +105,6 @@ def parse_cli_args() -> HyperionArgs:
         mode=args.mode,
         supervisor_config=args.supervisor_config,
         client_config=args.client_config,
+        debug_port=args.debug_port,
+        wait_for_debug_attach=args.wait_for_attach,
     )

tests/unit_tests/common/utils/test_debug.py

@@ -0,0 +1,25 @@
+from unittest.mock import MagicMock, patch
+
+from mx_bluesky.common.utils.debug import enable_debugging
+
+
+@patch("mx_bluesky.common.utils.debug.listen")
+@patch("mx_bluesky.common.utils.debug.wait_for_client")
+def test_enable_debugging_no_wait(
+    mock_wait_for_client: MagicMock,
+    mock_listen: MagicMock,
+):
+    enable_debugging(False, 1234)
+    mock_listen.assert_called_once_with(("0.0.0.0", 1234))
+    mock_wait_for_client.assert_not_called()
+
+
+@patch("mx_bluesky.common.utils.debug.listen")
+@patch("mx_bluesky.common.utils.debug.wait_for_client")
+def test_enable_debugging_wait(
+    mock_wait_for_client: MagicMock,
+    mock_listen: MagicMock,
+):
+    enable_debugging(True, 1234)
+    mock_listen.assert_called_once_with(("0.0.0.0", 1234))
+    mock_wait_for_client.assert_called_once()