Run behavioral tests on docker [draft]

.github/scripts/select_behavioral.py

@@ -48,6 +48,11 @@
     "eval/behavioral/conftest.py",
     "eval/behavioral/pytest.ini",
     "eval/behavioral/requirements.txt",
+    # The CI container definition (used on the self-hosted runners) is shared
+    # harness too: changing it can affect every skill's run, so re-run all.
+    "eval/behavioral/run_in_container.py",
+    "eval/behavioral/Dockerfile.linux",
+    "eval/behavioral/Dockerfile.windows",
     "eval/claude_eval.py",
     ".github/scripts/select_behavioral.py",
     ".github/workflows/behavioral.yml",

.github/workflows/behavioral.yml

@@ -89,6 +89,12 @@ jobs:
     if: needs.discover.outputs.any == 'true'
     # Self-hosted Strix Halo runners. The OS label (Linux / Windows) comes from
     # the matrix below so each skill is exercised on both platforms.
+    #
+    # Unlike a local run, CI runs the suite INSIDE a device-passthrough
+    # container (see eval/behavioral/run_in_container.py). The tested skill runs
+    # with --dangerously-skip-permissions and installs Lemonade / pulls models,
+    # so the container keeps it from modifying the runner host while still
+    # giving it the GPU/NPU it needs to actually run inference.
     runs-on: [self-hosted, strix_halo, "${{ matrix.os }}"]
     # Behavioral runs install local models and can take a while; cap it so a
     # hung agent or stalled model pull fails the job instead of burning minutes.
@@ -113,42 +119,29 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v4
 
+      # Host Python only needs to launch the container wrapper; Node, the claude
+      # CLI, and the pytest deps all live inside the image (see the Dockerfiles),
+      # so nothing the skill installs lands on the runner.
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: "3.12"
 
-      - name: Set up Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: "20"
-
-      - name: Install the claude CLI
-        run: npm install -g @anthropic-ai/claude-code
-
-      - name: Install behavioral test dependencies
-        run: pip install -r eval/behavioral/requirements.txt
-
-      - name: Run behavioral test for ${{ matrix.skill }} (Linux)
-        if: matrix.os == 'Linux'
-        working-directory: eval/behavioral
-        shell: bash
-        run: |
-          set -euo pipefail
-          test_file="../../skills/${{ matrix.skill }}/evals/evals.py"
-          echo "Running $test_file"
-          python -m pytest -c pytest.ini -p conftest "$test_file"
-
-      - name: Run behavioral test for ${{ matrix.skill }} (Windows)
-        if: matrix.os == 'Windows'
-        working-directory: eval/behavioral
-        shell: powershell
-        run: |
-          $ErrorActionPreference = "Stop"
-          $skill = "${{ matrix.skill }}"
-          $test_file = "../../skills/$skill/evals/evals.py"
-          Write-Host "Running $test_file"
-          python -m pytest -c pytest.ini -p conftest $test_file
+      # Make sure Docker is present on the runner, installing it if missing
+      # (Linux: official convenience script; Windows: best effort -- see the
+      # launcher). Kept as its own step so a provisioning failure is obvious.
+      - name: Ensure Docker is installed
+        run: python eval/behavioral/run_in_container.py --skill "${{ matrix.skill }}" --ensure-docker
+
+      # Build the image and confirm the GPU/NPU is actually visible inside the
+      # container before spending tokens. Fails loudly if passthrough is
+      # unavailable (notably the open question for Windows NPU) instead of
+      # silently producing no output.
+      - name: Verify accelerator is visible in the container
+        run: python eval/behavioral/run_in_container.py --skill "${{ matrix.skill }}" --preflight
+
+      - name: Run behavioral test for ${{ matrix.skill }} in a container
+        run: python eval/behavioral/run_in_container.py --skill "${{ matrix.skill }}"
 
   # Single aggregate gate. Mark THIS check required in branch protection.
   #

eval/behavioral/Dockerfile.linux

@@ -0,0 +1,51 @@
+# Behavioral-test runtime for the self-hosted Strix Halo *Linux* runners.
+#
+# The behavioral suite runs a real agent with --dangerously-skip-permissions,
+# so on CI we run it inside this container to keep a tested skill from touching
+# the runner host. The skill itself installs Lemonade Server and pulls models
+# *inside* this container, so the image only needs the AMD userspace runtime
+# that matches the passed-through devices (/dev/kfd, /dev/dri, /dev/accel/*)
+# plus Python / Node / the claude CLI. See eval/behavioral/run_in_container.py
+# for how it is built and run.
+#
+# Build context is the repo root (so requirements.txt can be copied for a
+# cache-friendly dependency layer).
+FROM rocm/dev-ubuntu-22.04:latest
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHONUNBUFFERED=1 \
+    NODE_MAJOR=20
+
+# Python 3.12 (via deadsnakes), git, curl, and Node 20 + the claude CLI.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        software-properties-common ca-certificates curl git gnupg \
+    && add-apt-repository -y ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        python3.12 python3.12-venv python3.12-dev \
+    && curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR}.x" | bash - \
+    && apt-get install -y --no-install-recommends nodejs \
+    && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 \
+    && npm install -g @anthropic-ai/claude-code \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Make python/python3 resolve to 3.12 for the harness subprocess calls.
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 \
+    && update-alternatives --install /usr/bin/python python /usr/bin/python3.12 1
+
+# Behavioral-test Python deps (pytest). Copied separately so this layer is
+# cached until requirements.txt actually changes.
+COPY eval/behavioral/requirements.txt /tmp/requirements.txt
+RUN python3.12 -m pip install --no-cache-dir -r /tmp/requirements.txt
+
+# Run as a non-root user that can reach the GPU/NPU render nodes. The launcher
+# additionally passes --group-add for the host's video/render GIDs, since those
+# numeric GIDs are host-specific and cannot be baked into the image.
+RUN groupadd -f -g 44 video \
+    && groupadd -f -g 109 render \
+    && useradd -m -s /bin/bash -G video,render behavioral
+USER behavioral
+
+WORKDIR /work/eval/behavioral

eval/behavioral/Dockerfile.windows

@@ -0,0 +1,35 @@
+# Behavioral-test runtime for the self-hosted Strix Halo *Windows* runners.
+#
+# Windows counterpart to Dockerfile.linux. It is run with process isolation
+# (see eval/behavioral/run_in_container.py) so the tested skill can reach the
+# host GPU/NPU while still being unable to modify the runner host outside the
+# container. NPU (XDNA) passthrough into Windows containers is not guaranteed;
+# the workflow runs a device preflight that fails loudly if it is unavailable.
+#
+# The base image tag MUST match the host Windows build for process isolation.
+# ltsc2022 is the default; override with --build-arg WINDOWS_BASE=... if the
+# runner is on a different build.
+ARG WINDOWS_BASE=mcr.microsoft.com/windows/servercore:ltsc2022
+FROM ${WINDOWS_BASE}
+
+SHELL ["powershell", "-NoLogo", "-NoProfile", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+
+ENV PYTHONUNBUFFERED=1
+
+# Python 3.12 (per-machine, added to PATH).
+RUN Invoke-WebRequest -Uri 'https://www.python.org/ftp/python/3.12.7/python-3.12.7-amd64.exe' -OutFile 'C:\\python.exe'; \
+    Start-Process -FilePath 'C:\\python.exe' -ArgumentList '/quiet','InstallAllUsers=1','PrependPath=1','Include_pip=1' -Wait; \
+    Remove-Item 'C:\\python.exe' -Force
+
+# Node 20 (msi, adds itself + the global npm prefix to PATH).
+RUN Invoke-WebRequest -Uri 'https://nodejs.org/dist/v20.18.0/node-v20.18.0-x64.msi' -OutFile 'C:\\node.msi'; \
+    Start-Process msiexec.exe -ArgumentList '/i','C:\\node.msi','/quiet','/norestart' -Wait; \
+    Remove-Item 'C:\\node.msi' -Force
+
+# claude CLI + behavioral-test Python deps. requirements.txt is copied first so
+# the pip layer is cached until it changes.
+RUN npm install -g @anthropic-ai/claude-code
+COPY eval/behavioral/requirements.txt C:/tmp/requirements.txt
+RUN python -m pip install --no-cache-dir -r C:/tmp/requirements.txt
+
+WORKDIR C:/work/eval/behavioral