diff --git a/.github/scripts/select_behavioral.py b/.github/scripts/select_behavioral.py index e9d965c..2356d62 100644 --- a/.github/scripts/select_behavioral.py +++ b/.github/scripts/select_behavioral.py @@ -48,6 +48,11 @@ "eval/behavioral/conftest.py", "eval/behavioral/pytest.ini", "eval/behavioral/requirements.txt", + # The CI container definition (used on the self-hosted runners) is shared + # harness too: changing it can affect every skill's run, so re-run all. + "eval/behavioral/run_in_container.py", + "eval/behavioral/Dockerfile.linux", + "eval/behavioral/Dockerfile.windows", "eval/claude_eval.py", ".github/scripts/select_behavioral.py", ".github/workflows/behavioral.yml", diff --git a/.github/workflows/behavioral.yml b/.github/workflows/behavioral.yml index 12ff88b..1f7bb63 100644 --- a/.github/workflows/behavioral.yml +++ b/.github/workflows/behavioral.yml @@ -89,6 +89,12 @@ jobs: if: needs.discover.outputs.any == 'true' # Self-hosted Strix Halo runners. The OS label (Linux / Windows) comes from # the matrix below so each skill is exercised on both platforms. + # + # Unlike a local run, CI runs the suite INSIDE a device-passthrough + # container (see eval/behavioral/run_in_container.py). The tested skill runs + # with --dangerously-skip-permissions and installs Lemonade / pulls models, + # so the container keeps it from modifying the runner host while still + # giving it the GPU/NPU it needs to actually run inference. runs-on: [self-hosted, strix_halo, "${{ matrix.os }}"] # Behavioral runs install local models and can take a while; cap it so a # hung agent or stalled model pull fails the job instead of burning minutes. @@ -113,42 +119,29 @@ jobs: - name: Check out repository uses: actions/checkout@v4 + # Host Python only needs to launch the container wrapper; Node, the claude + # CLI, and the pytest deps all live inside the image (see the Dockerfiles), + # so nothing the skill installs lands on the runner. - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Set up Node - uses: actions/setup-node@v4 - with: - node-version: "20" - - - name: Install the claude CLI - run: npm install -g @anthropic-ai/claude-code - - - name: Install behavioral test dependencies - run: pip install -r eval/behavioral/requirements.txt - - - name: Run behavioral test for ${{ matrix.skill }} (Linux) - if: matrix.os == 'Linux' - working-directory: eval/behavioral - shell: bash - run: | - set -euo pipefail - test_file="../../skills/${{ matrix.skill }}/evals/evals.py" - echo "Running $test_file" - python -m pytest -c pytest.ini -p conftest "$test_file" - - - name: Run behavioral test for ${{ matrix.skill }} (Windows) - if: matrix.os == 'Windows' - working-directory: eval/behavioral - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $skill = "${{ matrix.skill }}" - $test_file = "../../skills/$skill/evals/evals.py" - Write-Host "Running $test_file" - python -m pytest -c pytest.ini -p conftest $test_file + # Make sure Docker is present on the runner, installing it if missing + # (Linux: official convenience script; Windows: best effort -- see the + # launcher). Kept as its own step so a provisioning failure is obvious. + - name: Ensure Docker is installed + run: python eval/behavioral/run_in_container.py --skill "${{ matrix.skill }}" --ensure-docker + + # Build the image and confirm the GPU/NPU is actually visible inside the + # container before spending tokens. Fails loudly if passthrough is + # unavailable (notably the open question for Windows NPU) instead of + # silently producing no output. + - name: Verify accelerator is visible in the container + run: python eval/behavioral/run_in_container.py --skill "${{ matrix.skill }}" --preflight + + - name: Run behavioral test for ${{ matrix.skill }} in a container + run: python eval/behavioral/run_in_container.py --skill "${{ matrix.skill }}" # Single aggregate gate. Mark THIS check required in branch protection. # diff --git a/eval/behavioral/Dockerfile.linux b/eval/behavioral/Dockerfile.linux new file mode 100644 index 0000000..1c70a01 --- /dev/null +++ b/eval/behavioral/Dockerfile.linux @@ -0,0 +1,51 @@ +# Behavioral-test runtime for the self-hosted Strix Halo *Linux* runners. +# +# The behavioral suite runs a real agent with --dangerously-skip-permissions, +# so on CI we run it inside this container to keep a tested skill from touching +# the runner host. The skill itself installs Lemonade Server and pulls models +# *inside* this container, so the image only needs the AMD userspace runtime +# that matches the passed-through devices (/dev/kfd, /dev/dri, /dev/accel/*) +# plus Python / Node / the claude CLI. See eval/behavioral/run_in_container.py +# for how it is built and run. +# +# Build context is the repo root (so requirements.txt can be copied for a +# cache-friendly dependency layer). +FROM rocm/dev-ubuntu-22.04:latest + +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONUNBUFFERED=1 \ + NODE_MAJOR=20 + +# Python 3.12 (via deadsnakes), git, curl, and Node 20 + the claude CLI. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + software-properties-common ca-certificates curl git gnupg \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + python3.12 python3.12-venv python3.12-dev \ + && curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR}.x" | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 \ + && npm install -g @anthropic-ai/claude-code \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Make python/python3 resolve to 3.12 for the harness subprocess calls. +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.12 1 + +# Behavioral-test Python deps (pytest). Copied separately so this layer is +# cached until requirements.txt actually changes. +COPY eval/behavioral/requirements.txt /tmp/requirements.txt +RUN python3.12 -m pip install --no-cache-dir -r /tmp/requirements.txt + +# Run as a non-root user that can reach the GPU/NPU render nodes. The launcher +# additionally passes --group-add for the host's video/render GIDs, since those +# numeric GIDs are host-specific and cannot be baked into the image. +RUN groupadd -f -g 44 video \ + && groupadd -f -g 109 render \ + && useradd -m -s /bin/bash -G video,render behavioral +USER behavioral + +WORKDIR /work/eval/behavioral diff --git a/eval/behavioral/Dockerfile.windows b/eval/behavioral/Dockerfile.windows new file mode 100644 index 0000000..8a89d66 --- /dev/null +++ b/eval/behavioral/Dockerfile.windows @@ -0,0 +1,35 @@ +# Behavioral-test runtime for the self-hosted Strix Halo *Windows* runners. +# +# Windows counterpart to Dockerfile.linux. It is run with process isolation +# (see eval/behavioral/run_in_container.py) so the tested skill can reach the +# host GPU/NPU while still being unable to modify the runner host outside the +# container. NPU (XDNA) passthrough into Windows containers is not guaranteed; +# the workflow runs a device preflight that fails loudly if it is unavailable. +# +# The base image tag MUST match the host Windows build for process isolation. +# ltsc2022 is the default; override with --build-arg WINDOWS_BASE=... if the +# runner is on a different build. +ARG WINDOWS_BASE=mcr.microsoft.com/windows/servercore:ltsc2022 +FROM ${WINDOWS_BASE} + +SHELL ["powershell", "-NoLogo", "-NoProfile", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] + +ENV PYTHONUNBUFFERED=1 + +# Python 3.12 (per-machine, added to PATH). +RUN Invoke-WebRequest -Uri 'https://www.python.org/ftp/python/3.12.7/python-3.12.7-amd64.exe' -OutFile 'C:\\python.exe'; \ + Start-Process -FilePath 'C:\\python.exe' -ArgumentList '/quiet','InstallAllUsers=1','PrependPath=1','Include_pip=1' -Wait; \ + Remove-Item 'C:\\python.exe' -Force + +# Node 20 (msi, adds itself + the global npm prefix to PATH). +RUN Invoke-WebRequest -Uri 'https://nodejs.org/dist/v20.18.0/node-v20.18.0-x64.msi' -OutFile 'C:\\node.msi'; \ + Start-Process msiexec.exe -ArgumentList '/i','C:\\node.msi','/quiet','/norestart' -Wait; \ + Remove-Item 'C:\\node.msi' -Force + +# claude CLI + behavioral-test Python deps. requirements.txt is copied first so +# the pip layer is cached until it changes. +RUN npm install -g @anthropic-ai/claude-code +COPY eval/behavioral/requirements.txt C:/tmp/requirements.txt +RUN python -m pip install --no-cache-dir -r C:/tmp/requirements.txt + +WORKDIR C:/work/eval/behavioral diff --git a/eval/behavioral/run_in_container.py b/eval/behavioral/run_in_container.py new file mode 100644 index 0000000..b1c84d9 --- /dev/null +++ b/eval/behavioral/run_in_container.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# See LICENSE for license information. + +"""Run one skill's behavioral test inside a device-passthrough container. + +This is the CI-only isolation layer. Locally you still run the suite bare +(``python -m pytest -c pytest.ini ...``); nothing here touches the harness. +On the self-hosted Strix Halo runners the workflow calls this launcher so the +tested skill -- which runs with ``--dangerously-skip-permissions`` and installs +Lemonade / pulls models -- executes in a throwaway container instead of on the +runner host. + +Two entry points: + + python run_in_container.py --skill # build image + run the test + python run_in_container.py --skill --preflight # device check only + +The container gets the GPU/NPU passed through (so the skill can actually run +inference and produce out.png) but cannot modify the host outside the mounted +repo checkout. The host env that the model-pin depends on (CI / GITHUB_ACTIONS) +is forwarded explicitly, because ``docker run`` does not inherit host env. +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import platform +import shutil +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +BEHAVIORAL_DIR = Path(__file__).resolve().parent + +# Secrets / config the harness (and the claude CLI it spawns) needs. Passed by +# name so ``docker run`` reads the value from this process's environment -- +# this preserves multi-line values like ANTHROPIC_CUSTOM_HEADERS verbatim. +FORWARD_ENV = ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_CUSTOM_HEADERS", + "ANTHROPIC_MODEL", + "BEHAVIORAL_MODEL", + "BEHAVIORAL_EFFORT", + "CLAUDE_CODE_MAX_RETRIES", +) + + +def _linux_group_id(name: str) -> str | None: + """Return the host GID for ``name`` so the container user can reach the + GPU/NPU render nodes. Numeric GIDs are host-specific, hence resolved here + rather than baked into the image.""" + try: + import grp + + return str(grp.getgrnam(name).gr_gid) + except (KeyError, ImportError): + return None + + +def _linux_config() -> dict: + devices = ["/dev/kfd", "/dev/dri"] + # XDNA NPU node, only present when the amdxdna driver is loaded on the host. + if Path("/dev/accel/accel0").exists(): + devices.append("/dev/accel/accel0") + + device_args: list[str] = [] + for dev in devices: + if Path(dev).exists(): + device_args += ["--device", dev] + for group in ("video", "render"): + gid = _linux_group_id(group) + if gid is not None: + device_args += ["--group-add", gid] + + return { + "dockerfile": str(BEHAVIORAL_DIR / "Dockerfile.linux"), + "image": "behavioral-linux", + "mount": [f"{REPO_ROOT}:/work"], + "workdir": "/work/eval/behavioral", + "device_args": device_args, + # A cheap "can we see the accelerator?" probe used by --preflight. + "device_check": [ + "bash", + "-lc", + "ls -l /dev/kfd /dev/dri 2>/dev/null; " + "rocminfo 2>/dev/null | grep -m1 -i 'Marketing Name' || " + "{ echo 'no ROCm device visible in container' >&2; exit 1; }", + ], + } + + +def _windows_config() -> dict: + # DirectX GPU device class for Windows process-isolated containers. + gpu_class = "class/5B45201D-F2F2-4F3B-85BB-30FF1F953599" + return { + "dockerfile": str(BEHAVIORAL_DIR / "Dockerfile.windows"), + "image": "behavioral-windows", + "mount": [f"{REPO_ROOT}:C:\\work"], + "workdir": "C:/work/eval/behavioral", + # Process isolation is required for device passthrough on Windows. + "device_args": ["--isolation=process", "--device", gpu_class], + "device_check": [ + "powershell", + "-NoProfile", + "-Command", + "$g = Get-CimInstance Win32_VideoController; " + "if (-not $g) { Write-Error 'no GPU visible in container'; exit 1 }; " + "$g | Select-Object -First 1 -ExpandProperty Name", + ], + } + + +def _os_config(os_name: str) -> dict: + if os_name == "linux": + return _linux_config() + if os_name == "windows": + return _windows_config() + raise SystemExit(f"error: unsupported OS '{os_name}' (expected linux/windows)") + + +def _detect_os() -> str: + system = platform.system().lower() + if system.startswith("win"): + return "windows" + if system == "linux": + return "linux" + raise SystemExit(f"error: containerized behavioral runs are only wired for " + f"Linux/Windows runners, not '{system}'") + + +def _image_tag(cfg: dict) -> str: + """Content-hash the Dockerfile + requirements so cache hits are automatic + and a changed definition gets a fresh tag.""" + h = hashlib.sha256() + for path in (Path(cfg["dockerfile"]), BEHAVIORAL_DIR / "requirements.txt"): + h.update(path.read_bytes()) + return f"{cfg['image']}:{h.hexdigest()[:12]}" + + +def _run(cmd: list[str]) -> int: + printable = " ".join(cmd) + print(f"[run_in_container] $ {printable}", flush=True) + return subprocess.run(cmd).returncode + + +def _install_docker_linux() -> None: + """Install Docker Engine via the official convenience script.""" + sudo = ["sudo"] if hasattr(os, "geteuid") and os.geteuid() != 0 and shutil.which("sudo") else [] + dl = subprocess.run( + ["curl", "-fsSL", "https://get.docker.com", "-o", "/tmp/get-docker.sh"] + ) + if dl.returncode != 0: + raise SystemExit("error: failed to download the Docker install script (is curl available / network reachable?)") + rc = _run(sudo + ["sh", "/tmp/get-docker.sh"]) + if rc != 0: + raise SystemExit(f"error: Docker install script failed (exit {rc})") + # Best effort: make sure the daemon is running (systemd hosts). + if shutil.which("systemctl"): + _run(sudo + ["systemctl", "enable", "--now", "docker"]) + + +def _install_docker_windows() -> None: + """Best-effort Docker install for Windows self-hosted runners. + + Windows-container support generally needs the 'Containers' feature and a + reboot, which a single CI job cannot perform, so surface an actionable + error rather than hanging when that is the case. + """ + ps = ( + "$ErrorActionPreference='Stop';" + "Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force | Out-Null;" + "Install-Module -Name DockerMsftProvider -Repository PSGallery -Force;" + "Install-Package -Name docker -ProviderName DockerMsftProvider -Force;" + "Start-Service docker" + ) + rc = _run(["powershell", "-NoProfile", "-NoLogo", "-Command", ps]) + if rc != 0: + raise SystemExit( + "error: automatic Docker install on Windows failed. Windows-" + "container support usually needs the 'Containers' feature and a " + "reboot, which a CI job cannot perform. Install Docker (in " + "Windows-container mode) on this runner once, then retry." + ) + + +def _ensure_docker(os_name: str) -> list[str]: + """Make sure ``docker`` is usable, installing it if missing, and return the + command prefix to invoke it (``["sudo", "docker"]`` when the daemon socket + needs elevation on Linux).""" + if shutil.which("docker") is None: + print("[run_in_container] 'docker' not found on PATH; installing...", flush=True) + if os_name == "linux": + _install_docker_linux() + else: + _install_docker_windows() + if shutil.which("docker") is None: + raise SystemExit( + "error: 'docker' is still not available after attempting to " + "install it. Install Docker on this runner and retry." + ) + + if os_name != "linux": + return ["docker"] + + # On Linux the daemon socket is root-owned; if the current user can't reach + # it (e.g. freshly installed and not yet in the 'docker' group this + # session), fall back to sudo so we don't fail on a permissions error. + if subprocess.run(["docker", "info"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0: + return ["docker"] + if shutil.which("sudo") and subprocess.run( + ["sudo", "-n", "docker", "info"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ).returncode == 0: + return ["sudo", "docker"] + return ["docker"] + + +def _build(docker: list[str], cfg: dict, tag: str) -> int: + cmd = list(docker) + [ + "build", + "-f", cfg["dockerfile"], + "-t", tag, + str(REPO_ROOT), + ] + return _run(cmd) + + +def _docker_run_prefix(docker: list[str], cfg: dict, skill: str) -> list[str]: + cmd = list(docker) + [ + "run", "--rm", + "--security-opt", "no-new-privileges", + "--pids-limit", "1024", + "-w", cfg["workdir"], + ] + for mount in cfg["mount"]: + cmd += ["-v", mount] + cmd += cfg["device_args"] + + for name in FORWARD_ENV: + if os.environ.get(name) is not None: + cmd += ["-e", name] + # The tested skill selection + the CI markers the model-pin keys off of. + cmd += ["-e", f"BEHAVIORAL_SKILL={skill}"] + cmd += ["-e", "CI=true", "-e", "GITHUB_ACTIONS=true"] + return cmd + + +def _test_command(skill: str) -> list[str]: + test_file = f"../../skills/{skill}/evals/evals.py" + return ["python", "-m", "pytest", "-c", "pytest.ini", "-p", "conftest", test_file] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--skill", required=True, help="Skill name under skills/ to test.") + parser.add_argument( + "--os", + dest="os_name", + choices=["linux", "windows"], + default=_detect_os(), + help="Container platform (defaults to the host OS).", + ) + parser.add_argument( + "--ensure-docker", + action="store_true", + help="Only make sure Docker is installed/usable (installing it if " + "missing), then exit. Used as a visible CI setup step.", + ) + parser.add_argument( + "--preflight", + action="store_true", + help="Only build the image and verify the GPU/NPU is visible inside " + "the container; do not run the test.", + ) + parser.add_argument( + "--no-build", + action="store_true", + help="Reuse an existing image tag instead of building.", + ) + args = parser.parse_args(argv) + + docker = _ensure_docker(args.os_name) + if args.ensure_docker: + print(f"[run_in_container] docker ready: {' '.join(docker)}", flush=True) + return 0 + + cfg = _os_config(args.os_name) + tag = _image_tag(cfg) + + if not args.no_build: + rc = _build(docker, cfg, tag) + if rc != 0: + print(f"[run_in_container] image build failed (exit {rc})", file=sys.stderr) + return rc + + if args.preflight: + cmd = _docker_run_prefix(docker, cfg, args.skill) + [tag] + cfg["device_check"] + rc = _run(cmd) + if rc != 0: + print( + "[run_in_container] device preflight FAILED: the accelerator is " + "not visible inside the container. The skill cannot run local " + "inference here.", + file=sys.stderr, + ) + return rc + + cmd = _docker_run_prefix(docker, cfg, args.skill) + [tag] + _test_command(args.skill) + return _run(cmd) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/behavioral/test_evals.py b/eval/behavioral/test_evals.py new file mode 100644 index 0000000..4dada29 --- /dev/null +++ b/eval/behavioral/test_evals.py @@ -0,0 +1,118 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# See LICENSE for license information. + +"""Data-driven behavioral loader. + +Most skills describe their behavioral coverage declaratively in +``skills//evals/evals.json`` instead of hand-written pytest. This module +is the *only* code needed to run them: it discovers those datasets, stages the +skill workspace, runs the agent once per case, and applies the case's checks. + +Skill selection: + + * ``BEHAVIORAL_SKILL=`` (set per matrix job in CI) -> run only that + skill's ``evals.json``. + * unset (local full run) -> run every skill that has an ``evals.json``. + +Dataset shape (``evals.json``):: + + { + "skill": "local-ai-use", # optional, informational + "model": "opus", # optional default model for all cases + "effort": "high", # optional default effort for all cases + "cases": [ + { + "id": "generate_image_of_a_cat", # required, unique within the file + "prompt": "…", # required, the agent instruction + "model": "opus", # optional, overrides file default + "effort": "high", # optional, overrides file default + "workspace_files": { # optional seed files, path -> text + "main.py": "from openai import OpenAI\nclient = OpenAI()\n" + }, + "logs_contains": ["local-ai-use"], # deterministic substring checks + "workspace_contains": ["out.png"], # deterministic file-exists checks + "should": ["Install Lemonade…"], # LLM-judged positive expectations + "should_not": ["Use a cloud API"] # LLM-judged negative expectations + } + ] + } + +Each ``logs_contains`` / ``workspace_contains`` / ``should`` / ``should_not`` +value may be a single string or a list of strings. Anything a case needs beyond +this (custom setup, code-graded output) belongs in a hand-written ``evals.py`` +for that skill instead -- both are collected side by side. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from harness import DEFAULT_EFFORT, SKILLS_DIR, claude + +EVAL_JSON = Path("evals") / "evals.json" + + +def _target_skills() -> list[str]: + """Skills to load cases for: the pinned one in CI, else every JSON skill.""" + selected = os.environ.get("BEHAVIORAL_SKILL", "").strip() + if selected: + return [selected] + if not SKILLS_DIR.is_dir(): + return [] + return sorted( + p.name for p in SKILLS_DIR.iterdir() if (p / EVAL_JSON).is_file() + ) + + +def _as_list(value) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + return [value] + return list(value) + + +def _load_cases() -> list[tuple[str, dict]]: + """Flatten every target skill's ``evals.json`` into (skill, case) pairs.""" + pairs: list[tuple[str, dict]] = [] + for skill in _target_skills(): + dataset_path = SKILLS_DIR / skill / EVAL_JSON + if not dataset_path.is_file(): + continue + data = json.loads(dataset_path.read_text(encoding="utf-8")) + file_model = data.get("model", "opus") + file_effort = data.get("effort", DEFAULT_EFFORT) + for case in data.get("cases", []): + case.setdefault("_model", case.get("model", file_model)) + case.setdefault("_effort", case.get("effort", file_effort)) + pairs.append((skill, case)) + return pairs + + +_CASES = _load_cases() +_IDS = [f"{skill}::{case.get('id', 'case')}" for skill, case in _CASES] + + +@pytest.mark.parametrize(("skill", "case"), _CASES, ids=_IDS) +def test_behavioral(skill: str, case: dict) -> None: + with claude(case["_model"], skill=skill, effort=case["_effort"]) as agent: + for rel_path, content in (case.get("workspace_files") or {}).items(): + target = agent.workspace / rel_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + + run = agent.prompt(case["prompt"]) + + for text in _as_list(case.get("logs_contains")): + run.logs_contains(text) + for path in _as_list(case.get("workspace_contains")): + run.workspace_contains(path) + for statement in _as_list(case.get("should")): + run.should(statement) + for statement in _as_list(case.get("should_not")): + run.should_not(statement)