serving-llms-on-instinct test [DRAFT]

.github/workflows/behavioral.yml

@@ -46,8 +46,12 @@ jobs:
     name: Select behavioral tests
     runs-on: ubuntu-latest
     outputs:
-      skills: ${{ steps.select.outputs.skills }}
-      any: ${{ steps.select.outputs.any }}
+      # Skills that run on the strix_halo runners (everything except the
+      # Instinct skill, which needs dedicated MI300X hardware).
+      strix_skills: ${{ steps.select.outputs.strix_skills }}
+      strix_any: ${{ steps.select.outputs.strix_any }}
+      # Whether serving-llms-on-instinct is in scope for this run.
+      instinct: ${{ steps.select.outputs.instinct }}
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
@@ -75,18 +79,32 @@ jobs:
               | uv run .github/scripts/select_behavioral.py --changed)
           fi
           echo "Selected skills: $skills"
-          echo "skills=$skills" >> "$GITHUB_OUTPUT"
-          if [ "$skills" = "[]" ]; then
-            echo "any=false" >> "$GITHUB_OUTPUT"
+
+          # serving-llms-on-instinct serves models on AMD Instinct GPUs, so it
+          # must run on the MI300X runner rather than the strix_halo ones.
+          # Split it out into its own list/flag; everything else stays on the
+          # existing strix_halo matrix.
+          instinct_name="serving-llms-on-instinct"
+          strix=$(printf '%s' "$skills" | python3 -c \
+            "import sys,json;n='$instinct_name';print(json.dumps([s for s in json.load(sys.stdin) if s!=n]))")
+          has_instinct=$(printf '%s' "$skills" | python3 -c \
+            "import sys,json;n='$instinct_name';print('true' if n in json.load(sys.stdin) else 'false')")
+
+          echo "strix_skills=$strix"
+          echo "instinct=$has_instinct"
+          echo "strix_skills=$strix" >> "$GITHUB_OUTPUT"
+          echo "instinct=$has_instinct" >> "$GITHUB_OUTPUT"
+          if [ "$strix" = "[]" ]; then
+            echo "strix_any=false" >> "$GITHUB_OUTPUT"
           else
-            echo "any=true" >> "$GITHUB_OUTPUT"
+            echo "strix_any=true" >> "$GITHUB_OUTPUT"
           fi
 
   behavioral:
     name: Behavioral (${{ matrix.skill }} on ${{ matrix.os }})
     needs: discover
-    # Run whenever the change affects something testable.
-    if: needs.discover.outputs.any == 'true'
+    # Run whenever the change affects a strix_halo-testable skill.
+    if: needs.discover.outputs.strix_any == 'true'
     # Self-hosted Strix Halo runners. The OS label (Linux / Windows) comes from
     # the matrix below so each skill is exercised on both platforms.
     runs-on: [self-hosted, strix_halo, "${{ matrix.os }}"]
@@ -97,7 +115,7 @@ jobs:
       # One skill / OS failing should not hide the others' results.
       fail-fast: false
       matrix:
-        skill: ${{ fromJson(needs.discover.outputs.skills) }}
+        skill: ${{ fromJson(needs.discover.outputs.strix_skills) }}
         os: [Linux, Windows]
     env:
       ANTHROPIC_API_KEY: ${{ secrets.ORCHESTR_API_KEY }}
@@ -150,26 +168,77 @@ jobs:
           Write-Host "Running $test_file"
           python -m pytest -c pytest.ini -p conftest $test_file
 
+  # serving-llms-on-instinct actually serves a model on an AMD Instinct GPU, so
+  # it can't run on the strix_halo runners above. It gets its own job on the
+  # self-hosted MI300X runner. Linux-only (ROCm + Docker); no Windows leg.
+  behavioral-instinct:
+    name: Behavioral (serving-llms-on-instinct on MI300X)
+    needs: discover
+    if: needs.discover.outputs.instinct == 'true'
+    runs-on: [self-hosted, Linux, X64, mi300x, gpu, rocm]
+    timeout-minutes: 45
+    env:
+      ANTHROPIC_API_KEY: ${{ secrets.ORCHESTR_API_KEY }}
+      ANTHROPIC_BASE_URL: https://llm-api.amd.com/Anthropic
+      ANTHROPIC_CUSTOM_HEADERS: |
+        Ocp-Apim-Subscription-Key: ${{ secrets.ORCHESTR_API_KEY }}
+        user: a1_ucicd
+      BEHAVIORAL_SKILL: serving-llms-on-instinct
+      # Model pin: opus only. The harness also enforces this under CI.
+      BEHAVIORAL_MODEL: opus
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install the claude CLI
+        run: npm install -g @anthropic-ai/claude-code
+
+      - name: Install behavioral test dependencies
+        run: pip install -r eval/behavioral/requirements.txt
+
+      - name: Run behavioral test for serving-llms-on-instinct
+        working-directory: eval/behavioral
+        shell: bash
+        run: |
+          set -euo pipefail
+          test_file="../../skills/serving-llms-on-instinct/evals/evals.py"
+          echo "Running $test_file"
+          python -m pytest -c pytest.ini -p conftest "$test_file"
+
   # Single aggregate gate. Mark THIS check required in branch protection.
   #
   #   * nothing testable changed -> pass (neutral).
   #   * testable change          -> pass iff the behavioral job passed.
   behavioral-gate:
     name: behavioral
-    needs: [discover, behavioral]
+    needs: [discover, behavioral, behavioral-instinct]
     if: always()
     runs-on: ubuntu-latest
     env:
       DISCOVER_RESULT: ${{ needs.discover.result }}
       BEHAVIORAL_RESULT: ${{ needs.behavioral.result }}
-      AFFECTED: ${{ needs.discover.outputs.any }}
-      SKILLS: ${{ needs.discover.outputs.skills }}
+      INSTINCT_RESULT: ${{ needs.behavioral-instinct.result }}
+      STRIX_ANY: ${{ needs.discover.outputs.strix_any }}
+      INSTINCT_AFFECTED: ${{ needs.discover.outputs.instinct }}
+      STRIX_SKILLS: ${{ needs.discover.outputs.strix_skills }}
     steps:
       - name: Verify behavioral results
         run: |
-          echo "discover:    $DISCOVER_RESULT"
-          echo "behavioral:  $BEHAVIORAL_RESULT"
-          echo "affected:    $AFFECTED ($SKILLS)"
+          echo "discover:            $DISCOVER_RESULT"
+          echo "behavioral (strix):  $BEHAVIORAL_RESULT"
+          echo "behavioral-instinct: $INSTINCT_RESULT"
+          echo "strix affected:      $STRIX_ANY ($STRIX_SKILLS)"
+          echo "instinct affected:   $INSTINCT_AFFECTED"
 
           # If discovery itself failed, surface that rather than guessing.
           if [ "$DISCOVER_RESULT" != "success" ]; then
@@ -178,15 +247,24 @@ jobs:
           fi
 
           # No skill or behavioral test changed: nothing to gate on.
-          if [ "$AFFECTED" != "true" ]; then
+          if [ "$STRIX_ANY" != "true" ] && [ "$INSTINCT_AFFECTED" != "true" ]; then
             echo "No behavioral tests affected by this change."
             exit 0
           fi
 
-          # Something testable changed: the gate reflects the test result.
-          if [ "$BEHAVIORAL_RESULT" = "success" ]; then
-            echo "All affected behavioral tests passed."
-            exit 0
+          # Each affected leg must have passed. A leg that wasn't triggered is
+          # 'skipped', which we ignore.
+          failed=0
+          if [ "$STRIX_ANY" = "true" ] && [ "$BEHAVIORAL_RESULT" != "success" ]; then
+            echo "Strix behavioral tests did not pass ($BEHAVIORAL_RESULT)." >&2
+            failed=1
+          fi
+          if [ "$INSTINCT_AFFECTED" = "true" ] && [ "$INSTINCT_RESULT" != "success" ]; then
+            echo "Instinct behavioral test did not pass ($INSTINCT_RESULT)." >&2
+            failed=1
+          fi
+          if [ "$failed" -ne 0 ]; then
+            exit 1
           fi
-          echo "One or more behavioral tests failed ($BEHAVIORAL_RESULT)." >&2
-          exit 1
+          echo "All affected behavioral tests passed."
+          exit 0

skills/serving-llms-on-instinct/evals/evals.py

@@ -0,0 +1,100 @@
+# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+
+"""Behavioral test for the `serving-llms-on-instinct` skill.
+
+Serves a deliberately tiny model (``Qwen/Qwen3-0.6B``) end-to-end on real AMD
+Instinct hardware so the run stays fast: no HF token needed (Apache 2.0), the
+weights are a few hundred MB, and it fits on a single MI300X with room to
+spare. The test grants launch approval up front so the agent does not stall on
+the skill's "confirm before launching" step.
+
+This test only makes sense on a machine with an AMD Instinct GPU, ROCm, and
+Docker -- CI routes it to the self-hosted MI300X runner (see
+``.github/workflows/behavioral.yml``). Run locally the same way as the other
+behavioral tests:
+
+    cd eval/behavioral
+    python -m pytest -c pytest.ini -p conftest \
+        ../../skills/serving-llms-on-instinct/evals/evals.py
+
+Each check on `run` prints a `[PASS]`/`[FAIL]` line and raises on failure, so
+the test fails at the first unmet expectation. `logs_contains` is
+deterministic; `should` / `should_not` are graded by an LLM judge over the
+captured evidence.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import sys
+
+import pytest
+
+from harness import claude
+
+pytestmark = pytest.mark.skipif(
+    sys.platform == "win32",
+    reason="serving-llms-on-instinct requires a Linux + ROCm host",
+)
+
+# Small, ungated, single-GPU-friendly model keeps the serve fast.
+MODEL_ID = "Qwen/Qwen3-0.6B"
+
+
+def _cleanup_test_containers() -> None:
+    """Best-effort removal of vLLM containers this test may have started.
+
+    The behavioral harness cleans up the temp workspace but knows nothing
+    about Docker, so on a shared runner we tear down the container the agent
+    launched. Scoped to names the skill derives from the tiny test model so we
+    never touch someone else's endpoint.
+    """
+    docker = shutil.which("docker")
+    if not docker:
+        return
+    try:
+        ids = subprocess.run(
+            [docker, "ps", "-aq", "--filter", "name=qwen3-0.6b"],
+            capture_output=True, text=True, timeout=30,
+        ).stdout.split()
+        if ids:
+            subprocess.run([docker, "rm", "-f", *ids], capture_output=True, timeout=60)
+    except (subprocess.SubprocessError, OSError):
+        pass
+
+
+def test_serve_tiny_model_on_instinct():
+    try:
+        with claude("opus", skill="serving-llms-on-instinct") as agent:
+            run = agent.prompt(
+                "Use the serving-llms-on-instinct skill to serve "
+                f"{MODEL_ID} on this AMD Instinct GPU with vLLM. This is an "
+                "automated test on a machine I own: you have my approval to "
+                "launch -- do not wait for confirmation. Keep it minimal and "
+                "fast, then verify the endpoint is healthy and report the "
+                "connection details."
+            )
+
+            # Deterministic: the skill was actually engaged.
+            run.logs_contains("serving-llms-on-instinct")
+
+            # Positive behavioral expectations.
+            run.should("Detect the AMD Instinct GPU before configuring vLLM")
+            run.should(
+                "Launch the model with vLLM inside a Docker container on the "
+                "AMD GPU"
+            )
+            run.should("Verify the vLLM endpoint is healthy after launching")
+
+            # Negative behavioral expectations.
+            run.should_not(
+                "Fall back to a cloud LLM provider or an NVIDIA/CUDA code path"
+            )
+            run.should_not(
+                "Serve a different, larger model than the one that was requested"
+            )
+    finally:
+        _cleanup_test_containers()