diff --git a/.github/workflows/behavioral.yml b/.github/workflows/behavioral.yml index 12ff88b..80a8f0d 100644 --- a/.github/workflows/behavioral.yml +++ b/.github/workflows/behavioral.yml @@ -46,8 +46,12 @@ jobs: name: Select behavioral tests runs-on: ubuntu-latest outputs: - skills: ${{ steps.select.outputs.skills }} - any: ${{ steps.select.outputs.any }} + # Skills that run on the strix_halo runners (everything except the + # Instinct skill, which needs dedicated MI300X hardware). + strix_skills: ${{ steps.select.outputs.strix_skills }} + strix_any: ${{ steps.select.outputs.strix_any }} + # Whether serving-llms-on-instinct is in scope for this run. + instinct: ${{ steps.select.outputs.instinct }} steps: - name: Check out repository uses: actions/checkout@v4 @@ -75,18 +79,32 @@ jobs: | uv run .github/scripts/select_behavioral.py --changed) fi echo "Selected skills: $skills" - echo "skills=$skills" >> "$GITHUB_OUTPUT" - if [ "$skills" = "[]" ]; then - echo "any=false" >> "$GITHUB_OUTPUT" + + # serving-llms-on-instinct serves models on AMD Instinct GPUs, so it + # must run on the MI300X runner rather than the strix_halo ones. + # Split it out into its own list/flag; everything else stays on the + # existing strix_halo matrix. + instinct_name="serving-llms-on-instinct" + strix=$(printf '%s' "$skills" | python3 -c \ + "import sys,json;n='$instinct_name';print(json.dumps([s for s in json.load(sys.stdin) if s!=n]))") + has_instinct=$(printf '%s' "$skills" | python3 -c \ + "import sys,json;n='$instinct_name';print('true' if n in json.load(sys.stdin) else 'false')") + + echo "strix_skills=$strix" + echo "instinct=$has_instinct" + echo "strix_skills=$strix" >> "$GITHUB_OUTPUT" + echo "instinct=$has_instinct" >> "$GITHUB_OUTPUT" + if [ "$strix" = "[]" ]; then + echo "strix_any=false" >> "$GITHUB_OUTPUT" else - echo "any=true" >> "$GITHUB_OUTPUT" + echo "strix_any=true" >> "$GITHUB_OUTPUT" fi behavioral: name: Behavioral (${{ matrix.skill }} on ${{ matrix.os }}) needs: discover - # Run whenever the change affects something testable. - if: needs.discover.outputs.any == 'true' + # Run whenever the change affects a strix_halo-testable skill. + if: needs.discover.outputs.strix_any == 'true' # Self-hosted Strix Halo runners. The OS label (Linux / Windows) comes from # the matrix below so each skill is exercised on both platforms. runs-on: [self-hosted, strix_halo, "${{ matrix.os }}"] @@ -97,7 +115,7 @@ jobs: # One skill / OS failing should not hide the others' results. fail-fast: false matrix: - skill: ${{ fromJson(needs.discover.outputs.skills) }} + skill: ${{ fromJson(needs.discover.outputs.strix_skills) }} os: [Linux, Windows] env: ANTHROPIC_API_KEY: ${{ secrets.ORCHESTR_API_KEY }} @@ -150,26 +168,77 @@ jobs: Write-Host "Running $test_file" python -m pytest -c pytest.ini -p conftest $test_file + # serving-llms-on-instinct actually serves a model on an AMD Instinct GPU, so + # it can't run on the strix_halo runners above. It gets its own job on the + # self-hosted MI300X runner. Linux-only (ROCm + Docker); no Windows leg. + behavioral-instinct: + name: Behavioral (serving-llms-on-instinct on MI300X) + needs: discover + if: needs.discover.outputs.instinct == 'true' + runs-on: [self-hosted, Linux, X64, mi300x, gpu, rocm] + timeout-minutes: 45 + env: + ANTHROPIC_API_KEY: ${{ secrets.ORCHESTR_API_KEY }} + ANTHROPIC_BASE_URL: https://llm-api.amd.com/Anthropic + ANTHROPIC_CUSTOM_HEADERS: | + Ocp-Apim-Subscription-Key: ${{ secrets.ORCHESTR_API_KEY }} + user: a1_ucicd + BEHAVIORAL_SKILL: serving-llms-on-instinct + # Model pin: opus only. The harness also enforces this under CI. + BEHAVIORAL_MODEL: opus + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install the claude CLI + run: npm install -g @anthropic-ai/claude-code + + - name: Install behavioral test dependencies + run: pip install -r eval/behavioral/requirements.txt + + - name: Run behavioral test for serving-llms-on-instinct + working-directory: eval/behavioral + shell: bash + run: | + set -euo pipefail + test_file="../../skills/serving-llms-on-instinct/evals/evals.py" + echo "Running $test_file" + python -m pytest -c pytest.ini -p conftest "$test_file" + # Single aggregate gate. Mark THIS check required in branch protection. # # * nothing testable changed -> pass (neutral). # * testable change -> pass iff the behavioral job passed. behavioral-gate: name: behavioral - needs: [discover, behavioral] + needs: [discover, behavioral, behavioral-instinct] if: always() runs-on: ubuntu-latest env: DISCOVER_RESULT: ${{ needs.discover.result }} BEHAVIORAL_RESULT: ${{ needs.behavioral.result }} - AFFECTED: ${{ needs.discover.outputs.any }} - SKILLS: ${{ needs.discover.outputs.skills }} + INSTINCT_RESULT: ${{ needs.behavioral-instinct.result }} + STRIX_ANY: ${{ needs.discover.outputs.strix_any }} + INSTINCT_AFFECTED: ${{ needs.discover.outputs.instinct }} + STRIX_SKILLS: ${{ needs.discover.outputs.strix_skills }} steps: - name: Verify behavioral results run: | - echo "discover: $DISCOVER_RESULT" - echo "behavioral: $BEHAVIORAL_RESULT" - echo "affected: $AFFECTED ($SKILLS)" + echo "discover: $DISCOVER_RESULT" + echo "behavioral (strix): $BEHAVIORAL_RESULT" + echo "behavioral-instinct: $INSTINCT_RESULT" + echo "strix affected: $STRIX_ANY ($STRIX_SKILLS)" + echo "instinct affected: $INSTINCT_AFFECTED" # If discovery itself failed, surface that rather than guessing. if [ "$DISCOVER_RESULT" != "success" ]; then @@ -178,15 +247,24 @@ jobs: fi # No skill or behavioral test changed: nothing to gate on. - if [ "$AFFECTED" != "true" ]; then + if [ "$STRIX_ANY" != "true" ] && [ "$INSTINCT_AFFECTED" != "true" ]; then echo "No behavioral tests affected by this change." exit 0 fi - # Something testable changed: the gate reflects the test result. - if [ "$BEHAVIORAL_RESULT" = "success" ]; then - echo "All affected behavioral tests passed." - exit 0 + # Each affected leg must have passed. A leg that wasn't triggered is + # 'skipped', which we ignore. + failed=0 + if [ "$STRIX_ANY" = "true" ] && [ "$BEHAVIORAL_RESULT" != "success" ]; then + echo "Strix behavioral tests did not pass ($BEHAVIORAL_RESULT)." >&2 + failed=1 + fi + if [ "$INSTINCT_AFFECTED" = "true" ] && [ "$INSTINCT_RESULT" != "success" ]; then + echo "Instinct behavioral test did not pass ($INSTINCT_RESULT)." >&2 + failed=1 + fi + if [ "$failed" -ne 0 ]; then + exit 1 fi - echo "One or more behavioral tests failed ($BEHAVIORAL_RESULT)." >&2 - exit 1 + echo "All affected behavioral tests passed." + exit 0 diff --git a/skills/serving-llms-on-instinct/evals/evals.py b/skills/serving-llms-on-instinct/evals/evals.py new file mode 100644 index 0000000..028a254 --- /dev/null +++ b/skills/serving-llms-on-instinct/evals/evals.py @@ -0,0 +1,100 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# See LICENSE for license information. + +"""Behavioral test for the `serving-llms-on-instinct` skill. + +Serves a deliberately tiny model (``Qwen/Qwen3-0.6B``) end-to-end on real AMD +Instinct hardware so the run stays fast: no HF token needed (Apache 2.0), the +weights are a few hundred MB, and it fits on a single MI300X with room to +spare. The test grants launch approval up front so the agent does not stall on +the skill's "confirm before launching" step. + +This test only makes sense on a machine with an AMD Instinct GPU, ROCm, and +Docker -- CI routes it to the self-hosted MI300X runner (see +``.github/workflows/behavioral.yml``). Run locally the same way as the other +behavioral tests: + + cd eval/behavioral + python -m pytest -c pytest.ini -p conftest \ + ../../skills/serving-llms-on-instinct/evals/evals.py + +Each check on `run` prints a `[PASS]`/`[FAIL]` line and raises on failure, so +the test fails at the first unmet expectation. `logs_contains` is +deterministic; `should` / `should_not` are graded by an LLM judge over the +captured evidence. +""" + +from __future__ import annotations + +import shutil +import subprocess +import sys + +import pytest + +from harness import claude + +pytestmark = pytest.mark.skipif( + sys.platform == "win32", + reason="serving-llms-on-instinct requires a Linux + ROCm host", +) + +# Small, ungated, single-GPU-friendly model keeps the serve fast. +MODEL_ID = "Qwen/Qwen3-0.6B" + + +def _cleanup_test_containers() -> None: + """Best-effort removal of vLLM containers this test may have started. + + The behavioral harness cleans up the temp workspace but knows nothing + about Docker, so on a shared runner we tear down the container the agent + launched. Scoped to names the skill derives from the tiny test model so we + never touch someone else's endpoint. + """ + docker = shutil.which("docker") + if not docker: + return + try: + ids = subprocess.run( + [docker, "ps", "-aq", "--filter", "name=qwen3-0.6b"], + capture_output=True, text=True, timeout=30, + ).stdout.split() + if ids: + subprocess.run([docker, "rm", "-f", *ids], capture_output=True, timeout=60) + except (subprocess.SubprocessError, OSError): + pass + + +def test_serve_tiny_model_on_instinct(): + try: + with claude("opus", skill="serving-llms-on-instinct") as agent: + run = agent.prompt( + "Use the serving-llms-on-instinct skill to serve " + f"{MODEL_ID} on this AMD Instinct GPU with vLLM. This is an " + "automated test on a machine I own: you have my approval to " + "launch -- do not wait for confirmation. Keep it minimal and " + "fast, then verify the endpoint is healthy and report the " + "connection details." + ) + + # Deterministic: the skill was actually engaged. + run.logs_contains("serving-llms-on-instinct") + + # Positive behavioral expectations. + run.should("Detect the AMD Instinct GPU before configuring vLLM") + run.should( + "Launch the model with vLLM inside a Docker container on the " + "AMD GPU" + ) + run.should("Verify the vLLM endpoint is healthy after launching") + + # Negative behavioral expectations. + run.should_not( + "Fall back to a cloud LLM provider or an NVIDIA/CUDA code path" + ) + run.should_not( + "Serve a different, larger model than the one that was requested" + ) + finally: + _cleanup_test_containers()