test(ci): run the changed agents' smoke and the changed benchmarks' oracle

.github/workflows/test.yml

@@ -159,61 +159,34 @@ jobs:
   # zero matched tests is a FAILURE, and affected components with no recorded
   # fixture are named in a ::warning (a visible coverage gap, never a quiet
   # pass). PRs touching nothing under containers/ exit in ~30s.
-  affected-replay:
-    name: Replay affected (changed components)
+  affected:
+    name: Test affected (changed components)
     runs-on: ubuntu-latest
     if: github.event_name == 'pull_request'
-    timeout-minutes: 40
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v6.0.3
         with:
           fetch-depth: 0   # the base SHA must be resolvable for REF=<base> fleet-hash
 
-      - name: Select fixtures for affected components
+      - name: Select affected components
         id: sel
         env:
           BASE: ${{ github.event.pull_request.base.sha }}
-        run: |
-          set -eo pipefail
-          changed=$(comm -13 \
-            <(REF="$BASE" bash containers/scripts/fleet-hash.sh | LC_ALL=C sort) \
-            <(bash containers/scripts/fleet-hash.sh | LC_ALL=C sort) | cut -f1)
-          benches=$(sed -n 's/^benchmark-//p' <<< "$changed")
-          agents=$(sed -n 's/^agent-//p' <<< "$changed")
-          echo "affected: $(tr '\n' ' ' <<< "$changed")"
-          FIX=tests/run/replay/fixtures
-          picks=""; uncovered=""
-          pick() {  # $1 = fixture-name regex; no match = "" (|| true absorbs the
-            basename -a "$FIX"/*.traces.jsonl | grep -E "$1" | LC_ALL=C sort | head -1 || true
-          }         # grep miss, which pipefail would otherwise turn into an abort)
-          for b in $benches; do
-            f=$(pick "^${b}-[0-9]+-[a-z0-9-]+\.traces\.jsonl$")
-            if [ -n "$f" ]; then picks="$picks $f"; else uncovered="$uncovered benchmark/$b"; fi
-          done
-          for a in $agents; do
-            f=$(pick "^[a-z0-9-]+-[0-9]+-${a}\.traces\.jsonl$")
-            if [ -n "$f" ]; then picks="$picks $f"; else uncovered="$uncovered agent/$a"; fi
-          done
-          [ -z "$uncovered" ] || echo "::warning::affected components with no replay fixture (untested by this gate):$uncovered"
-          # Fixture name -> test ident; dedup; cap (a core change affects dozens
-          # of leaves — three fixtures exercise the cascade without an hour-long PR).
-          tests=$(tr ' ' '\n' <<< "$picks" | grep . | sed 's/\.traces\.jsonl$//' | tr '-' '_' | sed 's/^/replay_/' \
-            | LC_ALL=C sort -u | head -3 | tr '\n' ' ' || true)
-          echo "tests=${tests% }" >> "$GITHUB_OUTPUT"
-          echo "selected: ${tests:-<none>}"
+        run: tests/select-affected.sh
 
       - name: Install Rust toolchain
-        if: steps.sel.outputs.tests != ''
+        if: steps.sel.outputs.any == 'true'
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
 
       - name: Cache cargo
-        if: steps.sel.outputs.tests != ''
+        if: steps.sel.outputs.any == 'true'
         uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
         with:
           shared-key: replay
 
       - name: Log in to GHCR
-        if: steps.sel.outputs.tests != ''
+        if: steps.sel.outputs.any == 'true'
         uses: docker/login-action@v4
         with:
           registry: ghcr.io
@@ -235,6 +208,47 @@ jobs:
             echo "::error::machine-selected fixtures matched 0 tests: $TESTS"; exit 1
           fi
 
+      # The changed agents' own suite. AGENTS_SMOKE_BUILD makes it build from
+      # local source (the nightly path) — a PR's agent image does not exist in
+      # the registry yet, which is exactly what we need to test.
+      - name: Smoke the affected agents
+        if: ${{ !cancelled() && steps.sel.outputs.agents != '' }}
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          AGENTS_SMOKE_BUILD: "1"
+          IDS: ${{ steps.sel.outputs.agents }}
+        run: |
+          set -eo pipefail
+          # shellcheck disable=SC2086  # IDS is a space-separated ident list
+          cargo test -p eval-containers-tests-run --test agents \
+            -- --ignored --nocapture --test-threads=1 --exact $IDS \
+            | tee agents.log
+          if grep -qE "running 0 tests" agents.log; then
+            echo "::error::machine-selected agent smoke matched 0 tests: $IDS"; exit 1
+          fi
+
+      # The changed benchmarks' gold-solution check: the oracle must still score
+      # 1.0, so a broken grader or a moved dataset fails the PR that broke it.
+      # Per-task benchmarks are included — the suite covers each with one pinned
+      # task. Coverage comes from the harness (ORACLE_LIST), never re-derived
+      # here: a benchmark it deliberately cannot oracle (terminal-bench, whose
+      # upstream gold drifted) warns instead of failing the PR.
+      - name: Oracle the affected benchmarks
+        if: ${{ !cancelled() && steps.sel.outputs.oracle != '' }}
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          CAND: ${{ steps.sel.outputs.oracle }}
+        run: |
+          set -eo pipefail
+          # shellcheck disable=SC2086  # CAND is a space-separated name list
+          only=$(tests/select-affected.sh oracle-filter $CAND | tee /dev/stderr | tail -1)
+          [ -n "$only" ] || { echo "no covered benchmark to oracle"; exit 0; }
+          ORACLE_ONLY="$only" cargo test -p eval-containers-tests-run --test oracle \
+            -- --ignored --nocapture --test-threads=1 | tee oracle.log
+          if grep -qE "running 0 tests" oracle.log; then
+            echo "::error::oracle suite matched 0 tests for: $only"; exit 1
+          fi
+
   # Per-PR gate: cheapest full-stack fixture (bigcodebench + zerostack) on the real
   # stack (otelcol → bifrost → runner), so a gateway/otelcol regression fails the PR.
   replay-e2e:

tests/run/oracle/test.rs

@@ -84,8 +84,11 @@ async fn benchmarks_are_oracle_solvable() {
     // Collect every oracle check as (label, argv) up front so the heavy part —
     // each check `--local`-builds the benchmark image — can be fanned across a
     // matrix. ORACLE_SHARDS = matrix size, ORACLE_SHARD = this job's index; each
-    // job runs the round-robin slice `i % SHARDS == SHARD`. ORACLE_ONLY=<substr>
-    // narrows to matching labels (single-benchmark validation). All three unset
+    // job runs the round-robin slice `i % SHARDS == SHARD`. ORACLE_ONLY is a
+    // comma/space-separated list of substrings and narrows to labels matching
+    // ANY of them (the per-PR affected gate passes the changed benchmarks); a
+    // list that matches nothing is a hard error, never a green no-op — a
+    // machine-generated filter must not pass vacuously. All three unset
     // (the default) => SHARDS=1, SHARD=0, no filter => every check runs, so a
     // plain `cargo test --test oracle -- --ignored` is unchanged.
     let mut checks: Vec<(String, Vec<String>)> = Vec::new();
@@ -124,11 +127,27 @@ async fn benchmarks_are_oracle_solvable() {
         ));
     }
 
+    checks.sort();
+    // ORACLE_LIST=1 prints the coverage this suite has and returns — the
+    // per-PR gate asks the harness what it can oracle instead of
+    // re-deriving it (per-task benchmarks are covered by one pinned task
+    // each; a few, like terminal-bench, are deliberately uncovered).
+    if std::env::var_os("ORACLE_LIST").is_some() {
+        for (label, _) in &checks {
+            println!("oracle-label: {label}");
+        }
+        return;
+    }
+
     let only = std::env::var("ORACLE_ONLY").unwrap_or_default();
-    if !only.is_empty() {
-        checks.retain(|(label, _)| label.contains(&only));
+    let wanted: Vec<&str> = only.split([',', ' ']).filter(|s| !s.is_empty()).collect();
+    if !wanted.is_empty() {
+        checks.retain(|(label, _)| wanted.iter().any(|w| label.contains(w)));
+        assert!(
+            !checks.is_empty(),
+            "ORACLE_ONLY={only:?} matched no benchmark — refusing to pass vacuously"
+        );
     }
-    checks.sort(); // deterministic order so each shard's slice is stable
     let shard: usize = std::env::var("ORACLE_SHARD")
         .ok()
         .and_then(|s| s.parse().ok())

tests/select-affected.sh

@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+# tests/select-affected.sh — pick the tests a change actually needs.
+#
+# Affected components come from the build-input hashes: the targets whose hash
+# differs between BASE and HEAD (containers/scripts/fleet-hash.sh). That is a
+# pure git computation — no registry, no daemon — which is what lets the per-PR
+# gate use it (.agents/verification/RULES.md rule 1). Base-image cascades are
+# free: a core/ change moves every dependent leaf's bases component.
+#
+# Selection is machine-generated, so it never passes vacuously: a component
+# with no test of its own is named in a ::warning, and a cap that drops
+# candidates says what it dropped. The callers fail on "running 0 tests".
+#
+#   select-affected.sh [base-ref]        emit tests/agents/oracle/any (below)
+#   select-affected.sh oracle-filter …   keep only benchmarks the oracle covers
+#
+# Outputs (to $GITHUB_OUTPUT when set, else stdout):
+#   tests   replay test idents        agents  agent_smoke idents
+#   oracle  benchmark candidates      any     'true' if anything was selected
+# Env: BASE (base ref; default origin/main), REPLAY_CAP=3, SMOKE_CAP=2
+set -eo pipefail
+cd "$(dirname "$0")/.."
+
+FIX=tests/run/replay/fixtures
+# Warnings are collected, not printed inline: cap() runs inside command
+# substitution (a subshell), so a warning echoed there would be captured into
+# the value being computed, and one sent to stderr would not be rendered as a
+# workflow command. A file survives the subshell; the flush at the end puts
+# every warning on stdout, where GitHub parses it.
+WARNF=$(mktemp); trap 'rm -f "$WARNF"' EXIT
+warn() { echo "::warning::$1" >> "$WARNF"; }
+out() { if [ -n "${GITHUB_OUTPUT:-}" ]; then echo "$1=$2" >> "$GITHUB_OUTPUT"; else echo "$1=$2"; fi; }
+# Keep the first $1 entries of stdin, announcing what that drops — a cap that
+# hides candidates would read as "covered" when it is not.
+cap() {
+  local all n; all=$(cat); n=$(grep -c . <<< "$all" || true)
+  [ "$n" -le "$1" ] || warn "$2: $n affected, testing the first $1 ($(tr '\n' ' ' <<< "$all" | cut -c1-200))"
+  head -"$1" <<< "$all"
+}
+
+# ── oracle-filter: intersect candidates with the suite's declared coverage ──
+# Coverage is the harness's business (per-task benchmarks are covered by one
+# pinned task each; a few, like terminal-bench, are deliberately uncovered), so
+# ask it rather than re-deriving the rules here.
+if [ "${1:-}" = "oracle-filter" ]; then
+  shift
+  labels=$(ORACLE_LIST=1 cargo test -p eval-containers-tests-run --test oracle \
+    -- --ignored --nocapture 2>/dev/null | sed -n 's/^oracle-label: //p')
+  [ -n "$labels" ] || { echo "::error::ORACLE_LIST produced no labels"; exit 1; }
+  covered=""; uncovered=""
+  for b in "$@"; do
+    if grep -qE "^${b}( \(task |$)" <<< "$labels"; then covered="$covered$b"$'\n'
+    else uncovered="$uncovered $b"; fi
+  done
+  [ -z "$uncovered" ] || warn "no oracle check exists for:$uncovered (unverified by this gate)"
+  # Each check --local-builds a benchmark image (per-task ones pull a GB base).
+  filtered=$(grep . <<< "$covered" | cap "${ORACLE_CAP:-2}" oracle | tr '\n' ',' | sed 's/,$//')
+  cat "$WARNF"            # flush before the value, so stdout's last line is it
+  printf '%s\n' "$filtered"
+  exit 0
+fi
+
+# ── affected set ────────────────────────────────────────────────────────────
+BASE="${1:-${BASE:-origin/main}}"
+# Diff against the MERGE BASE, not the base branch's tip: a PR is responsible
+# for what it changed, not for what main changed after it branched (GitHub's
+# pull_request.base.sha is the tip, so comparing to it selects unrelated work).
+BASE=$(git merge-base "$BASE" HEAD 2>/dev/null || echo "$BASE")
+changed=$(comm -13 \
+  <(REF="$BASE" containers/scripts/fleet-hash.sh | LC_ALL=C sort) \
+  <(containers/scripts/fleet-hash.sh | LC_ALL=C sort) | cut -f1)
+benches=$(sed -n 's/^benchmark-//p' <<< "$changed")
+agents=$(sed -n 's/^agent-//p' <<< "$changed")
+echo "affected: $(tr '\n' ' ' <<< "$changed")"
+
+# ── replay: one recorded fixture per affected component ─────────────────────
+pick() { basename -a "$FIX"/*.traces.jsonl | grep -E "$1" | LC_ALL=C sort | head -1 || true; }
+picks=""; uncovered=""
+for b in $benches; do
+  f=$(pick "^${b}-[0-9]+-[a-z0-9-]+\.traces\.jsonl$")
+  if [ -n "$f" ]; then picks="$picks$f"$'\n'; else uncovered="$uncovered benchmark/$b"; fi
+done
+for a in $agents; do
+  f=$(pick "^[a-z0-9-]+-[0-9]+-${a}\.traces\.jsonl$")
+  if [ -n "$f" ]; then picks="$picks$f"$'\n'; else uncovered="$uncovered agent/$a"; fi
+done
+[ -z "$uncovered" ] || warn "affected components with no replay fixture (untested by this gate):$uncovered"
+tests=$(grep . <<< "$picks" | sed 's/\.traces\.jsonl$//' | tr '-' '_' | sed 's/^/replay_/' \
+  | LC_ALL=C sort -u | cap "${REPLAY_CAP:-3}" replay | tr '\n' ' ' || true)
+
+# ── agent smoke: the changed agents' own suite ──────────────────────────────
+# --exact idents downstream, so agent_claude_code never matches _rtk. bob has
+# no agent_smoke! by design (IBM-backend-tied; see tests/run/agents/test.rs).
+smoke=""
+for a in $agents; do
+  id="agent_$(tr '-' '_' <<< "$a")"
+  if grep -q "agent_smoke!($id," tests/run/agents/test.rs; then smoke="$smoke$id"$'\n'
+  else warn "agent $a has no agent_smoke! test (untested by this gate)"; fi
+done
+smoke=$(grep . <<< "$smoke" | LC_ALL=C sort -u | cap "${SMOKE_CAP:-2}" "agent smoke" | tr '\n' ' ' || true)
+
+# ── oracle: every changed benchmark is a candidate; oracle-filter decides ───
+orc=$(grep . <<< "$benches" | LC_ALL=C sort -u | tr '\n' ' ' || true)
+
+# Normalize before deciding: an empty list still leaves a separator space
+# behind (head emits a newline, tr turns it into ' '), which would make the
+# "anything to do?" flag true for a change that selected nothing.
+tests=$(tr -s ' ' <<< "$tests" | sed 's/^ *//; s/ *$//')
+smoke=$(tr -s ' ' <<< "$smoke" | sed 's/^ *//; s/ *$//')
+orc=$(tr -s ' ' <<< "$orc" | sed 's/^ *//; s/ *$//')
+
+cat "$WARNF"        # every warning, on stdout, outside any substitution
+out tests "$tests"
+out agents "$smoke"
+out oracle "$orc"
+out any "$([ -z "${tests}${smoke}${orc}" ] && echo false || echo true)"