fix(nightly): fail on empty filters, correct the coverage claim, detect upstream drift

.github/workflows/fleet-status.yml

@@ -10,8 +10,17 @@ name: Fleet status
 # Combos are deliberately absent: a combo is stale iff one of its parents
 # is (the hashes are derived), so the ~150 leaf reads below cover the
 # ~5,500-combo fleet without per-combo registry reads.
+#
+# It also runs nightly for the half no PR can cover: the input hash sees the
+# repository, so an upstream base rebuild changes what our images contain while
+# every hash — and therefore every verdict above — stays put. The drift job
+# resolves each external base's digest and compares it to the previous run, so
+# "the world moved while the repo stood still" becomes a report line instead of
+# months of silence (mmmu's dataset 404'd exactly that way).
 
 on:
+  schedule:
+    - cron: '17 6 * * *' # nightly, 06:17 UTC (after the test nightlies)
   workflow_dispatch:
     inputs:
       tag:
@@ -35,7 +44,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Compare recorded input-hashes against the repo
         run: |
-          REPORT=$(containers/scripts/fleet-status.sh "${{ inputs.tag }}")
+          REPORT=$(containers/scripts/fleet-status.sh "${{ inputs.tag || 'latest' }}")
           echo "$REPORT"
           {
             echo "## Fleet freshness @ :${{ inputs.tag }} vs $(git rev-parse --short HEAD)"
@@ -54,4 +63,43 @@ jobs:
             fi
           } >> "$GITHUB_STEP_SUMMARY"
           stale=$(grep -cE '	stale	' <<< "$REPORT" || true)
-          [ "$stale" -eq 0 ] || echo "::warning::$stale image(s) stale at :${{ inputs.tag }} — inputs changed since they were built"
+          [ "$stale" -eq 0 ] || echo "::warning::$stale image(s) stale at :${{ inputs.tag || 'latest' }} — inputs changed since they were built"
+
+  # Upstream drift: the one staleness the hash cannot see (rule 11 defers
+  # external digests to release time). Report-only — a moved base is picked up
+  # by dispatching Release the fleet with force_rebuild.
+  drift:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v6.0.3
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # The previous run's digests, keyed so each run restores the last one and
+      # saves its own (a cache key cannot be overwritten).
+      - uses: actions/cache@v4
+        with:
+          path: external-digests.tsv
+          key: external-digests-${{ github.run_id }}
+          restore-keys: external-digests-
+      - name: Resolve external base digests and compare with the last run
+        run: |
+          set -eo pipefail
+          # Never fail the report on drift — record it, warn, and let a human
+          # decide whether to force a rebuild.
+          if [ -f external-digests.tsv ]; then
+            containers/scripts/external-drift.sh external-digests.tsv > current.tsv || true
+          else
+            containers/scripts/external-drift.sh > current.tsv || true
+          fi
+          mv current.tsv external-digests.tsv
+          {
+            echo "## External base digests @ $(git rev-parse --short HEAD)"
+            echo
+            echo '| base | digest |'
+            echo '|---|---|'
+            awk -F'\t' '{printf "| `%s` | `%s` |\n", $1, substr($2,1,26) "…"}' external-digests.tsv
+          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/nightly-agents-smoke.yml

@@ -93,7 +93,7 @@ jobs:
           TOTAL=$(printf '%s\n' "${ALL}" | grep -c . || true)
           if [ "${TOTAL}" -eq 0 ]; then
             if [ -z "${FILTER}" ]; then echo "::error::no agent tests listed (compile failure?)"; exit 1; fi
-            echo "filter '${FILTER}' matched no tests"; exit 0
+            echo "::error::filter '${FILTER}' matched no tests — a filter that selects nothing must fail, not pass silently"; exit 1
           fi
           mapfile -t CASES < <(printf '%s\n' "${ALL}" | awk -v S="${SHARDS}" -v SH="${SHARD}" 'NF { if (c++ % S == SH) print }')
           echo "shard ${SHARD}/${SHARDS}: ${#CASES[@]} of ${TOTAL} agents"

.github/workflows/nightly-oracle.yml

@@ -4,7 +4,10 @@ name: Nightly oracle gate
 # through the benchmark's REAL grader, and a no-op must score < 1.0 — proving no
 # always-pass / always-fail grader. No agent, no model (`eval-containers oracle`).
 # Each check `--local`-builds the benchmark image, so this is the daemon-backed
-# lane the per-PR test.yml deliberately can't run. The ~78 checks are split
+# lane the per-PR test.yml deliberately can't run. The checks (27 today:
+# every auto-covered exact-match benchmark plus the pinned SPECIAL list —
+# `ORACLE_LIST=1 cargo test --test oracle -- --ignored --nocapture` prints
+# them) are split
 # across a parallel shard matrix; each shard bakes the core bases once, then
 # `--local`-builds and grades its round-robin slice.
 #
@@ -47,7 +50,7 @@ jobs:
     strategy:
       fail-fast: false # one shard's failure must not cancel the others
       matrix:
-        # N-way shard of the ~78 oracle checks. ORACLE_SHARDS is derived from
+        # N-way shard of the oracle checks. ORACLE_SHARDS is derived from
         # strategy.job-total (= this list's length), so edit only this list.
         shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
     steps:

.github/workflows/nightly-replay.yml

@@ -116,7 +116,7 @@ jobs:
           TOTAL=$(printf '%s\n' "${ALL}" | grep -c . || true)
           if [ "${TOTAL}" -eq 0 ]; then
             if [ -z "${FILTER}" ]; then echo "::error::no replay tests listed (compile failure?)"; exit 1; fi
-            echo "filter '${FILTER}' matched no tests"; exit 0
+            echo "::error::filter '${FILTER}' matched no tests — a filter that selects nothing must fail, not pass silently"; exit 1
           fi
           mapfile -t CASES < <(printf '%s\n' "${ALL}" | awk -v S="${SHARDS}" -v SH="${SHARD}" 'NF { if (c++ % S == SH) print }')
           echo "shard ${SHARD}/${SHARDS}: ${#CASES[@]} of ${TOTAL} tests"

containers/scripts/external-drift.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# external-drift — resolve the digest of every external base the fleet builds
+# FROM, so upstream movement becomes visible.
+#
+# The build-input hash sees the repository only (delivery/RULES.md rule 11
+# defers external digests to release time), so an upstream rebuild — a new
+# `python:3.12-slim`, a moved `:latest` — changes what our images contain while
+# every hash stays put. Nothing in the repo changed, so no PR runs and no push
+# rebuilds: the drift is invisible until someone forces a rebuild. This is the
+# one class of staleness only a scheduled check can catch, and the answer to it
+# is a `force_rebuild` dispatch.
+#
+# Usage:
+#   external-drift.sh                 # ref<TAB>digest for every external base
+#   external-drift.sh <previous.tsv>  # same, plus ::warning per moved digest;
+#                                     # exit 1 if any moved
+# Env: REF (default HEAD) — which committed tree's FROMs to read.
+set -euo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"
+
+# Externals come from fleet-hash's own parse (column 5), so this cannot drift
+# from what the fleet actually builds FROM. Refs still carrying `${…}` are
+# per-build (per-task bases) and have no fixed digest to track.
+refs=$("$HERE/fleet-hash.sh" | cut -f5 | tr ',' '\n' \
+  | grep -vE '^-$|\$\{' | LC_ALL=C sort -u)
+[ -n "$refs" ] || { echo "external-drift: no external bases found" >&2; exit 2; }
+
+now=$(mktemp); trap 'rm -f "$now"' EXIT
+while read -r ref; do
+  [ -n "$ref" ] || continue
+  d=$(docker buildx imagetools inspect "$ref" --format '{{.Manifest.Digest}}' 2>/dev/null || echo "unresolved")
+  printf '%s\t%s\n' "$ref" "$d"
+done <<< "$refs" > "$now"
+cat "$now"
+
+[ $# -ge 1 ] && [ -s "${1:-}" ] || exit 0
+
+# Compare against the previous run: a changed digest means the image we build
+# FROM is not the image we built FROM last time.
+moved=0
+while IFS=$'\t' read -r ref digest; do
+  was=$(awk -F'\t' -v r="$ref" '$1==r{print $2}' "$1")
+  [ -n "$was" ] || continue                    # new ref, nothing to compare
+  [ "$digest" != "unresolved" ] || continue    # transient/unauthenticated read
+  if [ "$was" != "$digest" ]; then
+    echo "::warning::upstream base moved: $ref ${was:0:19}… -> ${digest:0:19}… (dispatch Release the fleet with force_rebuild to pick it up)"
+    moved=$((moved + 1))
+  fi
+done < "$now"
+[ "$moved" -eq 0 ] || { echo "::error::$moved external base(s) moved since the last check"; exit 1; }