From 342744d9e6d5ff7af90a58d393a0ec1f2eed53a0 Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Wed, 12 Aug 2026 10:23:53 +0300 Subject: [PATCH] fix(nightly): fail on empty filters, correct the coverage claim, detect upstream drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nightlies' job changed when PRs started testing what changed: they are no longer the safety net for changed code, they are the only thing that can catch what changes when the repo does NOT. Three fixes: 1. Silent green. nightly-replay and nightly-agents-smoke exited 0 when a filter matched no tests, so a typo'd or stale dispatch filter reported success having run nothing. Now a loud failure. 2. Stale claim. nightly-oracle advertised '~78 checks' in two places; the suite actually has 27 (11 auto-covered exact-match benchmarks + 16 pinned SPECIAL). Comment now states the real number and how to list it (ORACLE_LIST=1). 3. Upstream drift — the gap no PR can close. The input hash sees the repository, so an upstream rebuild (a new python:3.12-slim, a moved :latest) changes what our images contain while every hash and every freshness verdict stays put; mmmu's dataset 404'd for months exactly that way. containers/scripts/external-drift.sh resolves the digest of all 18 external bases (from fleet-hash's own externals column, so it cannot drift from what we build FROM) and warns per moved digest; the Fleet status workflow now also runs nightly, comparing against the previous run's digests via the Actions cache and publishing the table to the job summary. Report-only: the response to drift is a force_rebuild dispatch, which stays a human decision. Deliberately NOT done: filtering the nightlies by the affected set. On a quiet day that set is empty, so a filtered nightly would test nothing — exactly when drift detection matters most. Verified live: 18 bases resolved; seeding a previous-digest file with one altered digest produces the warning and exit 1, and an unchanged file exits 0. Signed-off-by: Elron Bandel --- .github/workflows/fleet-status.yml | 52 +++++++++++++++++++++- .github/workflows/nightly-agents-smoke.yml | 2 +- .github/workflows/nightly-oracle.yml | 7 ++- .github/workflows/nightly-replay.yml | 2 +- containers/scripts/external-drift.sh | 50 +++++++++++++++++++++ 5 files changed, 107 insertions(+), 6 deletions(-) create mode 100755 containers/scripts/external-drift.sh diff --git a/.github/workflows/fleet-status.yml b/.github/workflows/fleet-status.yml index a0fa20cd..004d3b2b 100644 --- a/.github/workflows/fleet-status.yml +++ b/.github/workflows/fleet-status.yml @@ -10,8 +10,17 @@ name: Fleet status # Combos are deliberately absent: a combo is stale iff one of its parents # is (the hashes are derived), so the ~150 leaf reads below cover the # ~5,500-combo fleet without per-combo registry reads. +# +# It also runs nightly for the half no PR can cover: the input hash sees the +# repository, so an upstream base rebuild changes what our images contain while +# every hash — and therefore every verdict above — stays put. The drift job +# resolves each external base's digest and compares it to the previous run, so +# "the world moved while the repo stood still" becomes a report line instead of +# months of silence (mmmu's dataset 404'd exactly that way). on: + schedule: + - cron: '17 6 * * *' # nightly, 06:17 UTC (after the test nightlies) workflow_dispatch: inputs: tag: @@ -35,7 +44,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Compare recorded input-hashes against the repo run: | - REPORT=$(containers/scripts/fleet-status.sh "${{ inputs.tag }}") + REPORT=$(containers/scripts/fleet-status.sh "${{ inputs.tag || 'latest' }}") echo "$REPORT" { echo "## Fleet freshness @ :${{ inputs.tag }} vs $(git rev-parse --short HEAD)" @@ -54,4 +63,43 @@ jobs: fi } >> "$GITHUB_STEP_SUMMARY" stale=$(grep -cE ' stale ' <<< "$REPORT" || true) - [ "$stale" -eq 0 ] || echo "::warning::$stale image(s) stale at :${{ inputs.tag }} — inputs changed since they were built" + [ "$stale" -eq 0 ] || echo "::warning::$stale image(s) stale at :${{ inputs.tag || 'latest' }} — inputs changed since they were built" + + # Upstream drift: the one staleness the hash cannot see (rule 11 defers + # external digests to release time). Report-only — a moved base is picked up + # by dispatching Release the fleet with force_rebuild. + drift: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6.0.3 + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # The previous run's digests, keyed so each run restores the last one and + # saves its own (a cache key cannot be overwritten). + - uses: actions/cache@v4 + with: + path: external-digests.tsv + key: external-digests-${{ github.run_id }} + restore-keys: external-digests- + - name: Resolve external base digests and compare with the last run + run: | + set -eo pipefail + # Never fail the report on drift — record it, warn, and let a human + # decide whether to force a rebuild. + if [ -f external-digests.tsv ]; then + containers/scripts/external-drift.sh external-digests.tsv > current.tsv || true + else + containers/scripts/external-drift.sh > current.tsv || true + fi + mv current.tsv external-digests.tsv + { + echo "## External base digests @ $(git rev-parse --short HEAD)" + echo + echo '| base | digest |' + echo '|---|---|' + awk -F'\t' '{printf "| `%s` | `%s` |\n", $1, substr($2,1,26) "…"}' external-digests.tsv + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/nightly-agents-smoke.yml b/.github/workflows/nightly-agents-smoke.yml index 783b7b57..2386bfb5 100644 --- a/.github/workflows/nightly-agents-smoke.yml +++ b/.github/workflows/nightly-agents-smoke.yml @@ -93,7 +93,7 @@ jobs: TOTAL=$(printf '%s\n' "${ALL}" | grep -c . || true) if [ "${TOTAL}" -eq 0 ]; then if [ -z "${FILTER}" ]; then echo "::error::no agent tests listed (compile failure?)"; exit 1; fi - echo "filter '${FILTER}' matched no tests"; exit 0 + echo "::error::filter '${FILTER}' matched no tests — a filter that selects nothing must fail, not pass silently"; exit 1 fi mapfile -t CASES < <(printf '%s\n' "${ALL}" | awk -v S="${SHARDS}" -v SH="${SHARD}" 'NF { if (c++ % S == SH) print }') echo "shard ${SHARD}/${SHARDS}: ${#CASES[@]} of ${TOTAL} agents" diff --git a/.github/workflows/nightly-oracle.yml b/.github/workflows/nightly-oracle.yml index c96e5e56..fdea21d3 100644 --- a/.github/workflows/nightly-oracle.yml +++ b/.github/workflows/nightly-oracle.yml @@ -4,7 +4,10 @@ name: Nightly oracle gate # through the benchmark's REAL grader, and a no-op must score < 1.0 — proving no # always-pass / always-fail grader. No agent, no model (`eval-containers oracle`). # Each check `--local`-builds the benchmark image, so this is the daemon-backed -# lane the per-PR test.yml deliberately can't run. The ~78 checks are split +# lane the per-PR test.yml deliberately can't run. The checks (27 today: +# every auto-covered exact-match benchmark plus the pinned SPECIAL list — +# `ORACLE_LIST=1 cargo test --test oracle -- --ignored --nocapture` prints +# them) are split # across a parallel shard matrix; each shard bakes the core bases once, then # `--local`-builds and grades its round-robin slice. # @@ -47,7 +50,7 @@ jobs: strategy: fail-fast: false # one shard's failure must not cancel the others matrix: - # N-way shard of the ~78 oracle checks. ORACLE_SHARDS is derived from + # N-way shard of the oracle checks. ORACLE_SHARDS is derived from # strategy.job-total (= this list's length), so edit only this list. shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] steps: diff --git a/.github/workflows/nightly-replay.yml b/.github/workflows/nightly-replay.yml index e885ab6c..84eead3f 100644 --- a/.github/workflows/nightly-replay.yml +++ b/.github/workflows/nightly-replay.yml @@ -116,7 +116,7 @@ jobs: TOTAL=$(printf '%s\n' "${ALL}" | grep -c . || true) if [ "${TOTAL}" -eq 0 ]; then if [ -z "${FILTER}" ]; then echo "::error::no replay tests listed (compile failure?)"; exit 1; fi - echo "filter '${FILTER}' matched no tests"; exit 0 + echo "::error::filter '${FILTER}' matched no tests — a filter that selects nothing must fail, not pass silently"; exit 1 fi mapfile -t CASES < <(printf '%s\n' "${ALL}" | awk -v S="${SHARDS}" -v SH="${SHARD}" 'NF { if (c++ % S == SH) print }') echo "shard ${SHARD}/${SHARDS}: ${#CASES[@]} of ${TOTAL} tests" diff --git a/containers/scripts/external-drift.sh b/containers/scripts/external-drift.sh new file mode 100755 index 00000000..df6bd915 --- /dev/null +++ b/containers/scripts/external-drift.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# external-drift — resolve the digest of every external base the fleet builds +# FROM, so upstream movement becomes visible. +# +# The build-input hash sees the repository only (delivery/RULES.md rule 11 +# defers external digests to release time), so an upstream rebuild — a new +# `python:3.12-slim`, a moved `:latest` — changes what our images contain while +# every hash stays put. Nothing in the repo changed, so no PR runs and no push +# rebuilds: the drift is invisible until someone forces a rebuild. This is the +# one class of staleness only a scheduled check can catch, and the answer to it +# is a `force_rebuild` dispatch. +# +# Usage: +# external-drift.sh # refdigest for every external base +# external-drift.sh # same, plus ::warning per moved digest; +# # exit 1 if any moved +# Env: REF (default HEAD) — which committed tree's FROMs to read. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" + +# Externals come from fleet-hash's own parse (column 5), so this cannot drift +# from what the fleet actually builds FROM. Refs still carrying `${…}` are +# per-build (per-task bases) and have no fixed digest to track. +refs=$("$HERE/fleet-hash.sh" | cut -f5 | tr ',' '\n' \ + | grep -vE '^-$|\$\{' | LC_ALL=C sort -u) +[ -n "$refs" ] || { echo "external-drift: no external bases found" >&2; exit 2; } + +now=$(mktemp); trap 'rm -f "$now"' EXIT +while read -r ref; do + [ -n "$ref" ] || continue + d=$(docker buildx imagetools inspect "$ref" --format '{{.Manifest.Digest}}' 2>/dev/null || echo "unresolved") + printf '%s\t%s\n' "$ref" "$d" +done <<< "$refs" > "$now" +cat "$now" + +[ $# -ge 1 ] && [ -s "${1:-}" ] || exit 0 + +# Compare against the previous run: a changed digest means the image we build +# FROM is not the image we built FROM last time. +moved=0 +while IFS=$'\t' read -r ref digest; do + was=$(awk -F'\t' -v r="$ref" '$1==r{print $2}' "$1") + [ -n "$was" ] || continue # new ref, nothing to compare + [ "$digest" != "unresolved" ] || continue # transient/unauthenticated read + if [ "$was" != "$digest" ]; then + echo "::warning::upstream base moved: $ref ${was:0:19}… -> ${digest:0:19}… (dispatch Release the fleet with force_rebuild to pick it up)" + moved=$((moved + 1)) + fi +done < "$now" +[ "$moved" -eq 0 ] || { echo "::error::$moved external base(s) moved since the last check"; exit 1; }