diff --git a/.github/workflows/fleet-status.yml b/.github/workflows/fleet-status.yml new file mode 100644 index 00000000..a0fa20cd --- /dev/null +++ b/.github/workflows/fleet-status.yml @@ -0,0 +1,57 @@ +name: Fleet status + +# Freshness report (delivery/RULES.md rules 13–14): compares every fleet +# image's recorded eval.input-hash label at a tag against the repository's +# computed hash and reports fresh / stale / unlabeled / absent per image. +# Report-only — everything non-fresh is what the next release must rebuild +# or retag; nothing here gates. Replaces hand-inspecting GHCR (the v0.1.0 +# release shipped ~6,400 silently stale combos found only by hand — #233). +# +# Combos are deliberately absent: a combo is stale iff one of its parents +# is (the hashes are derived), so the ~150 leaf reads below cover the +# ~5,500-combo fleet without per-combo registry reads. + +on: + workflow_dispatch: + inputs: + tag: + description: "Tag to audit (e.g. v0.1.0); default = latest" + default: "latest" + +permissions: + contents: read + packages: read + +jobs: + status: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6.0.3 + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Compare recorded input-hashes against the repo + run: | + REPORT=$(containers/scripts/fleet-status.sh "${{ inputs.tag }}") + echo "$REPORT" + { + echo "## Fleet freshness @ :${{ inputs.tag }} vs $(git rev-parse --short HEAD)" + echo + echo "| verdict | images |" + echo "|---|---|" + cut -f2 <<< "$REPORT" | sort | uniq -c | awk '{printf "| %s | %s |\n", $2, $1}' + echo + if grep -qE ' (stale|unlabeled|absent) ' <<< "$REPORT"; then + echo "
non-fresh images (changed under rule 14)" + echo + grep -E ' (stale|unlabeled|absent) ' <<< "$REPORT" \ + | awk -F'\t' '{printf "- `%s` — %s\n", $1, $2}' + echo + echo "
" + fi + } >> "$GITHUB_STEP_SUMMARY" + stale=$(grep -cE ' stale ' <<< "$REPORT" || true) + [ "$stale" -eq 0 ] || echo "::warning::$stale image(s) stale at :${{ inputs.tag }} — inputs changed since they were built" diff --git a/containers/scripts/fleet-status.sh b/containers/scripts/fleet-status.sh new file mode 100644 index 00000000..9839d009 --- /dev/null +++ b/containers/scripts/fleet-status.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# fleet-status — compare every fleet image's recorded build-input hash against +# the repository's computed hash (delivery/RULES.md rules 13–14). +# +# For each static bake target: the registry ref is the graph's context column +# minus `containers/` (exact for every target, including dotted model dirs +# whose bake target names are lossy), the expected hash comes from fleet-hash, +# and the recorded hash is read from the image config at TAG via `imagetools +# inspect` — labels live in each arch image's config, never on the index, so +# the read resolves `{{json .Image}}` and selects a real platform. Verdicts: +# +# fresh recorded == computed +# stale recorded != computed → changed (rule 14) +# unlabeled image exists, no hash label → changed (rule 14, fail dirty) +# absent no image at TAG → changed (rule 14, fail dirty) +# +# Anything non-fresh MUST be rebuilt or retagged by the next release. +# +# Usage: fleet-status.sh [tag] (default: latest) +# Output (TSV): ref verdict computed-hash recorded-hash +# Env: REGISTRY (default ghcr.io/exgentic), REF (default HEAD), +# STATUS_JOBS (parallel inspects, default 8) +# Exit: 0 always when the sweep completes — freshness is a report, not a gate. +set -euo pipefail + +TAG="${1:-latest}" +REGISTRY="${REGISTRY:-ghcr.io/exgentic}" +HERE="$(cd "$(dirname "$0")" && pwd)" + +command -v jq >/dev/null || { echo "fleet-status: jq not found" >&2; exit 2; } + +# One fleet-hash run gives both the ref map (graph) and the expected hashes. +GRAPH=$("$HERE/fleet-hash.sh" graph) +ALL=$("$HERE/fleet-hash.sh") + +check_one() { + local ref=$1 want=$2 img got + if ! img=$(docker buildx imagetools inspect "$ref" --format '{{json .Image}}' 2>/dev/null); then + printf '%s\tabsent\t%s\t-\n' "$ref" "$want" + return + fi + # A manifest list yields a platform-keyed map (attestation entries live at + # unknown/unknown); a single-arch image yields the config object directly. + got=$(jq -r '(if has("linux/amd64") or has("linux/arm64") + then (.["linux/amd64"] // .["linux/arm64"]) else . end) + .config.Labels["eval.input-hash"] // ""' <<< "$img") + if [ -z "$got" ]; then printf '%s\tunlabeled\t%s\t-\n' "$ref" "$want" + elif [ "$got" = "$want" ]; then printf '%s\tfresh\t%s\t%s\n' "$ref" "$want" "$got" + else printf '%s\tstale\t%s\t%s\n' "$ref" "$want" "$got" + fi +} +export -f check_one + +# target|context|deps ⋈ targethash… → " " pairs, +# fanned out over STATUS_JOBS parallel inspects. +# shellcheck disable=SC2016 # $1/$2 belong to the xargs-spawned bash, not this shell +paste -d' ' \ + <(cut -d'|' -f2 <<< "$GRAPH" | sed "s|^containers/|${REGISTRY}/|;s|\$|:${TAG}|") \ + <(cut -f2 <<< "$ALL") \ + | xargs -P "${STATUS_JOBS:-8}" -n2 bash -c 'check_one "$1" "$2"' _ \ + | LC_ALL=C sort diff --git a/tests/static/Cargo.toml b/tests/static/Cargo.toml index b9ed1336..fa2a96f0 100644 --- a/tests/static/Cargo.toml +++ b/tests/static/Cargo.toml @@ -39,3 +39,7 @@ path = "grader.rs" [[test]] name = "input_hash" path = "input_hash.rs" + +[[test]] +name = "fleet_status" +path = "fleet_status.rs" diff --git a/tests/static/fleet_status.rs b/tests/static/fleet_status.rs new file mode 100644 index 00000000..23286eba --- /dev/null +++ b/tests/static/fleet_status.rs @@ -0,0 +1,138 @@ +//! Freshness comparison — the rule-14 judgment (delivery/RULES.md): recorded +//! build-input hash vs the repository's computed hash, absent/unreadable +//! failing dirty. +//! +//! `containers/scripts/fleet-status.sh` reads registry labels via `imagetools +//! inspect`, so the offline test (tests/static/RULES.md rule 1) stubs `docker` +//! on PATH with canned responses covering every read shape: a multi-arch +//! manifest list carrying attestation entries, a single-arch config object, a +//! labeled-but-hashless image, and an absent ref. The ref derivation (graph +//! context column, dot-safe for models like gpt-5.4) is asserted against the +//! real repo. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; +use test_support::repo_root; + +/// PATH-shimmed fake `docker`, removed on drop. +struct Stub(PathBuf); + +impl Drop for Stub { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn write_stub(script_body: &str) -> Stub { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!("fleet-status-{}-{nanos}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("docker"); + std::fs::write(&path, format!("#!/usr/bin/env bash\n{script_body}")).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + Stub(dir) +} + +/// ref -> (verdict, computed, recorded) +fn fleet_status(stub: &Stub) -> HashMap { + let root = repo_root(); + let path = format!( + "{}:{}", + stub.0.display(), + std::env::var("PATH").unwrap_or_default() + ); + let out = Command::new("bash") + .arg(root.join("containers/scripts/fleet-status.sh")) + .env("PATH", path) + .env("STATUS_JOBS", "8") + .output() + .expect("run fleet-status.sh"); + assert!( + out.status.success(), + "fleet-status failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout) + .expect("utf8") + .lines() + .map(|l| { + let f: Vec<&str> = l.split('\t').collect(); + assert_eq!(f.len(), 4, "malformed row: {l}"); + ( + f[0].to_string(), + (f[1].to_string(), f[2].to_string(), f[3].to_string()), + ) + }) + .collect() +} + +/// Every verdict class, every registry read shape, and the dot-safe ref map, +/// on the real repo with a stubbed registry. +#[test] +fn verdicts_cover_every_read_shape() { + let root = repo_root(); + let hashes = Command::new("bash") + .arg(root.join("containers/scripts/fleet-hash.sh")) + .output() + .expect("run fleet-hash.sh"); + assert!(hashes.status.success()); + let aime = String::from_utf8(hashes.stdout) + .unwrap() + .lines() + .find(|l| l.starts_with("benchmark-aime\t")) + .expect("aime row") + .split('\t') + .nth(1) + .unwrap() + .to_string(); + + // aime: fresh via a manifest list (attestation entry must be ignored); + // gsm8k: stale via a single-arch config object; arc: labels but no hash. + // Everything else: inspect fails => absent. + let stub = write_stub(&format!( + r#"ref="$4" +case "$ref" in + */benchmarks/aime:latest) + echo '{{"linux/amd64":{{"config":{{"Labels":{{"eval.input-hash":"{aime}"}}}}}},"unknown/unknown":{{"config":{{}}}}}}' ;; + */benchmarks/gsm8k:latest) + echo '{{"config":{{"Labels":{{"eval.input-hash":"deadbeef"}}}}}}' ;; + */benchmarks/arc:latest) + echo '{{"linux/amd64":{{"config":{{"Labels":{{"other":"x"}}}}}}}}' ;; + *) exit 1 ;; +esac +"# + )); + let rows = fleet_status(&stub); + assert_eq!(rows.len(), 153, "one row per static bake target"); + + let (v, want, got) = &rows["ghcr.io/exgentic/benchmarks/aime:latest"]; + assert_eq!((v.as_str(), got), ("fresh", want)); + assert_eq!(want, &aime); + assert_eq!(rows["ghcr.io/exgentic/benchmarks/gsm8k:latest"].0, "stale"); + assert_eq!( + rows["ghcr.io/exgentic/benchmarks/gsm8k:latest"].2, + "deadbeef" + ); + assert_eq!( + rows["ghcr.io/exgentic/benchmarks/arc:latest"].0, + "unlabeled" + ); + assert_eq!(rows["ghcr.io/exgentic/core/entrypoint:latest"].0, "absent"); + + // The ref map preserves dots that bake target names cannot carry. + assert!(rows.contains_key("ghcr.io/exgentic/models/gpt-5.4:latest")); + assert!(rows.contains_key("ghcr.io/exgentic/models/gpt-4.1-mini:latest")); + + // Rule 14: everything non-fresh is "changed"; exactly one image was fresh. + let fresh = rows.values().filter(|r| r.0 == "fresh").count(); + assert_eq!(fresh, 1); +}