feat(scripts): fleet-status — freshness report over recorded input-hashes

.github/workflows/fleet-status.yml

@@ -0,0 +1,57 @@
+name: Fleet status
+
+# Freshness report (delivery/RULES.md rules 13–14): compares every fleet
+# image's recorded eval.input-hash label at a tag against the repository's
+# computed hash and reports fresh / stale / unlabeled / absent per image.
+# Report-only — everything non-fresh is what the next release must rebuild
+# or retag; nothing here gates. Replaces hand-inspecting GHCR (the v0.1.0
+# release shipped ~6,400 silently stale combos found only by hand — #233).
+#
+# Combos are deliberately absent: a combo is stale iff one of its parents
+# is (the hashes are derived), so the ~150 leaf reads below cover the
+# ~5,500-combo fleet without per-combo registry reads.
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Tag to audit (e.g. v0.1.0); default = latest"
+        default: "latest"
+
+permissions:
+  contents: read
+  packages: read
+
+jobs:
+  status:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v6.0.3
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Compare recorded input-hashes against the repo
+        run: |
+          REPORT=$(containers/scripts/fleet-status.sh "${{ inputs.tag }}")
+          echo "$REPORT"
+          {
+            echo "## Fleet freshness @ :${{ inputs.tag }} vs $(git rev-parse --short HEAD)"
+            echo
+            echo "| verdict | images |"
+            echo "|---|---|"
+            cut -f2 <<< "$REPORT" | sort | uniq -c | awk '{printf "| %s | %s |\n", $2, $1}'
+            echo
+            if grep -qE '	(stale|unlabeled|absent)	' <<< "$REPORT"; then
+              echo "<details><summary>non-fresh images (changed under rule 14)</summary>"
+              echo
+              grep -E '	(stale|unlabeled|absent)	' <<< "$REPORT" \
+                | awk -F'\t' '{printf "- `%s` — %s\n", $1, $2}'
+              echo
+              echo "</details>"
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+          stale=$(grep -cE '	stale	' <<< "$REPORT" || true)
+          [ "$stale" -eq 0 ] || echo "::warning::$stale image(s) stale at :${{ inputs.tag }} — inputs changed since they were built"

containers/scripts/fleet-status.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# fleet-status — compare every fleet image's recorded build-input hash against
+# the repository's computed hash (delivery/RULES.md rules 13–14).
+#
+# For each static bake target: the registry ref is the graph's context column
+# minus `containers/` (exact for every target, including dotted model dirs
+# whose bake target names are lossy), the expected hash comes from fleet-hash,
+# and the recorded hash is read from the image config at TAG via `imagetools
+# inspect` — labels live in each arch image's config, never on the index, so
+# the read resolves `{{json .Image}}` and selects a real platform. Verdicts:
+#
+#   fresh      recorded == computed
+#   stale      recorded != computed          → changed (rule 14)
+#   unlabeled  image exists, no hash label   → changed (rule 14, fail dirty)
+#   absent     no image at TAG               → changed (rule 14, fail dirty)
+#
+# Anything non-fresh MUST be rebuilt or retagged by the next release.
+#
+# Usage: fleet-status.sh [tag]     (default: latest)
+# Output (TSV): ref  verdict  computed-hash  recorded-hash
+# Env: REGISTRY (default ghcr.io/exgentic), REF (default HEAD),
+#      STATUS_JOBS (parallel inspects, default 8)
+# Exit: 0 always when the sweep completes — freshness is a report, not a gate.
+set -euo pipefail
+
+TAG="${1:-latest}"
+REGISTRY="${REGISTRY:-ghcr.io/exgentic}"
+HERE="$(cd "$(dirname "$0")" && pwd)"
+
+command -v jq >/dev/null || { echo "fleet-status: jq not found" >&2; exit 2; }
+
+# One fleet-hash run gives both the ref map (graph) and the expected hashes.
+GRAPH=$("$HERE/fleet-hash.sh" graph)
+ALL=$("$HERE/fleet-hash.sh")
+
+check_one() {
+  local ref=$1 want=$2 img got
+  if ! img=$(docker buildx imagetools inspect "$ref" --format '{{json .Image}}' 2>/dev/null); then
+    printf '%s\tabsent\t%s\t-\n' "$ref" "$want"
+    return
+  fi
+  # A manifest list yields a platform-keyed map (attestation entries live at
+  # unknown/unknown); a single-arch image yields the config object directly.
+  got=$(jq -r '(if has("linux/amd64") or has("linux/arm64")
+                then (.["linux/amd64"] // .["linux/arm64"]) else . end)
+               .config.Labels["eval.input-hash"] // ""' <<< "$img")
+  if [ -z "$got" ]; then printf '%s\tunlabeled\t%s\t-\n' "$ref" "$want"
+  elif [ "$got" = "$want" ]; then printf '%s\tfresh\t%s\t%s\n' "$ref" "$want" "$got"
+  else printf '%s\tstale\t%s\t%s\n' "$ref" "$want" "$got"
+  fi
+}
+export -f check_one
+
+# target|context|deps  ⋈  target<TAB>hash…  →  "<ref> <expected-hash>" pairs,
+# fanned out over STATUS_JOBS parallel inspects.
+# shellcheck disable=SC2016  # $1/$2 belong to the xargs-spawned bash, not this shell
+paste -d' ' \
+  <(cut -d'|' -f2 <<< "$GRAPH" | sed "s|^containers/|${REGISTRY}/|;s|\$|:${TAG}|") \
+  <(cut -f2 <<< "$ALL") \
+  | xargs -P "${STATUS_JOBS:-8}" -n2 bash -c 'check_one "$1" "$2"' _ \
+  | LC_ALL=C sort

tests/static/Cargo.toml

@@ -39,3 +39,7 @@ path = "grader.rs"
 [[test]]
 name = "input_hash"
 path = "input_hash.rs"
+
+[[test]]
+name = "fleet_status"
+path = "fleet_status.rs"

tests/static/fleet_status.rs

@@ -0,0 +1,138 @@
+//! Freshness comparison — the rule-14 judgment (delivery/RULES.md): recorded
+//! build-input hash vs the repository's computed hash, absent/unreadable
+//! failing dirty.
+//!
+//! `containers/scripts/fleet-status.sh` reads registry labels via `imagetools
+//! inspect`, so the offline test (tests/static/RULES.md rule 1) stubs `docker`
+//! on PATH with canned responses covering every read shape: a multi-arch
+//! manifest list carrying attestation entries, a single-arch config object, a
+//! labeled-but-hashless image, and an absent ref. The ref derivation (graph
+//! context column, dot-safe for models like gpt-5.4) is asserted against the
+//! real repo.
+
+use std::collections::HashMap;
+use std::path::PathBuf;
+use std::process::Command;
+use std::time::{SystemTime, UNIX_EPOCH};
+use test_support::repo_root;
+
+/// PATH-shimmed fake `docker`, removed on drop.
+struct Stub(PathBuf);
+
+impl Drop for Stub {
+    fn drop(&mut self) {
+        let _ = std::fs::remove_dir_all(&self.0);
+    }
+}
+
+fn write_stub(script_body: &str) -> Stub {
+    let nanos = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_nanos();
+    let dir = std::env::temp_dir().join(format!("fleet-status-{}-{nanos}", std::process::id()));
+    std::fs::create_dir_all(&dir).unwrap();
+    let path = dir.join("docker");
+    std::fs::write(&path, format!("#!/usr/bin/env bash\n{script_body}")).unwrap();
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap();
+    }
+    Stub(dir)
+}
+
+/// ref -> (verdict, computed, recorded)
+fn fleet_status(stub: &Stub) -> HashMap<String, (String, String, String)> {
+    let root = repo_root();
+    let path = format!(
+        "{}:{}",
+        stub.0.display(),
+        std::env::var("PATH").unwrap_or_default()
+    );
+    let out = Command::new("bash")
+        .arg(root.join("containers/scripts/fleet-status.sh"))
+        .env("PATH", path)
+        .env("STATUS_JOBS", "8")
+        .output()
+        .expect("run fleet-status.sh");
+    assert!(
+        out.status.success(),
+        "fleet-status failed:\n{}",
+        String::from_utf8_lossy(&out.stderr)
+    );
+    String::from_utf8(out.stdout)
+        .expect("utf8")
+        .lines()
+        .map(|l| {
+            let f: Vec<&str> = l.split('\t').collect();
+            assert_eq!(f.len(), 4, "malformed row: {l}");
+            (
+                f[0].to_string(),
+                (f[1].to_string(), f[2].to_string(), f[3].to_string()),
+            )
+        })
+        .collect()
+}
+
+/// Every verdict class, every registry read shape, and the dot-safe ref map,
+/// on the real repo with a stubbed registry.
+#[test]
+fn verdicts_cover_every_read_shape() {
+    let root = repo_root();
+    let hashes = Command::new("bash")
+        .arg(root.join("containers/scripts/fleet-hash.sh"))
+        .output()
+        .expect("run fleet-hash.sh");
+    assert!(hashes.status.success());
+    let aime = String::from_utf8(hashes.stdout)
+        .unwrap()
+        .lines()
+        .find(|l| l.starts_with("benchmark-aime\t"))
+        .expect("aime row")
+        .split('\t')
+        .nth(1)
+        .unwrap()
+        .to_string();
+
+    // aime: fresh via a manifest list (attestation entry must be ignored);
+    // gsm8k: stale via a single-arch config object; arc: labels but no hash.
+    // Everything else: inspect fails => absent.
+    let stub = write_stub(&format!(
+        r#"ref="$4"
+case "$ref" in
+  */benchmarks/aime:latest)
+    echo '{{"linux/amd64":{{"config":{{"Labels":{{"eval.input-hash":"{aime}"}}}}}},"unknown/unknown":{{"config":{{}}}}}}' ;;
+  */benchmarks/gsm8k:latest)
+    echo '{{"config":{{"Labels":{{"eval.input-hash":"deadbeef"}}}}}}' ;;
+  */benchmarks/arc:latest)
+    echo '{{"linux/amd64":{{"config":{{"Labels":{{"other":"x"}}}}}}}}' ;;
+  *) exit 1 ;;
+esac
+"#
+    ));
+    let rows = fleet_status(&stub);
+    assert_eq!(rows.len(), 153, "one row per static bake target");
+
+    let (v, want, got) = &rows["ghcr.io/exgentic/benchmarks/aime:latest"];
+    assert_eq!((v.as_str(), got), ("fresh", want));
+    assert_eq!(want, &aime);
+    assert_eq!(rows["ghcr.io/exgentic/benchmarks/gsm8k:latest"].0, "stale");
+    assert_eq!(
+        rows["ghcr.io/exgentic/benchmarks/gsm8k:latest"].2,
+        "deadbeef"
+    );
+    assert_eq!(
+        rows["ghcr.io/exgentic/benchmarks/arc:latest"].0,
+        "unlabeled"
+    );
+    assert_eq!(rows["ghcr.io/exgentic/core/entrypoint:latest"].0, "absent");
+
+    // The ref map preserves dots that bake target names cannot carry.
+    assert!(rows.contains_key("ghcr.io/exgentic/models/gpt-5.4:latest"));
+    assert!(rows.contains_key("ghcr.io/exgentic/models/gpt-4.1-mini:latest"));
+
+    // Rule 14: everything non-fresh is "changed"; exactly one image was fresh.
+    let fresh = rows.values().filter(|r| r.0 == "fresh").count();
+    assert_eq!(fresh, 1);
+}