From ceb2a788a2c37d2633a4b1ac948f19b413450399 Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Sun, 9 Aug 2026 12:43:11 +0300 Subject: [PATCH 1/5] =?UTF-8?q?feat(scripts):=20fleet-hash=20=E2=80=94=20d?= =?UTF-8?q?eterministic=20build-input=20hashes=20over=20the=20bake=20graph?= =?UTF-8?q?=20(#297)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hash primitive for carried-forward releases (delivery/RULES.md rules 11-14): hash(target) = sha256(context tree hash + sorted recursive base hashes), read off the bake graph that principle 15.d keeps aligned with the Dockerfiles. Three providers behind one entry point: fleet-hash.sh # all 153 static bake targets fleet-hash.sh combo # eval + eval-standalone rows fleet-hash.sh per-task # per-task image row Output decomposes into hash / context-hash / bases-hash / externals so a consumer can tell WHY a target moved (context edit vs base cascade vs upstream drift). External FROM refs are emitted unresolved — digest resolution needs the network and happens at release time (rule 11); the script itself is offline and a pure function of the committed tree at REF. Design notes proven by tests/static/input_hash.rs on a synthetic git fixture: deterministic across runs, a leaf edit moves only that leaf's context component, a base edit cascades through dependents' bases component, REF pins the whole computation to a commit. The real-repo test asserts one row per per-artifact bake file and exercises all three providers. Combos hash from the combination Dockerfiles + parent target hashes, not the over-broad containers/core context. Dir-name lookups are context-based, so dotted model dirs (gpt-5.4) resolve despite their underscored target names. SQL FROM clauses on RUN continuation lines are not image refs. Portable to bash 3.2 (macOS); one awk parse + one batched git rev-parse keeps a full-fleet run ~5s. Part of #292's migration path, rung 2 of 5. Resolves #297. Signed-off-by: Elron Bandel Signed-off-by: Elron Bandel --- containers/scripts/fleet-hash.sh | 208 ++++++++++++++++++++++++++++ tests/static/Cargo.toml | 4 + tests/static/input_hash.rs | 230 +++++++++++++++++++++++++++++++ 3 files changed, 442 insertions(+) create mode 100755 containers/scripts/fleet-hash.sh create mode 100644 tests/static/input_hash.rs diff --git a/containers/scripts/fleet-hash.sh b/containers/scripts/fleet-hash.sh new file mode 100755 index 00000000..7dcad47a --- /dev/null +++ b/containers/scripts/fleet-hash.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +# fleet-hash — deterministic build-input hashes for every fleet image +# (delivery/RULES.md rules 11–14). +# +# The hash is a pure function of the committed tree at REF: +# hash(target) = sha256(context tree hash + sorted recursive base hashes) +# read off the bake graph (each target's context dir + its `target:` edges, +# which principle 15.d keeps aligned with the Dockerfile's FROMs). External +# base images are emitted as refs only — resolving their digests needs the +# network, so it happens at release time (rule 11); this script stays offline. +# +# Usage: +# fleet-hash.sh # every static bake target +# fleet-hash.sh combo # eval + eval-standalone rows +# fleet-hash.sh per-task # one per-task image row +# +# Output (TSV): target hash context-hash bases-hash externals +# Env: REF (default HEAD), REPO_ROOT (default: the repo containing this script) +# +# Portable to bash 3.2 (macOS, no associative arrays) and sized for the +# static-stage time budget: one awk parses every bake file, one git call +# hashes every context, maps are eval'd shell variables. +# +# shellcheck disable=SC2034,SC2154 # map variables are assigned and read via eval +set -euo pipefail +shopt -s nullglob + +REF="${REF:-HEAD}" +REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "$0")/../.." && pwd)}" +cd "$REPO_ROOT" + +# Captured before `set --` reuses the positional parameters below. +CMD="${1:-all}" NARGS=$# A2="${2:-}" A3="${3:-}" + +die() { echo "fleet-hash: $*" >&2; exit 2; } + +sha() { + local out + if command -v sha256sum >/dev/null 2>&1; then out=$(sha256sum); printf '%s' "${out%% *}" + else out=$(openssl dgst -sha256); printf '%s' "${out##*= }"; fi +} + +# Map key: target names are [A-Za-z0-9_-] (HCL forbids dots), so dash→underscore +# is the only rewrite; a collision between two live names would need a -/_ pair +# and is caught by the duplicate check below. Directory keys additionally carry +# slashes and dots (models/gpt-5.4), so dkey squashes every non-alnum char. +key() { printf '%s' "${1//-/_}"; } +dkey() { printf '%s' "${1//[^a-zA-Z0-9]/_}"; } + +# ── parse the bake graph: one awk over every per-artifact bake file ───────── +# (the parameterized combination file sits directly in containers/core/, so +# the subdir glob naturally excludes it; principle 15.a fixes one target per +# file, first `context =` wins, `contexts =` never matches the context line) +FILES=() +for f in containers/core/*/docker-bake.hcl containers/gateways/*/docker-bake.hcl \ + containers/agents/*/docker-bake.hcl containers/benchmarks/*/docker-bake.hcl \ + containers/models/*/docker-bake.hcl; do FILES+=("$f"); done +[ "${#FILES[@]}" -gt 0 ] || die "no bake files under $REPO_ROOT/containers" + +PARSED=$(awk ' + FNR==1 { tgt=""; ctx="" } + /^target "/ && tgt=="" { split($0, q, "\""); tgt=q[2]; deps[tgt]="" } + $1=="context" && $2=="=" && ctx=="" { split($0, q, "\""); ctx=q[2]; ctxof[tgt]=ctx } + { + s=$0 + while (match(s, /"target:[A-Za-z0-9_-]+"/)) { + d=substr(s, RSTART+8, RLENGTH-9) + if (index(" " deps[tgt] " ", " " d " ")==0) deps[tgt]=deps[tgt] d " " + s=substr(s, RSTART+RLENGTH) + } + } + END { for (t in ctxof) print t "|" ctxof[t] "|" deps[t] } +' "${FILES[@]}" | LC_ALL=C sort) + +TARGETS="" +PATHS=() +while IFS='|' read -r tgt ctx deps; do + [ -n "$tgt" ] || continue + k=$(key "$tgt") + eval "prev=\${NAME_$k:-}" + [ -z "$prev" ] || die "duplicate/colliding target $tgt vs $prev" + eval "NAME_$k=\$tgt CTX_$k=\$ctx DEPS_$k=\$deps" + TARGETS="$TARGETS $tgt" + PATHS+=("$REF:$ctx") +done </dev/null) || { + for p in "${PATHS[@]}"; do + git rev-parse "$p" >/dev/null 2>&1 || die "context ${p#"$REF":} is not in $REF (uncommitted?)" + done + die "git rev-parse failed" +} +i=0 +# shellcheck disable=SC2086 # TARGETS is a space-separated list, split intended +set -- $TARGETS +while IFS= read -r h; do + i=$((i+1)); eval "TREE_$(key "$1")=\$h"; shift +done </dev/null || die "$1 is not in $REF"; } + +case "$CMD" in +all) + for t in $TARGETS; do resolve "$t"; done + for t in $TARGETS; do out "$t"; done + ;; +combo) + [ "$NARGS" -eq 3 ] || die "usage: fleet-hash.sh combo " + b=$(target_for_dir "containers/benchmarks/$A2") + a=$(target_for_dir "containers/agents/$A3") + for t in "$b" "$a" gosu otel process-compose model-bifrost; do resolve "$t"; done + # The combination context is all of containers/core (over-broad); the real + # inputs are the two Dockerfiles + the bake file + the parent images. + eval "bh=\$HASH_$(key "$b")"; eval "ah=\$HASH_$(key "$a")" + eval_bases=$(printf '%s %s %s' "$bh" "$ah" "$HASH_gosu" | sha) + eval_ctx=$(printf '%s %s' "$(blob containers/core/combination.Dockerfile)" \ + "$(blob containers/core/combination.docker-bake.hcl)" | sha) + eval_hash=$(printf '%s %s' "$eval_ctx" "$eval_bases" | sha) + row "evals/$A2--$A3" "$eval_hash" "$eval_ctx" "$eval_bases" "-" + sa_bases=$(printf '%s %s %s %s' "$eval_hash" "$HASH_otel" \ + "$HASH_process_compose" "$HASH_model_bifrost" | sha) + sa_ctx=$(blob containers/core/standalone.Dockerfile) + row "evals/$A2--$A3-standalone" "$(printf '%s %s' "$sa_ctx" "$sa_bases" | sha)" \ + "$sa_ctx" "$sa_bases" "-" + ;; +per-task) + [ "$NARGS" -eq 3 ] || die "usage: fleet-hash.sh per-task " + [ -z "${SKILLS_BENCH_REF:-}" ] || die "SKILLS_BENCH_REF is set — an out-of-tree ref override defeats input hashing; pin the ref in the benchmark dir" + t=$(target_for_dir "containers/benchmarks/$A2") + resolve "$t" + k=$(key "$t") + eval "th=\$HASH_$k ch=\$CTXH_$k bsh=\$BASESH_$k" + row "per-task/$A2/$A3" "$(printf '%s %s' "$th" "$A3" | sha)" "$ch" "$bsh" "$(ext_of "$t")" + ;; +*) + die "unknown command $CMD (expected: all | combo | per-task)" + ;; +esac diff --git a/tests/static/Cargo.toml b/tests/static/Cargo.toml index c55e1705..b9ed1336 100644 --- a/tests/static/Cargo.toml +++ b/tests/static/Cargo.toml @@ -35,3 +35,7 @@ path = "task_inspection.rs" [[test]] name = "grader" path = "grader.rs" + +[[test]] +name = "input_hash" +path = "input_hash.rs" diff --git a/tests/static/input_hash.rs b/tests/static/input_hash.rs new file mode 100644 index 00000000..3e6c90b7 --- /dev/null +++ b/tests/static/input_hash.rs @@ -0,0 +1,230 @@ +//! Build-input hash primitive — the "repository's computed hash" side of the +//! carried-forward contract (delivery/RULES.md rules 11–14). +//! +//! `containers/scripts/fleet-hash.sh` must be a pure function of the committed +//! tree: deterministic, sensitive to any context change, and cascading through +//! the bake graph so a base edit dirties every dependent leaf. These are the +//! properties the selective-release machinery will trust, so they are proven +//! here on a synthetic git fixture (where we can commit mutations) and the +//! script is exercised over the real repo for coverage. Offline, daemon-free +//! (tests/static/RULES.md rule 1): only `git`, `bash`, and awk/sed run. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::process::Command; +use test_support::repo_root; + +fn script() -> PathBuf { + repo_root().join("containers/scripts/fleet-hash.sh") +} + +fn fleet_hash(repo: &Path, args: &[&str]) -> String { + let out = Command::new("bash") + .arg(script()) + .args(args) + .env("REPO_ROOT", repo) + .env("REF", "HEAD") + .output() + .expect("run fleet-hash.sh"); + assert!( + out.status.success(), + "fleet-hash {args:?} failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).expect("fleet-hash output is utf8") +} + +/// target -> (hash, context-hash, bases-hash, externals) +fn rows(output: &str) -> HashMap { + output + .lines() + .map(|l| { + let f: Vec<&str> = l.split('\t').collect(); + assert_eq!(f.len(), 5, "malformed row: {l}"); + ( + f[0].to_string(), + ( + f[1].to_string(), + f[2].to_string(), + f[3].to_string(), + f[4].to_string(), + ), + ) + }) + .collect() +} + +fn is_sha256(s: &str) -> bool { + s.len() == 64 && s.chars().all(|c| c.is_ascii_hexdigit()) +} + +// ── synthetic fixture ─────────────────────────────────────────────────────── + +fn git(repo: &Path, args: &[&str]) { + let out = Command::new("git") + .args([ + "-c", + "user.email=test@test", + "-c", + "user.name=test", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .current_dir(repo) + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_SYSTEM", "/dev/null") + .output() + .expect("run git"); + assert!( + out.status.success(), + "git {args:?} failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); +} + +fn write(repo: &Path, rel: &str, content: &str) { + let p = repo.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, content).unwrap(); +} + +/// A minimal fleet: one core base (external FROM), one leaf depending on it, +/// one independent leaf. Committed to a throwaway git repo so mutations can +/// be committed and hashed like the real tree. +fn fixture(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("fleet-hash-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + write( + &dir, + "containers/core/base-x/Dockerfile", + "FROM alpine:3.20\nRUN echo base\n", + ); + write( + &dir, + "containers/core/base-x/docker-bake.hcl", + "target \"base-x\" {\n context = \"containers/core/base-x\"\n tags = [\"${REGISTRY}/core/base-x:${TAG}\"]\n}\n", + ); + write( + &dir, + "containers/benchmarks/leaf-a/Dockerfile", + "FROM ${REGISTRY}/core/base-x:latest\nRUN echo a\n", + ); + write( + &dir, + "containers/benchmarks/leaf-a/docker-bake.hcl", + "target \"benchmark-leaf-a\" {\n context = \"containers/benchmarks/leaf-a\"\n contexts = {\n \"${REGISTRY}/core/base-x\" = \"target:base-x\"\n }\n tags = [\"${REGISTRY}/benchmarks/leaf-a:${TAG}\"]\n}\n", + ); + write( + &dir, + "containers/benchmarks/leaf-b/Dockerfile", + "FROM debian:12-slim\nRUN echo b\n", + ); + write( + &dir, + "containers/benchmarks/leaf-b/docker-bake.hcl", + "target \"benchmark-leaf-b\" {\n context = \"containers/benchmarks/leaf-b\"\n tags = [\"${REGISTRY}/benchmarks/leaf-b:${TAG}\"]\n}\n", + ); + git(&dir, &["init", "-q"]); + git(&dir, &["add", "."]); + git(&dir, &["commit", "-q", "-m", "fixture"]); + dir +} + +/// Deterministic; a leaf edit changes only that leaf (context component); a +/// base edit cascades to its dependents (bases component) and spares the +/// rest; REF pins the whole computation to a commit. +#[test] +fn hash_is_deterministic_source_sensitive_and_cascading() { + let repo = fixture("props"); + + let first = fleet_hash(&repo, &[]); + assert_eq!(first, fleet_hash(&repo, &[]), "two runs must be identical"); + let v0 = rows(&first); + assert_eq!(v0.len(), 3); + for (t, (hash, ctxh, basesh, _)) in &v0 { + assert!(is_sha256(hash), "{t}: hash not sha256"); + assert_eq!(ctxh.len(), 40, "{t}: context hash not a git tree hash"); + assert!(is_sha256(basesh), "{t}: bases hash not sha256"); + } + assert_eq!(v0["base-x"].3, "alpine:3.20", "external FROM must surface"); + assert_eq!( + v0["benchmark-leaf-a"].3, "-", + "in-repo FROM is not external" + ); + assert_eq!(v0["benchmark-leaf-b"].3, "debian:12-slim"); + + // Leaf edit: only leaf-a moves, and only its context component. + write(&repo, "containers/benchmarks/leaf-a/extra.txt", "changed\n"); + git(&repo, &["add", "."]); + git(&repo, &["commit", "-q", "-m", "edit leaf-a"]); + let v1 = rows(&fleet_hash(&repo, &[])); + assert_ne!(v1["benchmark-leaf-a"].0, v0["benchmark-leaf-a"].0); + assert_ne!(v1["benchmark-leaf-a"].1, v0["benchmark-leaf-a"].1); + assert_eq!(v1["benchmark-leaf-a"].2, v0["benchmark-leaf-a"].2); + assert_eq!(v1["base-x"], v0["base-x"]); + assert_eq!(v1["benchmark-leaf-b"], v0["benchmark-leaf-b"]); + + // Base edit: base-x and its dependent leaf-a move; leaf-a via its bases + // component only; independent leaf-b is untouched. + write(&repo, "containers/core/base-x/extra.txt", "changed\n"); + git(&repo, &["add", "."]); + git(&repo, &["commit", "-q", "-m", "edit base-x"]); + let v2 = rows(&fleet_hash(&repo, &[])); + assert_ne!(v2["base-x"].0, v1["base-x"].0); + assert_ne!(v2["benchmark-leaf-a"].0, v1["benchmark-leaf-a"].0); + assert_eq!(v2["benchmark-leaf-a"].1, v1["benchmark-leaf-a"].1); + assert_ne!(v2["benchmark-leaf-a"].2, v1["benchmark-leaf-a"].2); + assert_eq!(v2["benchmark-leaf-b"], v1["benchmark-leaf-b"]); + + // REF pins the computation: hashing HEAD~1 reproduces the prior state. + let pinned = Command::new("bash") + .arg(script()) + .env("REPO_ROOT", &repo) + .env("REF", "HEAD~1") + .output() + .expect("run fleet-hash.sh at HEAD~1"); + assert!(pinned.status.success()); + assert_eq!(rows(&String::from_utf8(pinned.stdout).unwrap()), v1); + + let _ = std::fs::remove_dir_all(&repo); +} + +/// The real repo: every per-artifact bake file yields exactly one row, and +/// the combo/per-task providers produce well-formed hashes. +#[test] +fn real_repo_hashes_every_target() { + let root = repo_root(); + let bake_files: usize = ["core", "gateways", "agents", "benchmarks", "models"] + .iter() + .map(|kind| { + std::fs::read_dir(root.join("containers").join(kind)) + .expect("read kind dir") + .filter_map(Result::ok) + .filter(|e| e.path().join("docker-bake.hcl").is_file()) + .count() + }) + .sum(); + + let all = rows(&fleet_hash(&root, &[])); + assert_eq!( + all.len(), + bake_files, + "one row per per-artifact bake file (duplicates collapse in the map)" + ); + for (t, (hash, _, _, _)) in &all { + assert!(is_sha256(hash), "{t}: hash not sha256"); + } + + let combo = rows(&fleet_hash(&root, &["combo", "aime", "claude-code"])); + assert_eq!(combo.len(), 2); + assert!(combo.contains_key("evals/aime--claude-code")); + assert!(combo.contains_key("evals/aime--claude-code-standalone")); + + let pt = rows(&fleet_hash( + &root, + &["per-task", "terminal-bench", "task-0"], + )); + assert!(is_sha256(&pt["per-task/terminal-bench/task-0"].0)); +} From cb226419ef16e6f9c5ae3bda60cd044fa149d27e Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Sun, 9 Aug 2026 15:02:42 +0300 Subject: [PATCH 2/5] polish: drop dead counter, thread REF through the test helper, note combo-list drift safety Signed-off-by: Elron Bandel --- containers/scripts/fleet-hash.sh | 8 +++++--- tests/static/input_hash.rs | 17 +++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/containers/scripts/fleet-hash.sh b/containers/scripts/fleet-hash.sh index 7dcad47a..4c96aa33 100755 --- a/containers/scripts/fleet-hash.sh +++ b/containers/scripts/fleet-hash.sh @@ -93,11 +93,10 @@ TREES=$(git rev-parse "${PATHS[@]}" 2>/dev/null) || { done die "git rev-parse failed" } -i=0 # shellcheck disable=SC2086 # TARGETS is a space-separated list, split intended set -- $TARGETS while IFS= read -r h; do - i=$((i+1)); eval "TREE_$(key "$1")=\$h"; shift + eval "TREE_$(key "$1")=\$h"; shift done < PathBuf { repo_root().join("containers/scripts/fleet-hash.sh") } -fn fleet_hash(repo: &Path, args: &[&str]) -> String { +fn fleet_hash_at(repo: &Path, git_ref: &str, args: &[&str]) -> String { let out = Command::new("bash") .arg(script()) .args(args) .env("REPO_ROOT", repo) - .env("REF", "HEAD") + .env("REF", git_ref) .output() .expect("run fleet-hash.sh"); assert!( @@ -34,6 +34,10 @@ fn fleet_hash(repo: &Path, args: &[&str]) -> String { String::from_utf8(out.stdout).expect("fleet-hash output is utf8") } +fn fleet_hash(repo: &Path, args: &[&str]) -> String { + fleet_hash_at(repo, "HEAD", args) +} + /// target -> (hash, context-hash, bases-hash, externals) fn rows(output: &str) -> HashMap { output @@ -179,14 +183,7 @@ fn hash_is_deterministic_source_sensitive_and_cascading() { assert_eq!(v2["benchmark-leaf-b"], v1["benchmark-leaf-b"]); // REF pins the computation: hashing HEAD~1 reproduces the prior state. - let pinned = Command::new("bash") - .arg(script()) - .env("REPO_ROOT", &repo) - .env("REF", "HEAD~1") - .output() - .expect("run fleet-hash.sh at HEAD~1"); - assert!(pinned.status.success()); - assert_eq!(rows(&String::from_utf8(pinned.stdout).unwrap()), v1); + assert_eq!(rows(&fleet_hash_at(&repo, "HEAD~1", &[])), v1); let _ = std::fs::remove_dir_all(&repo); } From 57b98fc9b5a0c0a4f2996792a63eb26ce694bd59 Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Sun, 9 Aug 2026 15:09:40 +0300 Subject: [PATCH 3/5] =?UTF-8?q?refactor:=20flat-closure=20hash=20=E2=80=94?= =?UTF-8?q?=20no=20recursion,=20no=20eval=20maps,=20one=20sha=20pass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hash(target) = sha256 of the sorted tree hashes of the target's context and every transitive base context. Sensitivity-equivalent to the Merkle chain (wiring changes edit bake files, which live inside a hashed context) but the whole computation is three small awk programs, one batched git rev-parse, and one sha256sum invocation: ~12 forks instead of ~460, 24s -> 5s -> 0.3s, and the bash 3.2 eval-map workaround is gone entirely. Combos become a flat union of parent closures + the combination file blobs; standalone folds the eval set in directly. Signed-off-by: Elron Bandel --- containers/scripts/fleet-hash.sh | 261 +++++++++++++------------------ 1 file changed, 111 insertions(+), 150 deletions(-) diff --git a/containers/scripts/fleet-hash.sh b/containers/scripts/fleet-hash.sh index 4c96aa33..843fc727 100755 --- a/containers/scripts/fleet-hash.sh +++ b/containers/scripts/fleet-hash.sh @@ -2,12 +2,14 @@ # fleet-hash — deterministic build-input hashes for every fleet image # (delivery/RULES.md rules 11–14). # -# The hash is a pure function of the committed tree at REF: -# hash(target) = sha256(context tree hash + sorted recursive base hashes) -# read off the bake graph (each target's context dir + its `target:` edges, -# which principle 15.d keeps aligned with the Dockerfile's FROMs). External -# base images are emitted as refs only — resolving their digests needs the -# network, so it happens at release time (rule 11); this script stays offline. +# hash(target) = sha256 of the sorted git tree hashes of the target's build +# context and every transitive in-repo base context — a pure function of the +# committed tree at REF, read off the bake graph (principle 15.d keeps each +# target's `contexts` aligned with its Dockerfile's FROMs). A flat set is +# sensitivity-equivalent to a Merkle chain here: wiring changes edit bake +# files, which live inside a hashed context. External FROMs are emitted as +# unresolved refs; digest resolution needs the network and happens at release +# time (rule 11), keeping this script offline. # # Usage: # fleet-hash.sh # every static bake target @@ -16,12 +18,6 @@ # # Output (TSV): target hash context-hash bases-hash externals # Env: REF (default HEAD), REPO_ROOT (default: the repo containing this script) -# -# Portable to bash 3.2 (macOS, no associative arrays) and sized for the -# static-stage time budget: one awk parses every bake file, one git call -# hashes every context, maps are eval'd shell variables. -# -# shellcheck disable=SC2034,SC2154 # map variables are assigned and read via eval set -euo pipefail shopt -s nullglob @@ -29,38 +25,27 @@ REF="${REF:-HEAD}" REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "$0")/../.." && pwd)}" cd "$REPO_ROOT" -# Captured before `set --` reuses the positional parameters below. -CMD="${1:-all}" NARGS=$# A2="${2:-}" A3="${3:-}" - die() { echo "fleet-hash: $*" >&2; exit 2; } +sha() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$@"; else shasum -a 256 "$@"; fi; } +hash_of() { sha < "$1" | cut -d' ' -f1; } +row() { printf '%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "$4" "$5"; } +M=$(mktemp -d) && trap 'rm -rf "$M"' EXIT && mkdir "$M/full" "$M/bases" -sha() { - local out - if command -v sha256sum >/dev/null 2>&1; then out=$(sha256sum); printf '%s' "${out%% *}" - else out=$(openssl dgst -sha256); printf '%s' "${out##*= }"; fi -} - -# Map key: target names are [A-Za-z0-9_-] (HCL forbids dots), so dash→underscore -# is the only rewrite; a collision between two live names would need a -/_ pair -# and is caught by the duplicate check below. Directory keys additionally carry -# slashes and dots (models/gpt-5.4), so dkey squashes every non-alnum char. -key() { printf '%s' "${1//-/_}"; } -dkey() { printf '%s' "${1//[^a-zA-Z0-9]/_}"; } - -# ── parse the bake graph: one awk over every per-artifact bake file ───────── -# (the parameterized combination file sits directly in containers/core/, so -# the subdir glob naturally excludes it; principle 15.a fixes one target per -# file, first `context =` wins, `contexts =` never matches the context line) -FILES=() -for f in containers/core/*/docker-bake.hcl containers/gateways/*/docker-bake.hcl \ - containers/agents/*/docker-bake.hcl containers/benchmarks/*/docker-bake.hcl \ - containers/models/*/docker-bake.hcl; do FILES+=("$f"); done +# ── graph: one awk over every per-artifact bake file → target|context|deps ── +# (one target per file, principle 15.a; the parameterized combination file +# sits directly in containers/core/, outside the subdir glob) +FILES=(containers/core/*/docker-bake.hcl containers/gateways/*/docker-bake.hcl + containers/agents/*/docker-bake.hcl containers/benchmarks/*/docker-bake.hcl + containers/models/*/docker-bake.hcl) [ "${#FILES[@]}" -gt 0 ] || die "no bake files under $REPO_ROOT/containers" - -PARSED=$(awk ' - FNR==1 { tgt=""; ctx="" } - /^target "/ && tgt=="" { split($0, q, "\""); tgt=q[2]; deps[tgt]="" } - $1=="context" && $2=="=" && ctx=="" { split($0, q, "\""); ctx=q[2]; ctxof[tgt]=ctx } +awk ' + FNR==1 { tgt="" } + /^target "/ && tgt=="" { + split($0, q, "\""); tgt=q[2] + if (tgt in seen) { print "fleet-hash: duplicate target " tgt > "/dev/stderr"; exit 2 } + seen[tgt]=1; ctx[tgt]=""; deps[tgt]="" + } + $1=="context" && $2=="=" && ctx[tgt]=="" { split($0, q, "\""); ctx[tgt]=q[2] } { s=$0 while (match(s, /"target:[A-Za-z0-9_-]+"/)) { @@ -69,142 +54,118 @@ PARSED=$(awk ' s=substr(s, RSTART+RLENGTH) } } - END { for (t in ctxof) print t "|" ctxof[t] "|" deps[t] } -' "${FILES[@]}" | LC_ALL=C sort) + END { for (t in ctx) print t "|" ctx[t] "|" deps[t] } +' "${FILES[@]}" | LC_ALL=C sort > "$M/graph" -TARGETS="" -PATHS=() -while IFS='|' read -r tgt ctx deps; do - [ -n "$tgt" ] || continue - k=$(key "$tgt") - eval "prev=\${NAME_$k:-}" - [ -z "$prev" ] || die "duplicate/colliding target $tgt vs $prev" - eval "NAME_$k=\$tgt CTX_$k=\$ctx DEPS_$k=\$deps" - TARGETS="$TARGETS $tgt" - PATHS+=("$REF:$ctx") -done </dev/null) || { - for p in "${PATHS[@]}"; do - git rev-parse "$p" >/dev/null 2>&1 || die "context ${p#"$REF":} is not in $REF (uncommitted?)" - done +# ── tree hashes: one git call over every context, paired by row order ─────── +# shellcheck disable=SC2046 # context paths never contain spaces +git rev-parse $(awk -F'|' -v r="$REF" '{print r ":" $2}' "$M/graph") > "$M/hashes" 2>/dev/null || { + while IFS='|' read -r t ctx _; do + git rev-parse "$REF:$ctx" >/dev/null 2>&1 || die "context $ctx of $t is not in $REF (uncommitted?)" + done < "$M/graph" die "git rev-parse failed" } -# shellcheck disable=SC2086 # TARGETS is a space-separated list, split intended -set -- $TARGETS -while IFS= read -r h; do - eval "TREE_$(key "$1")=\$h"; shift -done < "$M/trees" + +# ── closures: recursive walk in awk → one sorted tree-hash file per target ── +# full/ holds the target's own context tree + every transitive base tree; +# bases/ holds the base trees only (the cascade component). +while IFS='|' read -r t _; do : > "$M/full/$t"; : > "$M/bases/$t"; done < "$M/graph" +awk -F'|' ' + FNR==NR { ctx[$1]=$2; deps[$1]=$3; order[++n]=$1; next } + { tree[$1]=$2 } + END { for (i=1; i<=n; i++) { t=order[i]; delete hit; walk(t, t, 1) } } + function walk(root, t, isroot, m, p, j) { + if (t in hit) return; hit[t]=1 + if (!(t in ctx)) { print "fleet-hash: " root " depends on unknown target " t > "/dev/stderr"; exit 2 } + print "F|" root "|" tree[t] + if (!isroot) print "B|" root "|" tree[t] + m = split(deps[t], p, " ") + for (j=1; j<=m; j++) if (p[j]!="") walk(root, p[j], 0) + } +' "$M/graph" "$M/trees" | LC_ALL=C sort -u \ + | awk -F'|' -v m="$M" '{ f = m "/" ($1=="F" ? "full" : "bases") "/" $2; print $3 >> f }' -# ── externals: one awk over every context Dockerfile ──────────────────────── -# FROM refs that are neither a prior build stage, scratch, nor an in-repo -# ${REGISTRY} image; their ref strings are already inside the context tree -# hash — this column only feeds release-time digest resolution. +# ── externals: one awk over every context Dockerfile → dir|image ──────────── +# A FROM on a continuation line (trailing backslash above) is inside a RUN — +# e.g. SQL `FROM 'hf://…'` in duckdb heredocs — never a Dockerfile instruction. DFS=() -for t in $TARGETS; do - eval "ctx=\$CTX_$(key "$t")" +while IFS='|' read -r t ctx _; do [ -f "$ctx/Dockerfile" ] || die "$ctx/Dockerfile missing (target $t)" DFS+=("$ctx/Dockerfile") -done -# A FROM on a continuation line (trailing backslash above it) is inside a RUN — -# e.g. SQL `FROM 'hf://…'` in duckdb heredocs — never a Dockerfile instruction. -EXTS=$(awk ' +done < "$M/graph" +awk ' FNR==1 { delete alias; cont=0 } !cont && toupper($1)=="FROM" { img=$2; if (img ~ /^--platform/) img=$3 for (i=1; i<=NF; i++) if (toupper($i)=="AS") alias[$(i+1)]=1 - if (!(img in alias) && img!="scratch" && index(img,"${REGISTRY}")==0) - print FILENAME "|" img + if (!(img in alias) && img!="scratch" && index(img,"${REGISTRY}")==0) { + d=FILENAME; sub(/\/Dockerfile$/, "", d); print d "|" img + } } { cont = ($0 ~ /\\[[:space:]]*$/) } -' "${DFS[@]}" | LC_ALL=C sort -u) -while IFS='|' read -r df img; do - [ -n "$df" ] || continue - k=$(dkey "${df%/Dockerfile}") - eval "cur=\${EXT_$k:-}" - eval "EXT_$k=\"\${cur:+\$cur,}\$img\"" -done < "$M/ext" -# ── recursive hash with memoization ───────────────────────────────────────── -resolve() { - local t=$1 k dep deps dh="" ctxh bh h - k=$(key "$t") - eval "[ -z \"\${HASH_$k:-}\" ]" || return 0 - eval "[ -z \"\${VIS_$k:-}\" ]" || die "dependency cycle at $t" - eval "VIS_$k=1" - eval "ctxh=\${TREE_$k:-}" - [ -n "$ctxh" ] || die "unknown target $t" - eval "deps=\$DEPS_$k" - for dep in $deps; do - resolve "$dep" - eval "dh=\"\$dh\${HASH_$(key "$dep")} \"" - done - bh=$(printf '%s' "$dh" | sha) - h=$(printf '%s %s' "$ctxh" "$dh" | sha) - eval "CTXH_$k=\$ctxh BASESH_$k=\$bh HASH_$k=\$h VIS_$k=" -} - -row() { printf '%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "$4" "$5"; } -out() { local k; k=$(key "$1"); eval "row \"\$1\" \"\$HASH_$k\" \"\$CTXH_$k\" \"\$BASESH_$k\" \"\$(ext_of \"\$1\")\""; } +# ── one sha pass over every closure file, then a single join → the TSV ────── +(cd "$M" && sha full/* bases/*) > "$M/sums" +awk ' + BEGIN { FS="|" } + FILENAME ~ /graph$/ { ctxdir[$1]=$2; order[++n]=$1; next } + FILENAME ~ /trees$/ { tree[$1]=$2; next } + FILENAME ~ /ext$/ { ext[$1] = ($1 in ext) ? ext[$1] "," $2 : $2; next } + { + split($0, a, / +/) + if (split(a[2], b, "/") == 2) { if (b[1]=="full") full[b[2]]=a[1]; else bases[b[2]]=a[1] } + } + END { + for (i=1; i<=n; i++) { + t=order[i]; e=ext[ctxdir[t]] + print t "\t" full[t] "\t" tree[t] "\t" bases[t] "\t" (e=="" ? "-" : e) + } + } +' "$M/graph" "$M/trees" "$M/ext" "$M/sums" > "$M/all.tsv" -# Target whose context is the given artifact dir — name-agnostic, so targets -# whose name mangles the dir (dots → underscores) still resolve. +col() { awk -F'\t' -v t="$1" -v c="$2" '$1==t { print $c }' "$M/all.tsv"; } target_for_dir() { - local dir=$1 t ctx - for t in $TARGETS; do - eval "ctx=\$CTX_$(key "$t")" - [ "$ctx" = "$dir" ] && { printf '%s' "$t"; return; } - done - die "no bake target with context $dir" + awk -F'|' -v d="$1" '$2==d { print $1; found=1 } END { exit !found }' "$M/graph" \ + || die "no bake target with context $1" } +blobs() { git rev-parse "$@" 2>/dev/null || die "blob not in $REF"; } -blob() { git rev-parse "$REF:$1" 2>/dev/null || die "$1 is not in $REF"; } - -case "$CMD" in +case "${1:-all}" in all) - for t in $TARGETS; do resolve "$t"; done - for t in $TARGETS; do out "$t"; done + cat "$M/all.tsv" ;; combo) - [ "$NARGS" -eq 3 ] || die "usage: fleet-hash.sh combo " - b=$(target_for_dir "containers/benchmarks/$A2") - a=$(target_for_dir "containers/agents/$A3") - for t in "$b" "$a" gosu otel process-compose model-bifrost; do resolve "$t"; done + [ $# -eq 3 ] || die "usage: fleet-hash.sh combo " + b=$(target_for_dir "containers/benchmarks/$2") + a=$(target_for_dir "containers/agents/$3") # The combination context is all of containers/core (over-broad); the real - # inputs are the two Dockerfiles + the bake file + the parent images. The + # inputs are the combination files' blobs plus the parents' closures. The # parent list mirrors combination.docker-bake.hcl's variable defaults — - # changing that list edits the bake file, whose blob is hashed below, so - # every combo hash moves the moment the list drifts. - eval "bh=\$HASH_$(key "$b")"; eval "ah=\$HASH_$(key "$a")" - eval_bases=$(printf '%s %s %s' "$bh" "$ah" "$HASH_gosu" | sha) - eval_ctx=$(printf '%s %s' "$(blob containers/core/combination.Dockerfile)" \ - "$(blob containers/core/combination.docker-bake.hcl)" | sha) - eval_hash=$(printf '%s %s' "$eval_ctx" "$eval_bases" | sha) - row "evals/$A2--$A3" "$eval_hash" "$eval_ctx" "$eval_bases" "-" - sa_bases=$(printf '%s %s %s %s' "$eval_hash" "$HASH_otel" \ - "$HASH_process_compose" "$HASH_model_bifrost" | sha) - sa_ctx=$(blob containers/core/standalone.Dockerfile) - row "evals/$A2--$A3-standalone" "$(printf '%s %s' "$sa_ctx" "$sa_bases" | sha)" \ - "$sa_ctx" "$sa_bases" "-" + # changing that list edits a hashed blob, so every combo hash moves with it. + blobs "$REF:containers/core/combination.Dockerfile" \ + "$REF:containers/core/combination.docker-bake.hcl" | LC_ALL=C sort > "$M/eval.ctx" + LC_ALL=C sort -u "$M/full/$b" "$M/full/$a" "$M/full/gosu" > "$M/eval.bases" + LC_ALL=C sort -u "$M/eval.ctx" "$M/eval.bases" > "$M/eval.full" + row "evals/$2--$3" "$(hash_of "$M/eval.full")" "$(hash_of "$M/eval.ctx")" \ + "$(hash_of "$M/eval.bases")" "-" + blobs "$REF:containers/core/standalone.Dockerfile" > "$M/sa.ctx" + LC_ALL=C sort -u "$M/eval.full" "$M/full/otel" "$M/full/process-compose" \ + "$M/full/model-bifrost" > "$M/sa.bases" + LC_ALL=C sort -u "$M/sa.ctx" "$M/sa.bases" > "$M/sa.full" + row "evals/$2--$3-standalone" "$(hash_of "$M/sa.full")" "$(hash_of "$M/sa.ctx")" \ + "$(hash_of "$M/sa.bases")" "-" ;; per-task) - [ "$NARGS" -eq 3 ] || die "usage: fleet-hash.sh per-task " + [ $# -eq 3 ] || die "usage: fleet-hash.sh per-task " [ -z "${SKILLS_BENCH_REF:-}" ] || die "SKILLS_BENCH_REF is set — an out-of-tree ref override defeats input hashing; pin the ref in the benchmark dir" - t=$(target_for_dir "containers/benchmarks/$A2") - resolve "$t" - k=$(key "$t") - eval "th=\$HASH_$k ch=\$CTXH_$k bsh=\$BASESH_$k" - row "per-task/$A2/$A3" "$(printf '%s %s' "$th" "$A3" | sha)" "$ch" "$bsh" "$(ext_of "$t")" + t=$(target_for_dir "containers/benchmarks/$2") + h=$(col "$t" 2) + row "per-task/$2/$3" "$(printf '%s %s' "$h" "$3" | sha | cut -d' ' -f1)" \ + "$(col "$t" 3)" "$(col "$t" 4)" "$(col "$t" 5)" ;; *) - die "unknown command $CMD (expected: all | combo | per-task)" + die "unknown command $1 (expected: all | combo | per-task)" ;; esac From 3d63d58dc73c4d43a359df7343de75929b9665a9 Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Sun, 9 Aug 2026 15:17:26 +0300 Subject: [PATCH 4/5] gate: fleet-hash graph must equal plain bake --print The script reads the bake files directly (its static tests must run without the docker CLI), so pin that reading to bake's own evaluation: a build-lane test compares `fleet-hash.sh graph` (new subcommand, target|context|deps) against one `docker buildx bake --print` over the full -f set, target for target. The awk parse can never silently drift from the canonical consumer of the graph data (RULES.md principle 15). Verified locally: all 153 targets match. Signed-off-by: Elron Bandel --- Cargo.lock | 1 + containers/scripts/fleet-hash.sh | 6 ++ tests/build/Cargo.toml | 1 + tests/build/test.rs | 99 ++++++++++++++++++++++++++++++++ 4 files changed, 107 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 8f546f72..85624026 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -672,6 +672,7 @@ version = "0.0.0" dependencies = [ "eval-containers", "eval-containers-test-support", + "serde_json", "testcontainers", "tokio", ] diff --git a/containers/scripts/fleet-hash.sh b/containers/scripts/fleet-hash.sh index 843fc727..9329ad00 100755 --- a/containers/scripts/fleet-hash.sh +++ b/containers/scripts/fleet-hash.sh @@ -15,6 +15,9 @@ # fleet-hash.sh # every static bake target # fleet-hash.sh combo # eval + eval-standalone rows # fleet-hash.sh per-task # one per-task image row +# fleet-hash.sh graph # the parsed graph: target|context|deps +# # (gated against `bake --print` by +# # tests/build fleet_hash_graph_matches_bake_print) # # Output (TSV): target hash context-hash bases-hash externals # Env: REF (default HEAD), REPO_ROOT (default: the repo containing this script) @@ -136,6 +139,9 @@ case "${1:-all}" in all) cat "$M/all.tsv" ;; +graph) + cat "$M/graph" + ;; combo) [ $# -eq 3 ] || die "usage: fleet-hash.sh combo " b=$(target_for_dir "containers/benchmarks/$2") diff --git a/tests/build/Cargo.toml b/tests/build/Cargo.toml index 155ec814..37adfcbc 100644 --- a/tests/build/Cargo.toml +++ b/tests/build/Cargo.toml @@ -13,6 +13,7 @@ autotests = false # run live beside it as container-structure-test (`structure/`). [dependencies] eval-containers = { path = "../../cli" } +serde_json = "1" test-support = { package = "eval-containers-test-support", path = "../support" } [dev-dependencies] diff --git a/tests/build/test.rs b/tests/build/test.rs index cf7780f9..a88cd690 100644 --- a/tests/build/test.rs +++ b/tests/build/test.rs @@ -1038,3 +1038,102 @@ fn eval_local_resolves_from_full_graph() { ); } } + +/// The fleet-hash graph must equal plain bake's own evaluation. fleet-hash.sh +/// reads the bake files directly (its static-stage tests must run without the +/// docker CLI); this gate pins that reading to `bake --print` — the canonical +/// consumer of the graph data (RULES.md principle 15) — so the two can never +/// silently drift. +#[test] +#[ignore = "needs the docker buildx CLI; runs in the build lane like the other bake checks"] +fn fleet_hash_graph_matches_bake_print() { + let root = test_support::repo_root(); + let out = Command::new("bash") + .arg(root.join("containers/scripts/fleet-hash.sh")) + .arg("graph") + .output() + .expect("run fleet-hash.sh graph"); + assert!( + out.status.success(), + "fleet-hash graph failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + // target -> (context, sorted deps) + let mut script: HashMap)> = HashMap::new(); + for line in String::from_utf8(out.stdout).expect("utf8").lines() { + let f: Vec<&str> = line.split('|').collect(); + assert_eq!(f.len(), 3, "malformed graph row: {line}"); + let mut deps: Vec = f[2].split_whitespace().map(str::to_string).collect(); + deps.sort(); + script.insert(f[0].to_string(), (f[1].to_string(), deps)); + } + assert!(!script.is_empty()); + + let files = all_bake_files(); + let refs: Vec<&str> = files.iter().map(String::as_str).collect(); + let mut targets: Vec<&str> = script.keys().map(String::as_str).collect(); + targets.sort(); + // The combination file rides along in all_bake_files; its variables need + // values to evaluate (any values — those targets are not compared). + let env = [ + ("EVAL_BENCHMARK", "aime"), + ("EVAL_AGENT", "codex"), + ("BENCHMARK_IMAGE", "x"), + ("AGENT_IMAGE", "y"), + ]; + let mut args = vec!["buildx".to_string(), "bake".to_string()]; + for f in &refs { + args.push("-f".into()); + args.push((*f).into()); + } + args.push("--print".into()); + args.extend(targets.iter().map(|t| t.to_string())); + let out = Command::new("docker") + .current_dir(&root) + .args(&args) + .envs(env) + .output() + .expect("run docker buildx bake --print"); + assert!( + out.status.success(), + "bake --print rejected the fleet-hash target list:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + let json: serde_json::Value = serde_json::from_slice(&out.stdout).expect("bake --print JSON"); + let baked = json["target"].as_object().expect("target map"); + + let mut failures: Vec = Vec::new(); + for (t, (ctx, deps)) in &script { + let Some(b) = baked.get(t) else { + failures.push(format!("{t}: in fleet-hash graph but not in bake --print")); + continue; + }; + let bake_ctx = b["context"].as_str().unwrap_or(""); + if bake_ctx != ctx { + failures.push(format!( + "{t}: context {ctx} (fleet-hash) vs {bake_ctx} (bake)" + )); + } + let mut bake_deps: Vec = b["contexts"] + .as_object() + .map(|m| { + m.values() + .filter_map(|v| v.as_str()) + .filter_map(|v| v.strip_prefix("target:")) + .map(str::to_string) + .collect() + }) + .unwrap_or_default(); + bake_deps.sort(); + if &bake_deps != deps { + failures.push(format!( + "{t}: deps {deps:?} (fleet-hash) vs {bake_deps:?} (bake)" + )); + } + } + assert!( + failures.is_empty(), + "fleet-hash graph drifts from bake --print:\n{}", + failures.join("\n") + ); +} From 7530f6296dab2cb52acecda0243a42204f0cd277 Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Sun, 9 Aug 2026 15:47:11 +0300 Subject: [PATCH 5/5] =?UTF-8?q?fix:=20audit=20wave=20=E2=80=94=20REF=20pur?= =?UTF-8?q?ity,=20combo=20runner/=20closure,=20heredoc=20externals,=20ulim?= =?UTF-8?q?it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four adversarial audits on the PR surfaced real correctness bugs; all reproduced, fixed, and pinned by tests: - Combo closure was blind to containers/core/runner/ (and entrypoint/), which the combination Dockerfiles COPY from: a framework-launcher edit changed every combo image and moved zero hashes — the exact false-fresh rule 14 forbids. Reproduced on real commit b072ff1d. Both trees now join the combo context inputs. - The graph and Dockerfiles were read from the worktree while trees came from REF, so REF= emitted a hybrid and uncommitted edits leaked in. containers/ is now materialized from REF via git archive: the output is a pure function of the commit (and uncommitted state is invisible rather than fatal). - 15 phantom externals: python 'from X import Y' in heredoc RUN bodies parsed as FROM instructions. Only an unindented uppercase FROM outside a backslash continuation counts now; ARG defaults are expanded so externals are digest-resolvable (${…} left = per-build by design). - The closure splitter held 288 files open — unrunnable at stock macOS ulimit 256. Files close on change (input is sorted). - Fail-loud hardening: second target block in one bake file, missing context line, comment-quoted target: refs, unparsable sums lines, empty/whitespace per-task ids, graph missing from the usage error. - Combo parents derived from combination.docker-bake.hcl's *_IMAGE defaults instead of hardcoded names. Tests upgraded to kill every surviving mutant the audit found: depth-2 diamond fixture (transitivity is now asserted — 91/153 real targets are depth-2), externals shape fixture (AS/platform/scratch/ARG/heredoc), combo agent-axis + standalone sensitivity, per-task id sensitivity, error-path bad examples, real-repo byte-determinism, Fixture Drop cleanup + GIT_* env isolation. The bake --print gate moves from the never-run build lane to tests/static/fleet-hash.sweep.sh, wired into the per-PR static-composition job, and is now bidirectional: bake must also know no target fleet-hash missed. tests/build reverted to main. Signed-off-by: Elron Bandel --- .github/workflows/test.yml | 4 + Cargo.lock | 1 - containers/scripts/fleet-hash.sh | 149 +++++++---- tests/build/Cargo.toml | 1 - tests/build/test.rs | 99 -------- tests/static/fleet-hash.sweep.sh | 70 ++++++ tests/static/input_hash.rs | 410 +++++++++++++++++++++++-------- 7 files changed, 488 insertions(+), 246 deletions(-) create mode 100755 tests/static/fleet-hash.sweep.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bbb213ca..57026df7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -148,6 +148,10 @@ jobs: if: ${{ !cancelled() }} run: tests/static/model-paths.sweep.sh + - name: fleet-hash graph == bake --print (both directions) + if: ${{ !cancelled() }} + run: tests/static/fleet-hash.sweep.sh + # Per-PR gate: cheapest full-stack fixture (bigcodebench + zerostack) on the real # stack (otelcol → bifrost → runner), so a gateway/otelcol regression fails the PR. replay-e2e: diff --git a/Cargo.lock b/Cargo.lock index 85624026..8f546f72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -672,7 +672,6 @@ version = "0.0.0" dependencies = [ "eval-containers", "eval-containers-test-support", - "serde_json", "testcontainers", "tokio", ] diff --git a/containers/scripts/fleet-hash.sh b/containers/scripts/fleet-hash.sh index 9329ad00..efb32936 100755 --- a/containers/scripts/fleet-hash.sh +++ b/containers/scripts/fleet-hash.sh @@ -4,20 +4,24 @@ # # hash(target) = sha256 of the sorted git tree hashes of the target's build # context and every transitive in-repo base context — a pure function of the -# committed tree at REF, read off the bake graph (principle 15.d keeps each -# target's `contexts` aligned with its Dockerfile's FROMs). A flat set is -# sensitivity-equivalent to a Merkle chain here: wiring changes edit bake -# files, which live inside a hashed context. External FROMs are emitted as -# unresolved refs; digest resolution needs the network and happens at release -# time (rule 11), keeping this script offline. +# committed tree at REF (the containers/ tree is materialized from REF via +# `git archive`, so worktree state is invisible), read off the bake graph +# (principle 15.d keeps each target's `contexts` aligned with its Dockerfile's +# FROMs). A flat set is sensitivity-equivalent to a Merkle chain here: wiring +# changes edit bake files, which live inside a hashed context. External FROMs +# are emitted with same-Dockerfile ARG defaults expanded; refs that still +# carry `${…}` are per-build by design. Digest resolution needs the network +# and happens at release time (rule 11), keeping this script offline. # # Usage: # fleet-hash.sh # every static bake target # fleet-hash.sh combo # eval + eval-standalone rows # fleet-hash.sh per-task # one per-task image row -# fleet-hash.sh graph # the parsed graph: target|context|deps -# # (gated against `bake --print` by -# # tests/build fleet_hash_graph_matches_bake_print) +# fleet-hash.sh graph # target|context|deps — the context +# # column is also the registry ref +# # path (minus containers/); gated +# # against `bake --print` by +# # tests/static/fleet-hash.sweep.sh # # Output (TSV): target hash context-hash bases-hash externals # Env: REF (default HEAD), REPO_ROOT (default: the repo containing this script) @@ -32,39 +36,51 @@ die() { echo "fleet-hash: $*" >&2; exit 2; } sha() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$@"; else shasum -a 256 "$@"; fi; } hash_of() { sha < "$1" | cut -d' ' -f1; } row() { printf '%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "$4" "$5"; } -M=$(mktemp -d) && trap 'rm -rf "$M"' EXIT && mkdir "$M/full" "$M/bases" +M=$(mktemp -d) && trap 'rm -rf "$M"' EXIT && mkdir "$M/full" "$M/bases" "$M/src" + +# ── materialize containers/ at REF: all parsing below reads this tree ─────── +git archive "$REF" containers 2>/dev/null | tar -x -C "$M/src" \ + || die "cannot read containers/ at $REF" +S="$M/src" # ── graph: one awk over every per-artifact bake file → target|context|deps ── # (one target per file, principle 15.a; the parameterized combination file # sits directly in containers/core/, outside the subdir glob) -FILES=(containers/core/*/docker-bake.hcl containers/gateways/*/docker-bake.hcl - containers/agents/*/docker-bake.hcl containers/benchmarks/*/docker-bake.hcl - containers/models/*/docker-bake.hcl) -[ "${#FILES[@]}" -gt 0 ] || die "no bake files under $REPO_ROOT/containers" +FILES=("$S"/containers/core/*/docker-bake.hcl "$S"/containers/gateways/*/docker-bake.hcl + "$S"/containers/agents/*/docker-bake.hcl "$S"/containers/benchmarks/*/docker-bake.hcl + "$S"/containers/models/*/docker-bake.hcl) +[ "${#FILES[@]}" -gt 0 ] || die "no bake files under containers/ at $REF" awk ' FNR==1 { tgt="" } - /^target "/ && tgt=="" { + /^target "/ { + if (tgt != "") { print "fleet-hash: " FILENAME " declares a second target — one per file (principle 15.a)" > "/dev/stderr"; exit 2 } split($0, q, "\""); tgt=q[2] if (tgt in seen) { print "fleet-hash: duplicate target " tgt > "/dev/stderr"; exit 2 } seen[tgt]=1; ctx[tgt]=""; deps[tgt]="" } - $1=="context" && $2=="=" && ctx[tgt]=="" { split($0, q, "\""); ctx[tgt]=q[2] } - { - s=$0 - while (match(s, /"target:[A-Za-z0-9_-]+"/)) { + tgt != "" && $1=="context" && $2=="=" && ctx[tgt]=="" { split($0, q, "\""); ctx[tgt]=q[2] } + tgt != "" { + s=$0; sub(/#.*/, "", s) + while (match(s, /"target:[^"]+"/)) { d=substr(s, RSTART+8, RLENGTH-9) if (index(" " deps[tgt] " ", " " d " ")==0) deps[tgt]=deps[tgt] d " " s=substr(s, RSTART+RLENGTH) } } - END { for (t in ctx) print t "|" ctx[t] "|" deps[t] } + END { + for (t in ctx) { + if (ctx[t]=="") { print "fleet-hash: target " t " has no context line" > "/dev/stderr"; exit 2 } + print t "|" ctx[t] "|" deps[t] + } + } ' "${FILES[@]}" | LC_ALL=C sort > "$M/graph" # ── tree hashes: one git call over every context, paired by row order ─────── -# shellcheck disable=SC2046 # context paths never contain spaces -git rev-parse $(awk -F'|' -v r="$REF" '{print r ":" $2}' "$M/graph") > "$M/hashes" 2>/dev/null || { +PATHS=() +while IFS='|' read -r t ctx _; do PATHS+=("$REF:$ctx"); done < "$M/graph" +git rev-parse "${PATHS[@]}" > "$M/hashes" 2>/dev/null || { while IFS='|' read -r t ctx _; do - git rev-parse "$REF:$ctx" >/dev/null 2>&1 || die "context $ctx of $t is not in $REF (uncommitted?)" + git rev-parse "$REF:$ctx" >/dev/null 2>&1 || die "context $ctx of $t is not in $REF" done < "$M/graph" die "git rev-parse failed" } @@ -87,26 +103,41 @@ awk -F'|' ' for (j=1; j<=m; j++) if (p[j]!="") walk(root, p[j], 0) } ' "$M/graph" "$M/trees" | LC_ALL=C sort -u \ - | awk -F'|' -v m="$M" '{ f = m "/" ($1=="F" ? "full" : "bases") "/" $2; print $3 >> f }' + | awk -F'|' -v m="$M" '{ + f = m "/" ($1=="F" ? "full" : "bases") "/" $2 + if (f != prev) { if (prev != "") close(prev); prev = f } + print $3 >> f + }' # ── externals: one awk over every context Dockerfile → dir|image ──────────── -# A FROM on a continuation line (trailing backslash above) is inside a RUN — -# e.g. SQL `FROM 'hf://…'` in duckdb heredocs — never a Dockerfile instruction. +# Only an unindented uppercase FROM outside a backslash continuation is an +# instruction — SQL `FROM` fragments and Python `from … import` in heredoc +# RUN bodies are neither. `${VAR}` is expanded from same-file ARG defaults; +# a ref still carrying `${…}` is per-build by design (e.g. per-task bases). DFS=() while IFS='|' read -r t ctx _; do - [ -f "$ctx/Dockerfile" ] || die "$ctx/Dockerfile missing (target $t)" - DFS+=("$ctx/Dockerfile") + [ -f "$S/$ctx/Dockerfile" ] || die "$ctx/Dockerfile missing at $REF (target $t)" + DFS+=("$S/$ctx/Dockerfile") done < "$M/graph" -awk ' - FNR==1 { delete alias; cont=0 } - !cont && toupper($1)=="FROM" { +awk -v strip="$S/" ' + FNR==1 { delete alias; delete arg; cont=0 } + /^ARG [A-Za-z_]+=/ { eq=index($2,"="); arg[substr($2,1,eq-1)]=substr($2,eq+1) } + !cont && /^FROM[ \t]/ { img=$2; if (img ~ /^--platform/) img=$3 - for (i=1; i<=NF; i++) if (toupper($i)=="AS") alias[$(i+1)]=1 - if (!(img in alias) && img!="scratch" && index(img,"${REGISTRY}")==0) { - d=FILENAME; sub(/\/Dockerfile$/, "", d); print d "|" img + if (index(img, "${REGISTRY}") == 0) { + while (match(img, /\$\{[A-Za-z_]+\}/)) { + v=substr(img, RSTART+2, RLENGTH-3) + if (!(v in arg)) break + img = substr(img, 1, RSTART-1) arg[v] substr(img, RSTART+RLENGTH) + } + if (!(img in alias) && img != "scratch") { + d=substr(FILENAME, length(strip)+1); sub(/\/Dockerfile$/, "", d) + print d "|" img + } } + for (i=1; i<=NF; i++) if ($i=="AS") alias[$(i+1)]=1 } - { cont = ($0 ~ /\\[[:space:]]*$/) } + { cont = ($0 ~ /\\[ \t]*$/) } ' "${DFS[@]}" | LC_ALL=C sort -u > "$M/ext" # ── one sha pass over every closure file, then a single join → the TSV ────── @@ -118,7 +149,10 @@ awk ' FILENAME ~ /ext$/ { ext[$1] = ($1 in ext) ? ext[$1] "," $2 : $2; next } { split($0, a, / +/) - if (split(a[2], b, "/") == 2) { if (b[1]=="full") full[b[2]]=a[1]; else bases[b[2]]=a[1] } + if (split(a[2], b, "/") != 2) { print "fleet-hash: unparsable sums line: " $0 > "/dev/stderr"; exit 2 } + if (b[1]=="full") full[b[2]]=a[1] + else if (b[1]=="bases") bases[b[2]]=a[1] + else { print "fleet-hash: unparsable sums line: " $0 > "/dev/stderr"; exit 2 } } END { for (i=1; i<=n; i++) { @@ -130,10 +164,23 @@ awk ' col() { awk -F'\t' -v t="$1" -v c="$2" '$1==t { print $c }' "$M/all.tsv"; } target_for_dir() { - awk -F'|' -v d="$1" '$2==d { print $1; found=1 } END { exit !found }' "$M/graph" \ - || die "no bake target with context $1" + local t + t=$(awk -F'|' -v d="$1" '$2==d { print $1 }' "$M/graph") + [ -n "$t" ] || die "no bake target with context $1" + [ "$(printf '%s\n' "$t" | wc -l)" -eq 1 ] || die "multiple targets with context $1" + printf '%s' "$t" } blobs() { git rev-parse "$@" 2>/dev/null || die "blob not in $REF"; } +# Combo parents come from combination.docker-bake.hcl's *_IMAGE defaults, so a +# changed default re-points the closure at the new target automatically. +parent_target() { + local p + # shellcheck disable=SC2016 # the ${REGISTRY}/${TAG} literals are the match + p=$(grep "\"$1\"" "$S/containers/core/combination.docker-bake.hcl" \ + | sed -n 's|.*"${REGISTRY}/\(.*\):${TAG}".*|\1|p') + [ -n "$p" ] || die "cannot derive $1 from combination.docker-bake.hcl" + target_for_dir "containers/$p" +} case "${1:-all}" in all) @@ -143,28 +190,32 @@ graph) cat "$M/graph" ;; combo) - [ $# -eq 3 ] || die "usage: fleet-hash.sh combo " + { [ $# -eq 3 ] && [ -n "$2" ] && [ -n "$3" ]; } || die "usage: fleet-hash.sh combo " b=$(target_for_dir "containers/benchmarks/$2") a=$(target_for_dir "containers/agents/$3") - # The combination context is all of containers/core (over-broad); the real - # inputs are the combination files' blobs plus the parents' closures. The - # parent list mirrors combination.docker-bake.hcl's variable defaults — - # changing that list edits a hashed blob, so every combo hash moves with it. + gosu=$(parent_target GOSU_IMAGE) + # The combination Dockerfiles COPY from runner/ and entrypoint/ inside the + # containers/core context, so those trees are combo inputs alongside the + # Dockerfile + bake-file blobs and the parents' closures. blobs "$REF:containers/core/combination.Dockerfile" \ - "$REF:containers/core/combination.docker-bake.hcl" | LC_ALL=C sort > "$M/eval.ctx" - LC_ALL=C sort -u "$M/full/$b" "$M/full/$a" "$M/full/gosu" > "$M/eval.bases" + "$REF:containers/core/combination.docker-bake.hcl" \ + "$REF:containers/core/runner" "$REF:containers/core/entrypoint" \ + | LC_ALL=C sort > "$M/eval.ctx" + LC_ALL=C sort -u "$M/full/$b" "$M/full/$a" "$M/full/$gosu" > "$M/eval.bases" LC_ALL=C sort -u "$M/eval.ctx" "$M/eval.bases" > "$M/eval.full" row "evals/$2--$3" "$(hash_of "$M/eval.full")" "$(hash_of "$M/eval.ctx")" \ "$(hash_of "$M/eval.bases")" "-" blobs "$REF:containers/core/standalone.Dockerfile" > "$M/sa.ctx" - LC_ALL=C sort -u "$M/eval.full" "$M/full/otel" "$M/full/process-compose" \ - "$M/full/model-bifrost" > "$M/sa.bases" + LC_ALL=C sort -u "$M/eval.full" "$M/full/$(parent_target OTEL_IMAGE)" \ + "$M/full/$(parent_target PROCESS_COMPOSE_IMAGE)" \ + "$M/full/$(parent_target MODEL_IMAGE)" > "$M/sa.bases" LC_ALL=C sort -u "$M/sa.ctx" "$M/sa.bases" > "$M/sa.full" row "evals/$2--$3-standalone" "$(hash_of "$M/sa.full")" "$(hash_of "$M/sa.ctx")" \ "$(hash_of "$M/sa.bases")" "-" ;; per-task) - [ $# -eq 3 ] || die "usage: fleet-hash.sh per-task " + { [ $# -eq 3 ] && [ -n "$2" ] && [ -n "$3" ]; } || die "usage: fleet-hash.sh per-task " + case "$3" in *[[:space:]]*) die "task id must not contain whitespace" ;; esac [ -z "${SKILLS_BENCH_REF:-}" ] || die "SKILLS_BENCH_REF is set — an out-of-tree ref override defeats input hashing; pin the ref in the benchmark dir" t=$(target_for_dir "containers/benchmarks/$2") h=$(col "$t" 2) @@ -172,6 +223,6 @@ per-task) "$(col "$t" 3)" "$(col "$t" 4)" "$(col "$t" 5)" ;; *) - die "unknown command $1 (expected: all | combo | per-task)" + die "unknown command $1 (expected: all | combo | per-task | graph)" ;; esac diff --git a/tests/build/Cargo.toml b/tests/build/Cargo.toml index 37adfcbc..155ec814 100644 --- a/tests/build/Cargo.toml +++ b/tests/build/Cargo.toml @@ -13,7 +13,6 @@ autotests = false # run live beside it as container-structure-test (`structure/`). [dependencies] eval-containers = { path = "../../cli" } -serde_json = "1" test-support = { package = "eval-containers-test-support", path = "../support" } [dev-dependencies] diff --git a/tests/build/test.rs b/tests/build/test.rs index a88cd690..cf7780f9 100644 --- a/tests/build/test.rs +++ b/tests/build/test.rs @@ -1038,102 +1038,3 @@ fn eval_local_resolves_from_full_graph() { ); } } - -/// The fleet-hash graph must equal plain bake's own evaluation. fleet-hash.sh -/// reads the bake files directly (its static-stage tests must run without the -/// docker CLI); this gate pins that reading to `bake --print` — the canonical -/// consumer of the graph data (RULES.md principle 15) — so the two can never -/// silently drift. -#[test] -#[ignore = "needs the docker buildx CLI; runs in the build lane like the other bake checks"] -fn fleet_hash_graph_matches_bake_print() { - let root = test_support::repo_root(); - let out = Command::new("bash") - .arg(root.join("containers/scripts/fleet-hash.sh")) - .arg("graph") - .output() - .expect("run fleet-hash.sh graph"); - assert!( - out.status.success(), - "fleet-hash graph failed:\n{}", - String::from_utf8_lossy(&out.stderr) - ); - // target -> (context, sorted deps) - let mut script: HashMap)> = HashMap::new(); - for line in String::from_utf8(out.stdout).expect("utf8").lines() { - let f: Vec<&str> = line.split('|').collect(); - assert_eq!(f.len(), 3, "malformed graph row: {line}"); - let mut deps: Vec = f[2].split_whitespace().map(str::to_string).collect(); - deps.sort(); - script.insert(f[0].to_string(), (f[1].to_string(), deps)); - } - assert!(!script.is_empty()); - - let files = all_bake_files(); - let refs: Vec<&str> = files.iter().map(String::as_str).collect(); - let mut targets: Vec<&str> = script.keys().map(String::as_str).collect(); - targets.sort(); - // The combination file rides along in all_bake_files; its variables need - // values to evaluate (any values — those targets are not compared). - let env = [ - ("EVAL_BENCHMARK", "aime"), - ("EVAL_AGENT", "codex"), - ("BENCHMARK_IMAGE", "x"), - ("AGENT_IMAGE", "y"), - ]; - let mut args = vec!["buildx".to_string(), "bake".to_string()]; - for f in &refs { - args.push("-f".into()); - args.push((*f).into()); - } - args.push("--print".into()); - args.extend(targets.iter().map(|t| t.to_string())); - let out = Command::new("docker") - .current_dir(&root) - .args(&args) - .envs(env) - .output() - .expect("run docker buildx bake --print"); - assert!( - out.status.success(), - "bake --print rejected the fleet-hash target list:\n{}", - String::from_utf8_lossy(&out.stderr) - ); - let json: serde_json::Value = serde_json::from_slice(&out.stdout).expect("bake --print JSON"); - let baked = json["target"].as_object().expect("target map"); - - let mut failures: Vec = Vec::new(); - for (t, (ctx, deps)) in &script { - let Some(b) = baked.get(t) else { - failures.push(format!("{t}: in fleet-hash graph but not in bake --print")); - continue; - }; - let bake_ctx = b["context"].as_str().unwrap_or(""); - if bake_ctx != ctx { - failures.push(format!( - "{t}: context {ctx} (fleet-hash) vs {bake_ctx} (bake)" - )); - } - let mut bake_deps: Vec = b["contexts"] - .as_object() - .map(|m| { - m.values() - .filter_map(|v| v.as_str()) - .filter_map(|v| v.strip_prefix("target:")) - .map(str::to_string) - .collect() - }) - .unwrap_or_default(); - bake_deps.sort(); - if &bake_deps != deps { - failures.push(format!( - "{t}: deps {deps:?} (fleet-hash) vs {bake_deps:?} (bake)" - )); - } - } - assert!( - failures.is_empty(), - "fleet-hash graph drifts from bake --print:\n{}", - failures.join("\n") - ); -} diff --git a/tests/static/fleet-hash.sweep.sh b/tests/static/fleet-hash.sweep.sh new file mode 100755 index 00000000..042bd261 --- /dev/null +++ b/tests/static/fleet-hash.sweep.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# tests/static/fleet-hash.sweep.sh — pin fleet-hash's bake-graph reading to +# plain bake's own evaluation (the wiring gate for the build-input hash, +# alongside compose.config.sweep.sh and helm.sweep.sh). +# +# fleet-hash.sh parses the per-artifact bake files directly so its Rust tests +# run without the docker CLI. This sweep is the independent oracle: one +# `docker buildx bake --print` over the root + every per-artifact file +# (the combination file is parameterized and excluded on both sides), compared +# BIDIRECTIONALLY — every fleet-hash target must match bake's context and +# target: deps exactly, and bake must know no target fleet-hash missed (a +# dropped target is a silently unhashed, silently carried-forward image). +# `--print` is a client-side HCL evaluation: no daemon, no images, no creds. +# Fail loud; offline. +set -uo pipefail +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd) || exit 2 + +command -v docker >/dev/null || { echo "docker not found — required for the bake --print gate"; exit 1; } +command -v jq >/dev/null || { echo "jq not found — required for the bake --print gate"; exit 1; } +docker buildx version >/dev/null 2>&1 || { echo "docker buildx plugin not found"; exit 1; } + +shopt -s nullglob +cd "$ROOT" || exit 2 + +graph=$(bash containers/scripts/fleet-hash.sh graph) || { echo "fleet-hash graph failed"; exit 1; } + +args=(-f containers/docker-bake.hcl) +for f in containers/core/*/docker-bake.hcl containers/gateways/*/docker-bake.hcl \ + containers/agents/*/docker-bake.hcl containers/benchmarks/*/docker-bake.hcl \ + containers/models/*/docker-bake.hcl; do args+=(-f "$f"); done + +err=$(mktemp) +trap 'rm -f "$err"' EXIT +# shellcheck disable=SC2046 # target names never contain whitespace +print=$(docker buildx bake "${args[@]}" --print $(cut -d'|' -f1 <<<"$graph") 2>"$err") \ + || { echo "bake --print rejected the fleet-hash target list:"; cat "$err"; exit 1; } + +fails=0 + +# Reverse direction: bake's evaluated target set == fleet-hash's. +if ! diff <(cut -d'|' -f1 <<<"$graph" | LC_ALL=C sort) \ + <(jq -r '.target | keys[]' <<<"$print" | LC_ALL=C sort); then + echo "FAIL: target sets differ (fleet-hash vs bake --print)" + fails=$((fails + 1)) +fi + +# Forward direction: context and target: deps agree, target by target. +while IFS='|' read -r t ctx deps; do + bctx=$(jq -r --arg t "$t" '.target[$t].context // ""' <<<"$print") + if [ "$bctx" != "$ctx" ]; then + echo "FAIL: $t context — fleet-hash '$ctx' vs bake '$bctx'" + fails=$((fails + 1)) + fi + bdeps=$(jq -r --arg t "$t" \ + '[.target[$t].contexts // {} | .[] | select(startswith("target:")) | ltrimstr("target:")] | sort | join(" ")' \ + <<<"$print") + # shellcheck disable=SC2086 # deps is a space-separated list, split intended + sdeps=$(printf '%s\n' $deps | LC_ALL=C sort | paste -sd' ' - | sed 's/^ *//') + if [ "$bdeps" != "$sdeps" ]; then + echo "FAIL: $t deps — fleet-hash '$sdeps' vs bake '$bdeps'" + fails=$((fails + 1)) + fi +done <<<"$graph" + +n=$(wc -l <<<"$graph" | tr -d ' ') +if [ "$fails" -gt 0 ]; then + echo "fleet-hash graph drifts from bake --print: $fails failure(s) across $n targets" + exit 1 +fi +echo "OK: fleet-hash graph == bake --print for all $n targets (both directions)" diff --git a/tests/static/input_hash.rs b/tests/static/input_hash.rs index b7f85199..c99ad4ee 100644 --- a/tests/static/input_hash.rs +++ b/tests/static/input_hash.rs @@ -2,30 +2,44 @@ //! carried-forward contract (delivery/RULES.md rules 11–14). //! //! `containers/scripts/fleet-hash.sh` must be a pure function of the committed -//! tree: deterministic, sensitive to any context change, and cascading through -//! the bake graph so a base edit dirties every dependent leaf. These are the -//! properties the selective-release machinery will trust, so they are proven -//! here on a synthetic git fixture (where we can commit mutations) and the -//! script is exercised over the real repo for coverage. Offline, daemon-free -//! (tests/static/RULES.md rule 1): only `git`, `bash`, and awk/sed run. +//! tree at REF: deterministic, sensitive to any context change, cascading +//! through the *transitive* bake graph (a base-of-base edit dirties the leaf), +//! and blind to uncommitted worktree state. These are the properties the +//! selective-release machinery will trust, so they are proven on synthetic git +//! fixtures (where mutations are committed like the real tree) and the script +//! is exercised over the real repo. Offline, daemon-free (tests/static/RULES.md +//! rule 1): only `git`, `bash`, and awk/sed run. The graph parse itself is +//! separately gated against `docker buildx bake --print` by +//! `tests/static/fleet-hash.sweep.sh` (the static-composition CI job). use std::collections::HashMap; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Output}; +use std::time::{SystemTime, UNIX_EPOCH}; use test_support::repo_root; fn script() -> PathBuf { repo_root().join("containers/scripts/fleet-hash.sh") } -fn fleet_hash_at(repo: &Path, git_ref: &str, args: &[&str]) -> String { - let out = Command::new("bash") - .arg(script()) +fn run(repo: &Path, git_ref: &str, args: &[&str], env: &[(&str, &str)]) -> Output { + let mut cmd = Command::new("bash"); + cmd.arg(script()) .args(args) .env("REPO_ROOT", repo) .env("REF", git_ref) - .output() - .expect("run fleet-hash.sh"); + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env_remove("GIT_INDEX_FILE") + .env_remove("SKILLS_BENCH_REF"); + for (k, v) in env { + cmd.env(k, v); + } + cmd.output().expect("run fleet-hash.sh") +} + +fn fleet_hash_at(repo: &Path, git_ref: &str, args: &[&str]) -> String { + let out = run(repo, git_ref, args, &[]); assert!( out.status.success(), "fleet-hash {args:?} failed:\n{}", @@ -38,6 +52,20 @@ fn fleet_hash(repo: &Path, args: &[&str]) -> String { fleet_hash_at(repo, "HEAD", args) } +/// Expect a loud failure: non-zero exit and the given stderr fragment. +fn expect_die(repo: &Path, args: &[&str], env: &[(&str, &str)], msg: &str) { + let out = run(repo, "HEAD", args, env); + assert!( + !out.status.success(), + "fleet-hash {args:?} unexpectedly succeeded" + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains(msg), + "stderr for {args:?} lacks {msg:?}:\n{stderr}" + ); +} + /// target -> (hash, context-hash, bases-hash, externals) fn rows(output: &str) -> HashMap { output @@ -62,7 +90,66 @@ fn is_sha256(s: &str) -> bool { s.len() == 64 && s.chars().all(|c| c.is_ascii_hexdigit()) } -// ── synthetic fixture ─────────────────────────────────────────────────────── +fn is_tree_hash(s: &str) -> bool { + s.len() == 40 && s.chars().all(|c| c.is_ascii_hexdigit()) +} + +// ── synthetic fixtures ────────────────────────────────────────────────────── + +/// A throwaway git repo; the directory is removed on drop even when an +/// assertion fails mid-test. +struct Fixture(PathBuf); + +impl Drop for Fixture { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +impl Fixture { + fn new(name: &str) -> Self { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = + std::env::temp_dir().join(format!("fleet-hash-{name}-{}-{nanos}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + git(&dir, &["init", "-q"]); + Fixture(dir) + } + + fn write(&self, rel: &str, content: &str) { + let p = self.0.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, content).unwrap(); + } + + fn artifact(&self, kind: &str, name: &str, target: &str, deps: &[&str], dockerfile: &str) { + let dir = format!("containers/{kind}/{name}"); + self.write(&format!("{dir}/Dockerfile"), dockerfile); + let contexts = if deps.is_empty() { + String::new() + } else { + let entries: String = deps + .iter() + .map(|d| format!(" \"${{REGISTRY}}/core/{d}\" = \"target:{d}\"\n")) + .collect(); + format!(" contexts = {{\n{entries} }}\n") + }; + self.write( + &format!("{dir}/docker-bake.hcl"), + &format!( + "target \"{target}\" {{\n context = \"{dir}\"\n{contexts} tags = [\"${{REGISTRY}}/{kind}/{name}:${{TAG}}\"]\n}}\n" + ), + ); + } + + fn commit(&self, msg: &str) { + git(&self.0, &["add", "."]); + git(&self.0, &["commit", "-q", "-m", msg]); + } +} fn git(repo: &Path, args: &[&str]) { let out = Command::new("git") @@ -78,6 +165,9 @@ fn git(repo: &Path, args: &[&str]) { .current_dir(repo) .env("GIT_CONFIG_GLOBAL", "/dev/null") .env("GIT_CONFIG_SYSTEM", "/dev/null") + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env_remove("GIT_INDEX_FILE") .output() .expect("run git"); assert!( @@ -87,109 +177,201 @@ fn git(repo: &Path, args: &[&str]) { ); } -fn write(repo: &Path, rel: &str, content: &str) { - let p = repo.join(rel); - std::fs::create_dir_all(p.parent().unwrap()).unwrap(); - std::fs::write(p, content).unwrap(); -} - -/// A minimal fleet: one core base (external FROM), one leaf depending on it, -/// one independent leaf. Committed to a throwaway git repo so mutations can -/// be committed and hashed like the real tree. -fn fixture(name: &str) -> PathBuf { - let dir = std::env::temp_dir().join(format!("fleet-hash-{name}-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); - write( - &dir, - "containers/core/base-x/Dockerfile", - "FROM alpine:3.20\nRUN echo base\n", - ); - write( - &dir, - "containers/core/base-x/docker-bake.hcl", - "target \"base-x\" {\n context = \"containers/core/base-x\"\n tags = [\"${REGISTRY}/core/base-x:${TAG}\"]\n}\n", +/// Depth-2 chain with a diamond: leaf-a → base-y → base-x, leaf-c → {base-x, +/// base-y}, leaf-b independent. Deep enough that a non-transitive walk is +/// observably wrong (91/153 real targets sit at depth 2). +fn fleet_fixture(name: &str) -> Fixture { + let fx = Fixture::new(name); + fx.artifact( + "core", + "base-x", + "base-x", + &[], + "FROM alpine:3.20\nRUN echo x\n", ); - write( - &dir, - "containers/benchmarks/leaf-a/Dockerfile", - "FROM ${REGISTRY}/core/base-x:latest\nRUN echo a\n", + fx.artifact( + "core", + "base-y", + "base-y", + &["base-x"], + "FROM ${REGISTRY}/core/base-x:latest\nRUN echo y\n", ); - write( - &dir, - "containers/benchmarks/leaf-a/docker-bake.hcl", - "target \"benchmark-leaf-a\" {\n context = \"containers/benchmarks/leaf-a\"\n contexts = {\n \"${REGISTRY}/core/base-x\" = \"target:base-x\"\n }\n tags = [\"${REGISTRY}/benchmarks/leaf-a:${TAG}\"]\n}\n", + fx.artifact( + "benchmarks", + "leaf-a", + "benchmark-leaf-a", + &["base-y"], + "FROM ${REGISTRY}/core/base-y:latest\nRUN echo a\n", ); - write( - &dir, - "containers/benchmarks/leaf-b/Dockerfile", + fx.artifact( + "benchmarks", + "leaf-b", + "benchmark-leaf-b", + &[], "FROM debian:12-slim\nRUN echo b\n", ); - write( - &dir, - "containers/benchmarks/leaf-b/docker-bake.hcl", - "target \"benchmark-leaf-b\" {\n context = \"containers/benchmarks/leaf-b\"\n tags = [\"${REGISTRY}/benchmarks/leaf-b:${TAG}\"]\n}\n", + fx.artifact( + "benchmarks", + "leaf-c", + "benchmark-leaf-c", + &["base-x", "base-y"], + "FROM ${REGISTRY}/core/base-x:latest\nRUN echo c\n", ); - git(&dir, &["init", "-q"]); - git(&dir, &["add", "."]); - git(&dir, &["commit", "-q", "-m", "fixture"]); - dir + fx.commit("fixture"); + fx } -/// Deterministic; a leaf edit changes only that leaf (context component); a -/// base edit cascades to its dependents (bases component) and spares the -/// rest; REF pins the whole computation to a commit. +/// Deterministic; a leaf edit moves only that leaf (context component); a +/// base-of-base edit cascades transitively through the diamond (bases +/// component) and spares independent leaves; REF pins the computation; the +/// uncommitted worktree is invisible. #[test] -fn hash_is_deterministic_source_sensitive_and_cascading() { - let repo = fixture("props"); +fn hash_is_deterministic_transitive_and_pure() { + let fx = fleet_fixture("props"); + let repo = &fx.0; - let first = fleet_hash(&repo, &[]); - assert_eq!(first, fleet_hash(&repo, &[]), "two runs must be identical"); + let first = fleet_hash(repo, &[]); + assert_eq!(first, fleet_hash(repo, &[]), "two runs must be identical"); let v0 = rows(&first); - assert_eq!(v0.len(), 3); + assert_eq!(v0.len(), 5); for (t, (hash, ctxh, basesh, _)) in &v0 { assert!(is_sha256(hash), "{t}: hash not sha256"); - assert_eq!(ctxh.len(), 40, "{t}: context hash not a git tree hash"); + assert!(is_tree_hash(ctxh), "{t}: context hash not a git tree hash"); assert!(is_sha256(basesh), "{t}: bases hash not sha256"); } assert_eq!(v0["base-x"].3, "alpine:3.20", "external FROM must surface"); - assert_eq!( - v0["benchmark-leaf-a"].3, "-", - "in-repo FROM is not external" - ); + assert_eq!(v0["base-y"].3, "-", "in-repo FROM is not external"); assert_eq!(v0["benchmark-leaf-b"].3, "debian:12-slim"); // Leaf edit: only leaf-a moves, and only its context component. - write(&repo, "containers/benchmarks/leaf-a/extra.txt", "changed\n"); - git(&repo, &["add", "."]); - git(&repo, &["commit", "-q", "-m", "edit leaf-a"]); - let v1 = rows(&fleet_hash(&repo, &[])); + fx.write("containers/benchmarks/leaf-a/extra.txt", "changed\n"); + fx.commit("edit leaf-a"); + let v1 = rows(&fleet_hash(repo, &[])); assert_ne!(v1["benchmark-leaf-a"].0, v0["benchmark-leaf-a"].0); assert_ne!(v1["benchmark-leaf-a"].1, v0["benchmark-leaf-a"].1); assert_eq!(v1["benchmark-leaf-a"].2, v0["benchmark-leaf-a"].2); - assert_eq!(v1["base-x"], v0["base-x"]); - assert_eq!(v1["benchmark-leaf-b"], v0["benchmark-leaf-b"]); - - // Base edit: base-x and its dependent leaf-a move; leaf-a via its bases - // component only; independent leaf-b is untouched. - write(&repo, "containers/core/base-x/extra.txt", "changed\n"); - git(&repo, &["add", "."]); - git(&repo, &["commit", "-q", "-m", "edit base-x"]); - let v2 = rows(&fleet_hash(&repo, &[])); + for t in ["base-x", "base-y", "benchmark-leaf-b", "benchmark-leaf-c"] { + assert_eq!(v1[t], v0[t], "{t} must not move on a leaf-a edit"); + } + + // Base-of-base edit: base-x moves, and the cascade reaches base-y, + // leaf-a (TRANSITIVELY — its direct dep is base-y), and diamond leaf-c, + // in every case through the bases component only. leaf-b is untouched. + fx.write("containers/core/base-x/extra.txt", "changed\n"); + fx.commit("edit base-x"); + let v2 = rows(&fleet_hash(repo, &[])); assert_ne!(v2["base-x"].0, v1["base-x"].0); - assert_ne!(v2["benchmark-leaf-a"].0, v1["benchmark-leaf-a"].0); - assert_eq!(v2["benchmark-leaf-a"].1, v1["benchmark-leaf-a"].1); - assert_ne!(v2["benchmark-leaf-a"].2, v1["benchmark-leaf-a"].2); + for t in ["base-y", "benchmark-leaf-a", "benchmark-leaf-c"] { + assert_ne!(v2[t].0, v1[t].0, "{t} must cascade on a base-x edit"); + assert_eq!(v2[t].1, v1[t].1, "{t}: context component must not move"); + assert_ne!(v2[t].2, v1[t].2, "{t}: bases component must move"); + } assert_eq!(v2["benchmark-leaf-b"], v1["benchmark-leaf-b"]); // REF pins the computation: hashing HEAD~1 reproduces the prior state. - assert_eq!(rows(&fleet_hash_at(&repo, "HEAD~1", &[])), v1); + assert_eq!(rows(&fleet_hash_at(repo, "HEAD~1", &[])), v1); + + // Purity: uncommitted edits and uncommitted new artifacts are invisible. + fx.write("containers/core/base-x/Dockerfile", "FROM busybox\n"); + fx.artifact( + "benchmarks", + "leaf-new", + "benchmark-leaf-new", + &[], + "FROM scratch\n", + ); + let v3 = rows(&fleet_hash(repo, &[])); + assert_eq!( + v3, v2, + "uncommitted worktree state must not affect the hash" + ); + assert!(!v3.contains_key("benchmark-leaf-new")); +} + +/// The externals awk is a Dockerfile FROM parser; pin every branch: stage +/// aliases, --platform, scratch, ARG-default expansion, backslash-continued +/// SQL FROM, and heredoc `from … import` bodies. +#[test] +fn externals_parse_every_dockerfile_shape() { + let fx = Fixture::new("externals"); + fx.artifact( + "benchmarks", + "shapes", + "benchmark-shapes", + &[], + concat!( + "ARG GO_VERSION=1.23\n", + "FROM golang:${GO_VERSION} AS build\n", + "FROM --platform=$BUILDPLATFORM alpine:3.20 AS helper\n", + "FROM redis:7 AS redis\n", + "FROM redis\n", + "FROM build\n", + "FROM scratch\n", + "RUN duckdb -c \"COPY (SELECT 1) TO 'x' \\\n", + " FROM 'hf://datasets/fake@~parquet/x.parquet'\"\n", + "RUN python3 <<'PYEOF'\n", + "from difflib import SequenceMatcher\n", + "from collections import Counter\n", + "PYEOF\n", + ), + ); + fx.commit("shapes"); + let ext = &rows(&fleet_hash(&fx.0, &[]))["benchmark-shapes"].3; + assert_eq!( + ext, "alpine:3.20,golang:1.23,redis:7", + "externals must expand ARG defaults, keep aliased first-use images, and \ + ignore stage reuse, scratch, SQL FROM continuations, and heredoc imports" + ); +} + +/// Malformed inputs and misuse must fail loudly (exit 2 + a named cause), +/// never produce a hash — a wrong hash fails open into a stale release. +#[test] +fn error_paths_fail_loud() { + let two = Fixture::new("two-targets"); + two.artifact("core", "ok", "ok", &[], "FROM scratch\n"); + two.write( + "containers/core/ok/docker-bake.hcl", + "target \"ok\" {\n context = \"containers/core/ok\"\n}\ntarget \"sneaky\" {\n context = \"containers/core/ok\"\n}\n", + ); + two.commit("two targets"); + expect_die(&two.0, &[], &[], "declares a second target"); + + let noctx = Fixture::new("no-context"); + noctx.artifact("core", "ok", "ok", &[], "FROM scratch\n"); + noctx.write( + "containers/core/ok/docker-bake.hcl", + "target \"ok\" {\n tags = [\"x\"]\n}\n", + ); + noctx.commit("no context"); + expect_die(&noctx.0, &[], &[], "has no context line"); + + let unknown = Fixture::new("unknown-dep"); + unknown.artifact("core", "ok", "ok", &["ghost"], "FROM scratch\n"); + unknown.commit("unknown dep"); + expect_die(&unknown.0, &[], &[], "depends on unknown target ghost"); - let _ = std::fs::remove_dir_all(&repo); + let nodf = Fixture::new("no-dockerfile"); + nodf.artifact("core", "ok", "ok", &[], "FROM scratch\n"); + std::fs::remove_file(nodf.0.join("containers/core/ok/Dockerfile")).unwrap(); + nodf.commit("no dockerfile"); + expect_die(&nodf.0, &[], &[], "Dockerfile missing"); + + let ok = fleet_fixture("misuse"); + expect_die(&ok.0, &["frobnicate"], &[], "unknown command"); + expect_die(&ok.0, &["per-task", "leaf-a", ""], &[], "usage:"); + expect_die(&ok.0, &["per-task", "leaf-a", "a b"], &[], "whitespace"); + expect_die( + &ok.0, + &["per-task", "leaf-a", "t0"], + &[("SKILLS_BENCH_REF", "x")], + "out-of-tree ref override", + ); } -/// The real repo: every per-artifact bake file yields exactly one row, and -/// the combo/per-task providers produce well-formed hashes. +/// The real repo: every per-artifact bake file yields exactly one row, +/// byte-identically across runs, and the combo/per-task providers are +/// sensitive to each of their inputs. #[test] fn real_repo_hashes_every_target() { let root = repo_root(); @@ -204,24 +386,60 @@ fn real_repo_hashes_every_target() { }) .sum(); - let all = rows(&fleet_hash(&root, &[])); + let raw = fleet_hash(&root, &[]); + assert_eq!( + raw, + fleet_hash(&root, &[]), + "real-repo runs must be byte-identical" + ); + let all = rows(&raw); assert_eq!( all.len(), bake_files, - "one row per per-artifact bake file (duplicates collapse in the map)" + "one row per per-artifact bake file (note: this mirrors the script's \ + own glob; the independent oracle is fleet-hash.sweep.sh vs bake --print)" ); - for (t, (hash, _, _, _)) in &all { + for (t, (hash, ctxh, _, _)) in &all { assert!(is_sha256(hash), "{t}: hash not sha256"); + assert!(is_tree_hash(ctxh), "{t}: context hash not a git tree hash"); } + assert!(all.contains_key("benchmark-aime") && all.contains_key("entrypoint")); - let combo = rows(&fleet_hash(&root, &["combo", "aime", "claude-code"])); - assert_eq!(combo.len(), 2); - assert!(combo.contains_key("evals/aime--claude-code")); - assert!(combo.contains_key("evals/aime--claude-code-standalone")); + // Combo: sensitive to the agent axis, and standalone differs from lean. + let cc = rows(&fleet_hash(&root, &["combo", "aime", "claude-code"])); + let cx = rows(&fleet_hash(&root, &["combo", "aime", "codex"])); + let lean = &cc["evals/aime--claude-code"]; + assert_ne!( + lean.0, cx["evals/aime--codex"].0, + "combo must track the agent" + ); + assert_ne!( + lean.0, cc["evals/aime--claude-code-standalone"].0, + "standalone must differ from the lean combo" + ); - let pt = rows(&fleet_hash( + // Per-task: sensitive to the task id, sharing the benchmark's components. + let t0 = rows(&fleet_hash( &root, &["per-task", "terminal-bench", "task-0"], )); - assert!(is_sha256(&pt["per-task/terminal-bench/task-0"].0)); + let t1 = rows(&fleet_hash( + &root, + &["per-task", "terminal-bench", "task-1"], + )); + let (r0, r1) = ( + &t0["per-task/terminal-bench/task-0"], + &t1["per-task/terminal-bench/task-1"], + ); + assert_ne!(r0.0, r1.0, "per-task hash must track the task id"); + let bench = &all["benchmark-terminal-bench"]; + assert_ne!( + r0.0, bench.0, + "per-task hash must differ from the benchmark's" + ); + assert_eq!( + (&r0.1, &r0.2, &r0.3), + (&bench.1, &bench.2, &bench.3), + "per-task rows share the benchmark's components" + ); }