diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bbb213ca..57026df7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -148,6 +148,10 @@ jobs: if: ${{ !cancelled() }} run: tests/static/model-paths.sweep.sh + - name: fleet-hash graph == bake --print (both directions) + if: ${{ !cancelled() }} + run: tests/static/fleet-hash.sweep.sh + # Per-PR gate: cheapest full-stack fixture (bigcodebench + zerostack) on the real # stack (otelcol → bifrost → runner), so a gateway/otelcol regression fails the PR. replay-e2e: diff --git a/containers/scripts/fleet-hash.sh b/containers/scripts/fleet-hash.sh new file mode 100755 index 00000000..efb32936 --- /dev/null +++ b/containers/scripts/fleet-hash.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# fleet-hash — deterministic build-input hashes for every fleet image +# (delivery/RULES.md rules 11–14). +# +# hash(target) = sha256 of the sorted git tree hashes of the target's build +# context and every transitive in-repo base context — a pure function of the +# committed tree at REF (the containers/ tree is materialized from REF via +# `git archive`, so worktree state is invisible), read off the bake graph +# (principle 15.d keeps each target's `contexts` aligned with its Dockerfile's +# FROMs). A flat set is sensitivity-equivalent to a Merkle chain here: wiring +# changes edit bake files, which live inside a hashed context. External FROMs +# are emitted with same-Dockerfile ARG defaults expanded; refs that still +# carry `${…}` are per-build by design. Digest resolution needs the network +# and happens at release time (rule 11), keeping this script offline. +# +# Usage: +# fleet-hash.sh # every static bake target +# fleet-hash.sh combo # eval + eval-standalone rows +# fleet-hash.sh per-task # one per-task image row +# fleet-hash.sh graph # target|context|deps — the context +# # column is also the registry ref +# # path (minus containers/); gated +# # against `bake --print` by +# # tests/static/fleet-hash.sweep.sh +# +# Output (TSV): target hash context-hash bases-hash externals +# Env: REF (default HEAD), REPO_ROOT (default: the repo containing this script) +set -euo pipefail +shopt -s nullglob + +REF="${REF:-HEAD}" +REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "$0")/../.." && pwd)}" +cd "$REPO_ROOT" + +die() { echo "fleet-hash: $*" >&2; exit 2; } +sha() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$@"; else shasum -a 256 "$@"; fi; } +hash_of() { sha < "$1" | cut -d' ' -f1; } +row() { printf '%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "$4" "$5"; } +M=$(mktemp -d) && trap 'rm -rf "$M"' EXIT && mkdir "$M/full" "$M/bases" "$M/src" + +# ── materialize containers/ at REF: all parsing below reads this tree ─────── +git archive "$REF" containers 2>/dev/null | tar -x -C "$M/src" \ + || die "cannot read containers/ at $REF" +S="$M/src" + +# ── graph: one awk over every per-artifact bake file → target|context|deps ── +# (one target per file, principle 15.a; the parameterized combination file +# sits directly in containers/core/, outside the subdir glob) +FILES=("$S"/containers/core/*/docker-bake.hcl "$S"/containers/gateways/*/docker-bake.hcl + "$S"/containers/agents/*/docker-bake.hcl "$S"/containers/benchmarks/*/docker-bake.hcl + "$S"/containers/models/*/docker-bake.hcl) +[ "${#FILES[@]}" -gt 0 ] || die "no bake files under containers/ at $REF" +awk ' + FNR==1 { tgt="" } + /^target "/ { + if (tgt != "") { print "fleet-hash: " FILENAME " declares a second target — one per file (principle 15.a)" > "/dev/stderr"; exit 2 } + split($0, q, "\""); tgt=q[2] + if (tgt in seen) { print "fleet-hash: duplicate target " tgt > "/dev/stderr"; exit 2 } + seen[tgt]=1; ctx[tgt]=""; deps[tgt]="" + } + tgt != "" && $1=="context" && $2=="=" && ctx[tgt]=="" { split($0, q, "\""); ctx[tgt]=q[2] } + tgt != "" { + s=$0; sub(/#.*/, "", s) + while (match(s, /"target:[^"]+"/)) { + d=substr(s, RSTART+8, RLENGTH-9) + if (index(" " deps[tgt] " ", " " d " ")==0) deps[tgt]=deps[tgt] d " " + s=substr(s, RSTART+RLENGTH) + } + } + END { + for (t in ctx) { + if (ctx[t]=="") { print "fleet-hash: target " t " has no context line" > "/dev/stderr"; exit 2 } + print t "|" ctx[t] "|" deps[t] + } + } +' "${FILES[@]}" | LC_ALL=C sort > "$M/graph" + +# ── tree hashes: one git call over every context, paired by row order ─────── +PATHS=() +while IFS='|' read -r t ctx _; do PATHS+=("$REF:$ctx"); done < "$M/graph" +git rev-parse "${PATHS[@]}" > "$M/hashes" 2>/dev/null || { + while IFS='|' read -r t ctx _; do + git rev-parse "$REF:$ctx" >/dev/null 2>&1 || die "context $ctx of $t is not in $REF" + done < "$M/graph" + die "git rev-parse failed" +} +paste -d'|' <(cut -d'|' -f1 "$M/graph") "$M/hashes" > "$M/trees" + +# ── closures: recursive walk in awk → one sorted tree-hash file per target ── +# full/ holds the target's own context tree + every transitive base tree; +# bases/ holds the base trees only (the cascade component). +while IFS='|' read -r t _; do : > "$M/full/$t"; : > "$M/bases/$t"; done < "$M/graph" +awk -F'|' ' + FNR==NR { ctx[$1]=$2; deps[$1]=$3; order[++n]=$1; next } + { tree[$1]=$2 } + END { for (i=1; i<=n; i++) { t=order[i]; delete hit; walk(t, t, 1) } } + function walk(root, t, isroot, m, p, j) { + if (t in hit) return; hit[t]=1 + if (!(t in ctx)) { print "fleet-hash: " root " depends on unknown target " t > "/dev/stderr"; exit 2 } + print "F|" root "|" tree[t] + if (!isroot) print "B|" root "|" tree[t] + m = split(deps[t], p, " ") + for (j=1; j<=m; j++) if (p[j]!="") walk(root, p[j], 0) + } +' "$M/graph" "$M/trees" | LC_ALL=C sort -u \ + | awk -F'|' -v m="$M" '{ + f = m "/" ($1=="F" ? "full" : "bases") "/" $2 + if (f != prev) { if (prev != "") close(prev); prev = f } + print $3 >> f + }' + +# ── externals: one awk over every context Dockerfile → dir|image ──────────── +# Only an unindented uppercase FROM outside a backslash continuation is an +# instruction — SQL `FROM` fragments and Python `from … import` in heredoc +# RUN bodies are neither. `${VAR}` is expanded from same-file ARG defaults; +# a ref still carrying `${…}` is per-build by design (e.g. per-task bases). +DFS=() +while IFS='|' read -r t ctx _; do + [ -f "$S/$ctx/Dockerfile" ] || die "$ctx/Dockerfile missing at $REF (target $t)" + DFS+=("$S/$ctx/Dockerfile") +done < "$M/graph" +awk -v strip="$S/" ' + FNR==1 { delete alias; delete arg; cont=0 } + /^ARG [A-Za-z_]+=/ { eq=index($2,"="); arg[substr($2,1,eq-1)]=substr($2,eq+1) } + !cont && /^FROM[ \t]/ { + img=$2; if (img ~ /^--platform/) img=$3 + if (index(img, "${REGISTRY}") == 0) { + while (match(img, /\$\{[A-Za-z_]+\}/)) { + v=substr(img, RSTART+2, RLENGTH-3) + if (!(v in arg)) break + img = substr(img, 1, RSTART-1) arg[v] substr(img, RSTART+RLENGTH) + } + if (!(img in alias) && img != "scratch") { + d=substr(FILENAME, length(strip)+1); sub(/\/Dockerfile$/, "", d) + print d "|" img + } + } + for (i=1; i<=NF; i++) if ($i=="AS") alias[$(i+1)]=1 + } + { cont = ($0 ~ /\\[ \t]*$/) } +' "${DFS[@]}" | LC_ALL=C sort -u > "$M/ext" + +# ── one sha pass over every closure file, then a single join → the TSV ────── +(cd "$M" && sha full/* bases/*) > "$M/sums" +awk ' + BEGIN { FS="|" } + FILENAME ~ /graph$/ { ctxdir[$1]=$2; order[++n]=$1; next } + FILENAME ~ /trees$/ { tree[$1]=$2; next } + FILENAME ~ /ext$/ { ext[$1] = ($1 in ext) ? ext[$1] "," $2 : $2; next } + { + split($0, a, / +/) + if (split(a[2], b, "/") != 2) { print "fleet-hash: unparsable sums line: " $0 > "/dev/stderr"; exit 2 } + if (b[1]=="full") full[b[2]]=a[1] + else if (b[1]=="bases") bases[b[2]]=a[1] + else { print "fleet-hash: unparsable sums line: " $0 > "/dev/stderr"; exit 2 } + } + END { + for (i=1; i<=n; i++) { + t=order[i]; e=ext[ctxdir[t]] + print t "\t" full[t] "\t" tree[t] "\t" bases[t] "\t" (e=="" ? "-" : e) + } + } +' "$M/graph" "$M/trees" "$M/ext" "$M/sums" > "$M/all.tsv" + +col() { awk -F'\t' -v t="$1" -v c="$2" '$1==t { print $c }' "$M/all.tsv"; } +target_for_dir() { + local t + t=$(awk -F'|' -v d="$1" '$2==d { print $1 }' "$M/graph") + [ -n "$t" ] || die "no bake target with context $1" + [ "$(printf '%s\n' "$t" | wc -l)" -eq 1 ] || die "multiple targets with context $1" + printf '%s' "$t" +} +blobs() { git rev-parse "$@" 2>/dev/null || die "blob not in $REF"; } +# Combo parents come from combination.docker-bake.hcl's *_IMAGE defaults, so a +# changed default re-points the closure at the new target automatically. +parent_target() { + local p + # shellcheck disable=SC2016 # the ${REGISTRY}/${TAG} literals are the match + p=$(grep "\"$1\"" "$S/containers/core/combination.docker-bake.hcl" \ + | sed -n 's|.*"${REGISTRY}/\(.*\):${TAG}".*|\1|p') + [ -n "$p" ] || die "cannot derive $1 from combination.docker-bake.hcl" + target_for_dir "containers/$p" +} + +case "${1:-all}" in +all) + cat "$M/all.tsv" + ;; +graph) + cat "$M/graph" + ;; +combo) + { [ $# -eq 3 ] && [ -n "$2" ] && [ -n "$3" ]; } || die "usage: fleet-hash.sh combo " + b=$(target_for_dir "containers/benchmarks/$2") + a=$(target_for_dir "containers/agents/$3") + gosu=$(parent_target GOSU_IMAGE) + # The combination Dockerfiles COPY from runner/ and entrypoint/ inside the + # containers/core context, so those trees are combo inputs alongside the + # Dockerfile + bake-file blobs and the parents' closures. + blobs "$REF:containers/core/combination.Dockerfile" \ + "$REF:containers/core/combination.docker-bake.hcl" \ + "$REF:containers/core/runner" "$REF:containers/core/entrypoint" \ + | LC_ALL=C sort > "$M/eval.ctx" + LC_ALL=C sort -u "$M/full/$b" "$M/full/$a" "$M/full/$gosu" > "$M/eval.bases" + LC_ALL=C sort -u "$M/eval.ctx" "$M/eval.bases" > "$M/eval.full" + row "evals/$2--$3" "$(hash_of "$M/eval.full")" "$(hash_of "$M/eval.ctx")" \ + "$(hash_of "$M/eval.bases")" "-" + blobs "$REF:containers/core/standalone.Dockerfile" > "$M/sa.ctx" + LC_ALL=C sort -u "$M/eval.full" "$M/full/$(parent_target OTEL_IMAGE)" \ + "$M/full/$(parent_target PROCESS_COMPOSE_IMAGE)" \ + "$M/full/$(parent_target MODEL_IMAGE)" > "$M/sa.bases" + LC_ALL=C sort -u "$M/sa.ctx" "$M/sa.bases" > "$M/sa.full" + row "evals/$2--$3-standalone" "$(hash_of "$M/sa.full")" "$(hash_of "$M/sa.ctx")" \ + "$(hash_of "$M/sa.bases")" "-" + ;; +per-task) + { [ $# -eq 3 ] && [ -n "$2" ] && [ -n "$3" ]; } || die "usage: fleet-hash.sh per-task " + case "$3" in *[[:space:]]*) die "task id must not contain whitespace" ;; esac + [ -z "${SKILLS_BENCH_REF:-}" ] || die "SKILLS_BENCH_REF is set — an out-of-tree ref override defeats input hashing; pin the ref in the benchmark dir" + t=$(target_for_dir "containers/benchmarks/$2") + h=$(col "$t" 2) + row "per-task/$2/$3" "$(printf '%s %s' "$h" "$3" | sha | cut -d' ' -f1)" \ + "$(col "$t" 3)" "$(col "$t" 4)" "$(col "$t" 5)" + ;; +*) + die "unknown command $1 (expected: all | combo | per-task | graph)" + ;; +esac diff --git a/tests/static/Cargo.toml b/tests/static/Cargo.toml index c55e1705..b9ed1336 100644 --- a/tests/static/Cargo.toml +++ b/tests/static/Cargo.toml @@ -35,3 +35,7 @@ path = "task_inspection.rs" [[test]] name = "grader" path = "grader.rs" + +[[test]] +name = "input_hash" +path = "input_hash.rs" diff --git a/tests/static/fleet-hash.sweep.sh b/tests/static/fleet-hash.sweep.sh new file mode 100755 index 00000000..042bd261 --- /dev/null +++ b/tests/static/fleet-hash.sweep.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# tests/static/fleet-hash.sweep.sh — pin fleet-hash's bake-graph reading to +# plain bake's own evaluation (the wiring gate for the build-input hash, +# alongside compose.config.sweep.sh and helm.sweep.sh). +# +# fleet-hash.sh parses the per-artifact bake files directly so its Rust tests +# run without the docker CLI. This sweep is the independent oracle: one +# `docker buildx bake --print` over the root + every per-artifact file +# (the combination file is parameterized and excluded on both sides), compared +# BIDIRECTIONALLY — every fleet-hash target must match bake's context and +# target: deps exactly, and bake must know no target fleet-hash missed (a +# dropped target is a silently unhashed, silently carried-forward image). +# `--print` is a client-side HCL evaluation: no daemon, no images, no creds. +# Fail loud; offline. +set -uo pipefail +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd) || exit 2 + +command -v docker >/dev/null || { echo "docker not found — required for the bake --print gate"; exit 1; } +command -v jq >/dev/null || { echo "jq not found — required for the bake --print gate"; exit 1; } +docker buildx version >/dev/null 2>&1 || { echo "docker buildx plugin not found"; exit 1; } + +shopt -s nullglob +cd "$ROOT" || exit 2 + +graph=$(bash containers/scripts/fleet-hash.sh graph) || { echo "fleet-hash graph failed"; exit 1; } + +args=(-f containers/docker-bake.hcl) +for f in containers/core/*/docker-bake.hcl containers/gateways/*/docker-bake.hcl \ + containers/agents/*/docker-bake.hcl containers/benchmarks/*/docker-bake.hcl \ + containers/models/*/docker-bake.hcl; do args+=(-f "$f"); done + +err=$(mktemp) +trap 'rm -f "$err"' EXIT +# shellcheck disable=SC2046 # target names never contain whitespace +print=$(docker buildx bake "${args[@]}" --print $(cut -d'|' -f1 <<<"$graph") 2>"$err") \ + || { echo "bake --print rejected the fleet-hash target list:"; cat "$err"; exit 1; } + +fails=0 + +# Reverse direction: bake's evaluated target set == fleet-hash's. +if ! diff <(cut -d'|' -f1 <<<"$graph" | LC_ALL=C sort) \ + <(jq -r '.target | keys[]' <<<"$print" | LC_ALL=C sort); then + echo "FAIL: target sets differ (fleet-hash vs bake --print)" + fails=$((fails + 1)) +fi + +# Forward direction: context and target: deps agree, target by target. +while IFS='|' read -r t ctx deps; do + bctx=$(jq -r --arg t "$t" '.target[$t].context // ""' <<<"$print") + if [ "$bctx" != "$ctx" ]; then + echo "FAIL: $t context — fleet-hash '$ctx' vs bake '$bctx'" + fails=$((fails + 1)) + fi + bdeps=$(jq -r --arg t "$t" \ + '[.target[$t].contexts // {} | .[] | select(startswith("target:")) | ltrimstr("target:")] | sort | join(" ")' \ + <<<"$print") + # shellcheck disable=SC2086 # deps is a space-separated list, split intended + sdeps=$(printf '%s\n' $deps | LC_ALL=C sort | paste -sd' ' - | sed 's/^ *//') + if [ "$bdeps" != "$sdeps" ]; then + echo "FAIL: $t deps — fleet-hash '$sdeps' vs bake '$bdeps'" + fails=$((fails + 1)) + fi +done <<<"$graph" + +n=$(wc -l <<<"$graph" | tr -d ' ') +if [ "$fails" -gt 0 ]; then + echo "fleet-hash graph drifts from bake --print: $fails failure(s) across $n targets" + exit 1 +fi +echo "OK: fleet-hash graph == bake --print for all $n targets (both directions)" diff --git a/tests/static/input_hash.rs b/tests/static/input_hash.rs new file mode 100644 index 00000000..c99ad4ee --- /dev/null +++ b/tests/static/input_hash.rs @@ -0,0 +1,445 @@ +//! Build-input hash primitive — the "repository's computed hash" side of the +//! carried-forward contract (delivery/RULES.md rules 11–14). +//! +//! `containers/scripts/fleet-hash.sh` must be a pure function of the committed +//! tree at REF: deterministic, sensitive to any context change, cascading +//! through the *transitive* bake graph (a base-of-base edit dirties the leaf), +//! and blind to uncommitted worktree state. These are the properties the +//! selective-release machinery will trust, so they are proven on synthetic git +//! fixtures (where mutations are committed like the real tree) and the script +//! is exercised over the real repo. Offline, daemon-free (tests/static/RULES.md +//! rule 1): only `git`, `bash`, and awk/sed run. The graph parse itself is +//! separately gated against `docker buildx bake --print` by +//! `tests/static/fleet-hash.sweep.sh` (the static-composition CI job). + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::time::{SystemTime, UNIX_EPOCH}; +use test_support::repo_root; + +fn script() -> PathBuf { + repo_root().join("containers/scripts/fleet-hash.sh") +} + +fn run(repo: &Path, git_ref: &str, args: &[&str], env: &[(&str, &str)]) -> Output { + let mut cmd = Command::new("bash"); + cmd.arg(script()) + .args(args) + .env("REPO_ROOT", repo) + .env("REF", git_ref) + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env_remove("GIT_INDEX_FILE") + .env_remove("SKILLS_BENCH_REF"); + for (k, v) in env { + cmd.env(k, v); + } + cmd.output().expect("run fleet-hash.sh") +} + +fn fleet_hash_at(repo: &Path, git_ref: &str, args: &[&str]) -> String { + let out = run(repo, git_ref, args, &[]); + assert!( + out.status.success(), + "fleet-hash {args:?} failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).expect("fleet-hash output is utf8") +} + +fn fleet_hash(repo: &Path, args: &[&str]) -> String { + fleet_hash_at(repo, "HEAD", args) +} + +/// Expect a loud failure: non-zero exit and the given stderr fragment. +fn expect_die(repo: &Path, args: &[&str], env: &[(&str, &str)], msg: &str) { + let out = run(repo, "HEAD", args, env); + assert!( + !out.status.success(), + "fleet-hash {args:?} unexpectedly succeeded" + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains(msg), + "stderr for {args:?} lacks {msg:?}:\n{stderr}" + ); +} + +/// target -> (hash, context-hash, bases-hash, externals) +fn rows(output: &str) -> HashMap { + output + .lines() + .map(|l| { + let f: Vec<&str> = l.split('\t').collect(); + assert_eq!(f.len(), 5, "malformed row: {l}"); + ( + f[0].to_string(), + ( + f[1].to_string(), + f[2].to_string(), + f[3].to_string(), + f[4].to_string(), + ), + ) + }) + .collect() +} + +fn is_sha256(s: &str) -> bool { + s.len() == 64 && s.chars().all(|c| c.is_ascii_hexdigit()) +} + +fn is_tree_hash(s: &str) -> bool { + s.len() == 40 && s.chars().all(|c| c.is_ascii_hexdigit()) +} + +// ── synthetic fixtures ────────────────────────────────────────────────────── + +/// A throwaway git repo; the directory is removed on drop even when an +/// assertion fails mid-test. +struct Fixture(PathBuf); + +impl Drop for Fixture { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +impl Fixture { + fn new(name: &str) -> Self { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = + std::env::temp_dir().join(format!("fleet-hash-{name}-{}-{nanos}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + git(&dir, &["init", "-q"]); + Fixture(dir) + } + + fn write(&self, rel: &str, content: &str) { + let p = self.0.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, content).unwrap(); + } + + fn artifact(&self, kind: &str, name: &str, target: &str, deps: &[&str], dockerfile: &str) { + let dir = format!("containers/{kind}/{name}"); + self.write(&format!("{dir}/Dockerfile"), dockerfile); + let contexts = if deps.is_empty() { + String::new() + } else { + let entries: String = deps + .iter() + .map(|d| format!(" \"${{REGISTRY}}/core/{d}\" = \"target:{d}\"\n")) + .collect(); + format!(" contexts = {{\n{entries} }}\n") + }; + self.write( + &format!("{dir}/docker-bake.hcl"), + &format!( + "target \"{target}\" {{\n context = \"{dir}\"\n{contexts} tags = [\"${{REGISTRY}}/{kind}/{name}:${{TAG}}\"]\n}}\n" + ), + ); + } + + fn commit(&self, msg: &str) { + git(&self.0, &["add", "."]); + git(&self.0, &["commit", "-q", "-m", msg]); + } +} + +fn git(repo: &Path, args: &[&str]) { + let out = Command::new("git") + .args([ + "-c", + "user.email=test@test", + "-c", + "user.name=test", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .current_dir(repo) + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_SYSTEM", "/dev/null") + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env_remove("GIT_INDEX_FILE") + .output() + .expect("run git"); + assert!( + out.status.success(), + "git {args:?} failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); +} + +/// Depth-2 chain with a diamond: leaf-a → base-y → base-x, leaf-c → {base-x, +/// base-y}, leaf-b independent. Deep enough that a non-transitive walk is +/// observably wrong (91/153 real targets sit at depth 2). +fn fleet_fixture(name: &str) -> Fixture { + let fx = Fixture::new(name); + fx.artifact( + "core", + "base-x", + "base-x", + &[], + "FROM alpine:3.20\nRUN echo x\n", + ); + fx.artifact( + "core", + "base-y", + "base-y", + &["base-x"], + "FROM ${REGISTRY}/core/base-x:latest\nRUN echo y\n", + ); + fx.artifact( + "benchmarks", + "leaf-a", + "benchmark-leaf-a", + &["base-y"], + "FROM ${REGISTRY}/core/base-y:latest\nRUN echo a\n", + ); + fx.artifact( + "benchmarks", + "leaf-b", + "benchmark-leaf-b", + &[], + "FROM debian:12-slim\nRUN echo b\n", + ); + fx.artifact( + "benchmarks", + "leaf-c", + "benchmark-leaf-c", + &["base-x", "base-y"], + "FROM ${REGISTRY}/core/base-x:latest\nRUN echo c\n", + ); + fx.commit("fixture"); + fx +} + +/// Deterministic; a leaf edit moves only that leaf (context component); a +/// base-of-base edit cascades transitively through the diamond (bases +/// component) and spares independent leaves; REF pins the computation; the +/// uncommitted worktree is invisible. +#[test] +fn hash_is_deterministic_transitive_and_pure() { + let fx = fleet_fixture("props"); + let repo = &fx.0; + + let first = fleet_hash(repo, &[]); + assert_eq!(first, fleet_hash(repo, &[]), "two runs must be identical"); + let v0 = rows(&first); + assert_eq!(v0.len(), 5); + for (t, (hash, ctxh, basesh, _)) in &v0 { + assert!(is_sha256(hash), "{t}: hash not sha256"); + assert!(is_tree_hash(ctxh), "{t}: context hash not a git tree hash"); + assert!(is_sha256(basesh), "{t}: bases hash not sha256"); + } + assert_eq!(v0["base-x"].3, "alpine:3.20", "external FROM must surface"); + assert_eq!(v0["base-y"].3, "-", "in-repo FROM is not external"); + assert_eq!(v0["benchmark-leaf-b"].3, "debian:12-slim"); + + // Leaf edit: only leaf-a moves, and only its context component. + fx.write("containers/benchmarks/leaf-a/extra.txt", "changed\n"); + fx.commit("edit leaf-a"); + let v1 = rows(&fleet_hash(repo, &[])); + assert_ne!(v1["benchmark-leaf-a"].0, v0["benchmark-leaf-a"].0); + assert_ne!(v1["benchmark-leaf-a"].1, v0["benchmark-leaf-a"].1); + assert_eq!(v1["benchmark-leaf-a"].2, v0["benchmark-leaf-a"].2); + for t in ["base-x", "base-y", "benchmark-leaf-b", "benchmark-leaf-c"] { + assert_eq!(v1[t], v0[t], "{t} must not move on a leaf-a edit"); + } + + // Base-of-base edit: base-x moves, and the cascade reaches base-y, + // leaf-a (TRANSITIVELY — its direct dep is base-y), and diamond leaf-c, + // in every case through the bases component only. leaf-b is untouched. + fx.write("containers/core/base-x/extra.txt", "changed\n"); + fx.commit("edit base-x"); + let v2 = rows(&fleet_hash(repo, &[])); + assert_ne!(v2["base-x"].0, v1["base-x"].0); + for t in ["base-y", "benchmark-leaf-a", "benchmark-leaf-c"] { + assert_ne!(v2[t].0, v1[t].0, "{t} must cascade on a base-x edit"); + assert_eq!(v2[t].1, v1[t].1, "{t}: context component must not move"); + assert_ne!(v2[t].2, v1[t].2, "{t}: bases component must move"); + } + assert_eq!(v2["benchmark-leaf-b"], v1["benchmark-leaf-b"]); + + // REF pins the computation: hashing HEAD~1 reproduces the prior state. + assert_eq!(rows(&fleet_hash_at(repo, "HEAD~1", &[])), v1); + + // Purity: uncommitted edits and uncommitted new artifacts are invisible. + fx.write("containers/core/base-x/Dockerfile", "FROM busybox\n"); + fx.artifact( + "benchmarks", + "leaf-new", + "benchmark-leaf-new", + &[], + "FROM scratch\n", + ); + let v3 = rows(&fleet_hash(repo, &[])); + assert_eq!( + v3, v2, + "uncommitted worktree state must not affect the hash" + ); + assert!(!v3.contains_key("benchmark-leaf-new")); +} + +/// The externals awk is a Dockerfile FROM parser; pin every branch: stage +/// aliases, --platform, scratch, ARG-default expansion, backslash-continued +/// SQL FROM, and heredoc `from … import` bodies. +#[test] +fn externals_parse_every_dockerfile_shape() { + let fx = Fixture::new("externals"); + fx.artifact( + "benchmarks", + "shapes", + "benchmark-shapes", + &[], + concat!( + "ARG GO_VERSION=1.23\n", + "FROM golang:${GO_VERSION} AS build\n", + "FROM --platform=$BUILDPLATFORM alpine:3.20 AS helper\n", + "FROM redis:7 AS redis\n", + "FROM redis\n", + "FROM build\n", + "FROM scratch\n", + "RUN duckdb -c \"COPY (SELECT 1) TO 'x' \\\n", + " FROM 'hf://datasets/fake@~parquet/x.parquet'\"\n", + "RUN python3 <<'PYEOF'\n", + "from difflib import SequenceMatcher\n", + "from collections import Counter\n", + "PYEOF\n", + ), + ); + fx.commit("shapes"); + let ext = &rows(&fleet_hash(&fx.0, &[]))["benchmark-shapes"].3; + assert_eq!( + ext, "alpine:3.20,golang:1.23,redis:7", + "externals must expand ARG defaults, keep aliased first-use images, and \ + ignore stage reuse, scratch, SQL FROM continuations, and heredoc imports" + ); +} + +/// Malformed inputs and misuse must fail loudly (exit 2 + a named cause), +/// never produce a hash — a wrong hash fails open into a stale release. +#[test] +fn error_paths_fail_loud() { + let two = Fixture::new("two-targets"); + two.artifact("core", "ok", "ok", &[], "FROM scratch\n"); + two.write( + "containers/core/ok/docker-bake.hcl", + "target \"ok\" {\n context = \"containers/core/ok\"\n}\ntarget \"sneaky\" {\n context = \"containers/core/ok\"\n}\n", + ); + two.commit("two targets"); + expect_die(&two.0, &[], &[], "declares a second target"); + + let noctx = Fixture::new("no-context"); + noctx.artifact("core", "ok", "ok", &[], "FROM scratch\n"); + noctx.write( + "containers/core/ok/docker-bake.hcl", + "target \"ok\" {\n tags = [\"x\"]\n}\n", + ); + noctx.commit("no context"); + expect_die(&noctx.0, &[], &[], "has no context line"); + + let unknown = Fixture::new("unknown-dep"); + unknown.artifact("core", "ok", "ok", &["ghost"], "FROM scratch\n"); + unknown.commit("unknown dep"); + expect_die(&unknown.0, &[], &[], "depends on unknown target ghost"); + + let nodf = Fixture::new("no-dockerfile"); + nodf.artifact("core", "ok", "ok", &[], "FROM scratch\n"); + std::fs::remove_file(nodf.0.join("containers/core/ok/Dockerfile")).unwrap(); + nodf.commit("no dockerfile"); + expect_die(&nodf.0, &[], &[], "Dockerfile missing"); + + let ok = fleet_fixture("misuse"); + expect_die(&ok.0, &["frobnicate"], &[], "unknown command"); + expect_die(&ok.0, &["per-task", "leaf-a", ""], &[], "usage:"); + expect_die(&ok.0, &["per-task", "leaf-a", "a b"], &[], "whitespace"); + expect_die( + &ok.0, + &["per-task", "leaf-a", "t0"], + &[("SKILLS_BENCH_REF", "x")], + "out-of-tree ref override", + ); +} + +/// The real repo: every per-artifact bake file yields exactly one row, +/// byte-identically across runs, and the combo/per-task providers are +/// sensitive to each of their inputs. +#[test] +fn real_repo_hashes_every_target() { + let root = repo_root(); + let bake_files: usize = ["core", "gateways", "agents", "benchmarks", "models"] + .iter() + .map(|kind| { + std::fs::read_dir(root.join("containers").join(kind)) + .expect("read kind dir") + .filter_map(Result::ok) + .filter(|e| e.path().join("docker-bake.hcl").is_file()) + .count() + }) + .sum(); + + let raw = fleet_hash(&root, &[]); + assert_eq!( + raw, + fleet_hash(&root, &[]), + "real-repo runs must be byte-identical" + ); + let all = rows(&raw); + assert_eq!( + all.len(), + bake_files, + "one row per per-artifact bake file (note: this mirrors the script's \ + own glob; the independent oracle is fleet-hash.sweep.sh vs bake --print)" + ); + for (t, (hash, ctxh, _, _)) in &all { + assert!(is_sha256(hash), "{t}: hash not sha256"); + assert!(is_tree_hash(ctxh), "{t}: context hash not a git tree hash"); + } + assert!(all.contains_key("benchmark-aime") && all.contains_key("entrypoint")); + + // Combo: sensitive to the agent axis, and standalone differs from lean. + let cc = rows(&fleet_hash(&root, &["combo", "aime", "claude-code"])); + let cx = rows(&fleet_hash(&root, &["combo", "aime", "codex"])); + let lean = &cc["evals/aime--claude-code"]; + assert_ne!( + lean.0, cx["evals/aime--codex"].0, + "combo must track the agent" + ); + assert_ne!( + lean.0, cc["evals/aime--claude-code-standalone"].0, + "standalone must differ from the lean combo" + ); + + // Per-task: sensitive to the task id, sharing the benchmark's components. + let t0 = rows(&fleet_hash( + &root, + &["per-task", "terminal-bench", "task-0"], + )); + let t1 = rows(&fleet_hash( + &root, + &["per-task", "terminal-bench", "task-1"], + )); + let (r0, r1) = ( + &t0["per-task/terminal-bench/task-0"], + &t1["per-task/terminal-bench/task-1"], + ); + assert_ne!(r0.0, r1.0, "per-task hash must track the task id"); + let bench = &all["benchmark-terminal-bench"]; + assert_ne!( + r0.0, bench.0, + "per-task hash must differ from the benchmark's" + ); + assert_eq!( + (&r0.1, &r0.2, &r0.3), + (&bench.1, &bench.2, &bench.3), + "per-task rows share the benchmark's components" + ); +}