diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index e9e44dc0..c65f7a36 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -160,9 +160,20 @@ jobs: mapfile -t DEPS < <(find containers/core containers/gateways -name docker-bake.hcl \ -exec grep -hoE '^target "[^"]+"' {} + | sed -E 's/^target "([^"]+)"/\1/' | sort -u) echo "Building ${#DEPS[@]} shared deps (${ARCH}): ${DEPS[*]}" + # Stamp each base's build-input hash (delivery/RULES.md rule 12) — + # per-target, never `*.labels`: this one invocation bakes ~19 targets + # with ~19 different hashes. The label records the repo-computed + # inputs at this commit; arch-independent by construction. + HASHES=$(bash containers/scripts/fleet-hash.sh) + LBL=() + for d in "${DEPS[@]}"; do + h=$(awk -F'\t' -v t="$d" '$1==t{print $2}' <<< "$HASHES") + [ -n "$h" ] || { echo "::error::no input-hash for $d"; exit 1; } + LBL+=(--set "${d}.labels.eval.input-hash=${h}") + done ACT=(--provenance=mode=max --sbom=true "${DEPS[@]}" --push) [ "${{ inputs.dry_run }}" = "true" ] && ACT=("${DEPS[@]}" --print) - docker buildx bake "${FARGS[@]}" \ + docker buildx bake "${FARGS[@]}" "${LBL[@]}" \ --set "*.args.REGISTRY=${REGISTRY}" \ --set "*.labels.org.opencontainers.image.source=https://github.com/Exgentic/eval-containers" \ --set "*.secrets=id=HF_TOKEN,env=HF_TOKEN" \ @@ -346,10 +357,18 @@ jobs: echo "::warning::build attempt $n/$max failed; backing off"; sleep $(( n*30 + (RANDOM % 30) )) done } + # Stamp the leaf's build-input hash (delivery/RULES.md rule 12). The + # label records the repo-computed inputs at this commit; on a frozen- + # bases dev dispatch the leaf may build FROM an older frozen base than + # the hash's base component claims — tagged releases always rebuild + # bases first, so release labels are exact. + H=$(bash containers/scripts/fleet-hash.sh | awk -F'\t' -v t="$T" '$1==t{print $2}') + [ -n "$H" ] || { echo "::error::no input-hash for $T"; exit 1; } # dry_run validates config only (bake --print); else build + attest + push. ACT=(--provenance=mode=max --sbom=true "$T" --push) [ "${{ inputs.dry_run }}" = "true" ] && ACT=("$T" --print) retry docker buildx bake "${FARGS[@]}" "${OV[@]}" \ + --set "${T}.labels.eval.input-hash=${H}" \ --set "*.args.REGISTRY=${REGISTRY}" \ --set "*.labels.org.opencontainers.image.source=https://github.com/Exgentic/eval-containers" \ --set "*.secrets=id=HF_TOKEN,env=HF_TOKEN" \ @@ -418,12 +437,17 @@ jobs: cache="${REGISTRY}/buildcache/${B}-${tid}-${ARCH}" if [ "${{ inputs.skip_published }}" = "true" ] && docker buildx imagetools inspect "$ref" >/dev/null 2>&1; then echo "skip-published: $ref"; continue; fi if [ "$DRY" = "true" ]; then echo "dry-run: $ref ($KIND)"; continue; fi + # Per-task build-input hash (delivery/RULES.md rule 12): the raw + # task id is part of the preimage; build.sh stamps it via + # EVAL_INPUT_HASH, bake via a per-target label --set. + H=$(bash containers/scripts/fleet-hash.sh per-task "$B" "$TASK" | cut -f2) + [ -n "$H" ] || { echo "::error::no input-hash for $B/$TASK"; fails=$((fails+1)); continue; } echo "::group::$ref ($KIND)" if [ "$KIND" = "script" ]; then # It builds native (per-arch); push only if the built arch matches this # runner — a task whose upstream pins a different arch is skipped so the # merge keeps it single-arch (never a mislabeled :TAG-). - if retry bash "containers/benchmarks/$B/build.sh" "$ref" "$TASK"; then + if EVAL_INPUT_HASH="$H" retry bash "containers/benchmarks/$B/build.sh" "$ref" "$TASK"; then built=$(podman image inspect "$ref" --format '{{.Architecture}}' 2>/dev/null || echo unknown) if [ "$built" = "$ARCH" ]; then retry podman push "$ref" || { echo "::error::push failed: $ref"; fails=$((fails+1)); } @@ -434,6 +458,7 @@ jobs: else # bake: FROM the Epoch per-task base + EVAL_TASK_ID; pull the frozen entrypoint. if retry docker buildx bake -f containers/docker-bake.hcl -f "containers/benchmarks/$B/docker-bake.hcl" \ + --set "benchmark-${B}.labels.eval.input-hash=${H}" \ --set "benchmark-${B}.args.EVAL_TASK_ID=${TASK}" \ --set "benchmark-${B}.args.EVAL_BASE_ARCH=${EPOCH_ARCH}" \ --set "benchmark-${B}.contexts.${REGISTRY}/core/entrypoint=docker-image://${REGISTRY}/core/entrypoint:${TAG}" \ @@ -606,6 +631,11 @@ jobs: PLAT="${PLAT:-linux/amd64}" # fall back to amd64 if inspect is unreadable fi if [ "${{ inputs.skip_published }}" = "true" ] && docker buildx imagetools inspect "${REGISTRY}/evals/${eb}--${A}:${TAG}" >/dev/null 2>&1; then echo "skip-published: evals/${eb}--${A}"; continue; fi + # Combo build-input hashes (delivery/RULES.md rule 12) — per-target, + # never `*.labels`: eval and eval-standalone hash differently. + HROWS=$(bash containers/scripts/fleet-hash.sh combo "$B" "$A" ${TASK:+"$TASK"}) \ + || { echo "::error::no input-hash for evals/${eb}--${A}"; fails=$((fails+1)); continue; } + EH=$(sed -n 1p <<< "$HROWS" | cut -f2); SH=$(sed -n 2p <<< "$HROWS" | cut -f2) # eval = lean base (sidecar mode); eval-standalone = single-container # bundle (gateway+otelcol+process-compose in-image). bake builds eval # once, then layers standalone on it via the eval-base context. @@ -616,6 +646,8 @@ jobs: if EVAL_BENCHMARK="$eb" EVAL_AGENT="$A" \ BENCHMARK_IMAGE="$bench_img" AGENT_IMAGE="${REGISTRY}/agents/${A}:${TAG}" \ retry docker buildx bake -f containers/docker-bake.hcl -f containers/core/combination.docker-bake.hcl \ + --set "eval.labels.eval.input-hash=${EH}" \ + --set "eval-standalone.labels.eval.input-hash=${SH}" \ --set "*.platform=${PLAT}" \ "${ACT[@]}"; then :; \ else echo "::error::combo failed: evals/${eb}--${A}"; fails=$((fails+1)); fi diff --git a/containers/benchmarks/skills-bench/build.sh b/containers/benchmarks/skills-bench/build.sh index 7f6c7f7e..4ccbf6f5 100755 --- a/containers/benchmarks/skills-bench/build.sh +++ b/containers/benchmarks/skills-bench/build.sh @@ -43,7 +43,11 @@ echo "[skills-bench] 1/2 building task env for '${TASK}' (environment/Dockerfile docker build -t "${ENVIMG}" "${TASKDIR}/environment" echo "[skills-bench] 2/2 overlaying the eval pipeline -> ${IMAGE}" +# EVAL_INPUT_HASH (optional): the release stamps the build-input hash here +# (delivery/RULES.md rule 12) — this path has no bake invocation to --set it on. +# shellcheck disable=SC2086 # the hash is hex; empty expands to no arg docker build -t "${IMAGE}" \ + ${EVAL_INPUT_HASH:+--label=eval.input-hash=${EVAL_INPUT_HASH}} \ --build-arg "TASK_BASE=${ENVIMG}" \ --build-arg "EVAL_TASK_ID=${TASK}" \ --build-arg "SB_REF=${REF}" \ diff --git a/containers/benchmarks/terminal-bench/build.sh b/containers/benchmarks/terminal-bench/build.sh index 2fb470f4..54962713 100755 --- a/containers/benchmarks/terminal-bench/build.sh +++ b/containers/benchmarks/terminal-bench/build.sh @@ -33,7 +33,11 @@ echo "[terminal-bench] 1/2 building task env for '${TASK}' (environment/Dockerfi docker build -t "${ENVIMG}" "${REPO}#${REF}:tasks/${TASK}/environment" echo "[terminal-bench] 2/2 overlaying the eval pipeline -> ${IMAGE}" +# EVAL_INPUT_HASH (optional): the release stamps the build-input hash here +# (delivery/RULES.md rule 12) — this path has no bake invocation to --set it on. +# shellcheck disable=SC2086 # the hash is hex; empty expands to no arg docker build -t "${IMAGE}" \ + ${EVAL_INPUT_HASH:+--label=eval.input-hash=${EVAL_INPUT_HASH}} \ --build-arg "TASK_BASE=${ENVIMG}" \ --build-arg "EVAL_TASK_ID=${TASK}" \ --build-arg "TBENCH_REF=${REF}" \ diff --git a/containers/scripts/fleet-hash.sh b/containers/scripts/fleet-hash.sh index efb32936..55ab4b9c 100755 --- a/containers/scripts/fleet-hash.sh +++ b/containers/scripts/fleet-hash.sh @@ -15,7 +15,8 @@ # # Usage: # fleet-hash.sh # every static bake target -# fleet-hash.sh combo # eval + eval-standalone rows +# fleet-hash.sh combo [task] # eval + eval-standalone rows +# # (task ⇒ the per-task combo variant) # fleet-hash.sh per-task # one per-task image row # fleet-hash.sh graph # target|context|deps — the context # # column is also the registry ref @@ -190,7 +191,10 @@ graph) cat "$M/graph" ;; combo) - { [ $# -eq 3 ] && [ -n "$2" ] && [ -n "$3" ]; } || die "usage: fleet-hash.sh combo " + { [ $# -ge 3 ] && [ $# -le 4 ] && [ -n "$2" ] && [ -n "$3" ]; } \ + || die "usage: fleet-hash.sh combo [task]" + task="${4:-}" + case "$task" in *[[:space:]]*) die "task id must not contain whitespace" ;; esac b=$(target_for_dir "containers/benchmarks/$2") a=$(target_for_dir "containers/agents/$3") gosu=$(parent_target GOSU_IMAGE) @@ -203,15 +207,23 @@ combo) | LC_ALL=C sort > "$M/eval.ctx" LC_ALL=C sort -u "$M/full/$b" "$M/full/$a" "$M/full/$gosu" > "$M/eval.bases" LC_ALL=C sort -u "$M/eval.ctx" "$M/eval.bases" > "$M/eval.full" - row "evals/$2--$3" "$(hash_of "$M/eval.full")" "$(hash_of "$M/eval.ctx")" \ - "$(hash_of "$M/eval.bases")" "-" + # A per-task combo mixes the task id into the hash the same way per-task + # does, and names its rows with the release's - convention. + with_task() { + if [ -n "$task" ]; then printf '%s %s' "$1" "$task" | sha | cut -d' ' -f1 + else printf '%s' "$1"; fi + } + eb="$2" + [ -z "$task" ] || eb="$2-$(printf '%s' "$task" | tr '[:upper:]' '[:lower:]')" + row "evals/$eb--$3" "$(with_task "$(hash_of "$M/eval.full")")" \ + "$(hash_of "$M/eval.ctx")" "$(hash_of "$M/eval.bases")" "-" blobs "$REF:containers/core/standalone.Dockerfile" > "$M/sa.ctx" LC_ALL=C sort -u "$M/eval.full" "$M/full/$(parent_target OTEL_IMAGE)" \ "$M/full/$(parent_target PROCESS_COMPOSE_IMAGE)" \ "$M/full/$(parent_target MODEL_IMAGE)" > "$M/sa.bases" LC_ALL=C sort -u "$M/sa.ctx" "$M/sa.bases" > "$M/sa.full" - row "evals/$2--$3-standalone" "$(hash_of "$M/sa.full")" "$(hash_of "$M/sa.ctx")" \ - "$(hash_of "$M/sa.bases")" "-" + row "evals/$eb--$3-standalone" "$(with_task "$(hash_of "$M/sa.full")")" \ + "$(hash_of "$M/sa.ctx")" "$(hash_of "$M/sa.bases")" "-" ;; per-task) { [ $# -eq 3 ] && [ -n "$2" ] && [ -n "$3" ]; } || die "usage: fleet-hash.sh per-task " diff --git a/tests/static/input_hash.rs b/tests/static/input_hash.rs index c99ad4ee..2eadd88b 100644 --- a/tests/static/input_hash.rs +++ b/tests/static/input_hash.rs @@ -418,6 +418,24 @@ fn real_repo_hashes_every_target() { "standalone must differ from the lean combo" ); + // Per-task combo: rows follow the - naming, the hash tracks + // the task id, and the no-task hashes above stay the definition-frozen ones. + let ct0 = rows(&fleet_hash(&root, &["combo", "aime", "claude-code", "T-0"])); + let ct1 = rows(&fleet_hash(&root, &["combo", "aime", "claude-code", "T-1"])); + let p0 = &ct0["evals/aime-t-0--claude-code"]; + assert_ne!( + p0.0, ct1["evals/aime-t-1--claude-code"].0, + "per-task combo hash must track the task id" + ); + assert_ne!( + p0.0, lean.0, + "per-task combo must differ from the shared combo" + ); + assert_ne!( + p0.0, ct0["evals/aime-t-0--claude-code-standalone"].0, + "per-task standalone must differ from its lean variant" + ); + // Per-task: sensitive to the task id, sharing the benchmark's components. let t0 = rows(&fleet_hash( &root,