From 34e97ad372f24f1184c9e33e024fac39a42d86bc Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Sun, 9 Aug 2026 17:47:43 +0300 Subject: [PATCH 1/2] =?UTF-8?q?feat(release):=20carried-forward=20retag=20?= =?UTF-8?q?=E2=80=94=20unchanged=20images=20keep=20their=20digests=20(#306?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rung 5, the last of the carried-forward ladder (#292): the release acts on freshness. Every build job first asks fleet-status whether its image's inputs are unchanged from the prior release (:latest, the promoted last release; :latest- for per-arch jobs) and, when fresh, retags that digest with imagetools create — the same manifest copy the :latest promotion already uses — instead of rebuilding (delivery/RULES.md rule 13). Same bits, new tag; provenance/SBOM ride along and honestly name the original build. Per site: - bases: the whole-set FROZEN sentinel becomes a per-target judgment — stale bases now rebuild even on dev dispatches (which also eliminates the stamped-label skew that frozen bases could cause), fresh ones retag; the bake covers only the stale subset - leaves: check + retag before the bake; the ref comes from the graph context column, so dotted models (gpt-5.4) are judged correctly — the old skip_published mapping never matched them - per-task: per-item check keyed on the task-mixed hash - combos: eval and eval-standalone judge independently — a stale standalone can rebuild while the lean combo carries forward fleet-status.sh gains the one-ref form the workflow calls: fleet-status.sh check # exit 0 = fresh so the label-read logic keeps a single home. skip_published is deleted — the freshness check subsumes it with hash-correct semantics (tag existence said nothing about content). A new force_rebuild input bypasses every check: upstream base/package drift is invisible to the input hash, so CVE refreshes force-rebuild (principle 9 classes them as patch releases). The CVE gate scans whatever :TAG points to, carried or fresh (rule 15). docs/concepts/release-freshness.md documents what a version tag now means (a coherent input set, not a build timestamp) per docs rule 15. Signed-off-by: Elron Bandel --- .github/workflows/release-images.yml | 100 ++++++++++++++++++++------- containers/scripts/fleet-status.sh | 33 ++++++--- docs/README.md | 1 + docs/concepts/release-freshness.md | 45 ++++++++++++ 4 files changed, 144 insertions(+), 35 deletions(-) create mode 100644 docs/concepts/release-freshness.md diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index c65f7a36..e5be4ab9 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -69,8 +69,8 @@ on: only: description: "Debug: restrict matrices to these space-separated leaf targets (e.g. 'benchmark-gsm8k agent-claude-code'); blank = whole fleet" default: "" - skip_published: - description: "Incremental: skip building any image already at :TAG (build only missing/failed). Use when re-running and the published images are still current." + force_rebuild: + description: "Rebuild every image even when its recorded input-hash is fresh (use for CVE/base refreshes — upstream base drift is invisible to the input hash)" type: boolean default: false @@ -150,29 +150,44 @@ jobs: ARCH: ${{ matrix.arch }} run: | export TAG="${TAG}-${ARCH}" # per-arch tag; merge builds the :TAG manifest list - if [ "${{ inputs.rebuild_bases }}" != "true" ] && \ - docker buildx imagetools inspect "${REGISTRY}/core/entrypoint:${TAG}" >/dev/null 2>&1; then - echo "Bases present at :${TAG} — FROZEN (skip; stable digest → leaf caches hit)." - exit 0 - fi mapfile -t FILES < <(find containers -name docker-bake.hcl -not -name 'combination*') FARGS=(); for f in "${FILES[@]}"; do FARGS+=(-f "$f"); done mapfile -t DEPS < <(find containers/core containers/gateways -name docker-bake.hcl \ -exec grep -hoE '^target "[^"]+"' {} + | sed -E 's/^target "([^"]+)"/\1/' | sort -u) - echo "Building ${#DEPS[@]} shared deps (${ARCH}): ${DEPS[*]}" # Stamp each base's build-input hash (delivery/RULES.md rule 12) — # per-target, never `*.labels`: this one invocation bakes ~19 targets # with ~19 different hashes. The label records the repo-computed # inputs at this commit; arch-independent by construction. HASHES=$(bash containers/scripts/fleet-hash.sh) - LBL=() + GRAPH=$(bash containers/scripts/fleet-hash.sh graph) + # Carried-forward (rule 13): a base whose inputs are unchanged from + # the prior release (:latest-) is retagged, never rebuilt — + # digest-stable, so leaf caches keep hitting; this subsumes the old + # whole-set FROZEN sentinel with a per-target judgment. rebuild_bases + # / force_rebuild override it (CVE refreshes — upstream base drift is + # invisible to the input hash). The CVE gate scans whatever :TAG + # points to, carried or fresh (rule 15). + BUILD=(); LBL=() for d in "${DEPS[@]}"; do h=$(awk -F'\t' -v t="$d" '$1==t{print $2}' <<< "$HASHES") [ -n "$h" ] || { echo "::error::no input-hash for $d"; exit 1; } - LBL+=(--set "${d}.labels.eval.input-hash=${h}") + ref="${REGISTRY}/$(awk -F'|' -v t="$d" '$1==t{print $2}' <<< "$GRAPH" | sed 's|^containers/||')" + if [ "${{ inputs.rebuild_bases }}" != "true" ] && [ "${{ inputs.force_rebuild }}" != "true" ] && \ + [ "${{ inputs.dry_run }}" != "true" ] && \ + bash containers/scripts/fleet-status.sh check "${ref}:latest-${ARCH}" "$h" >/dev/null; then + if [ "$TAG" = "latest-${ARCH}" ]; then echo "fresh: ${ref}:${TAG} (unchanged; kept)" + else + docker buildx imagetools create --tag "${ref}:${TAG}" "${ref}:latest-${ARCH}" + echo "carried forward: ${ref}:${TAG} <- :latest-${ARCH}" + fi + continue + fi + BUILD+=("$d"); LBL+=(--set "${d}.labels.eval.input-hash=${h}") done - ACT=(--provenance=mode=max --sbom=true "${DEPS[@]}" --push) - [ "${{ inputs.dry_run }}" = "true" ] && ACT=("${DEPS[@]}" --print) + [ "${#BUILD[@]}" -gt 0 ] || { echo "All ${#DEPS[@]} bases fresh — nothing to build."; exit 0; } + echo "Building ${#BUILD[@]}/${#DEPS[@]} shared deps (${ARCH}): ${BUILD[*]}" + ACT=(--provenance=mode=max --sbom=true "${BUILD[@]}" --push) + [ "${{ inputs.dry_run }}" = "true" ] && ACT=("${BUILD[@]}" --print) docker buildx bake "${FARGS[@]}" "${LBL[@]}" \ --set "*.args.REGISTRY=${REGISTRY}" \ --set "*.labels.org.opencontainers.image.source=https://github.com/Exgentic/eval-containers" \ @@ -332,10 +347,22 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} # pragma: allowlist secret run: | export TAG="${TAG}-${ARCH}" # per-arch tag; merge builds the :TAG manifest list - # Incremental (skip_published): if this leaf is already at :TAG-arch, skip it. - if [ "${{ inputs.skip_published }}" = "true" ] && \ - docker buildx imagetools inspect "${REGISTRY}/${T%%-*}s/${T#*-}:${TAG}" >/dev/null 2>&1; then - echo "skip-published: $T already at :$TAG"; exit 0 + # Stamp + carry forward (delivery/RULES.md rules 12–13): the hash is + # the repo-computed inputs at this commit; the ref comes from the + # graph's context column (dot-safe — target names mangle gpt-5.4). + # Unchanged inputs vs the prior release (:latest-) ⇒ retag its + # digest instead of rebuilding; force_rebuild overrides (CVE refresh). + H=$(bash containers/scripts/fleet-hash.sh | awk -F'\t' -v t="$T" '$1==t{print $2}') + [ -n "$H" ] || { echo "::error::no input-hash for $T"; exit 1; } + ref="${REGISTRY}/$(bash containers/scripts/fleet-hash.sh graph | awk -F'|' -v t="$T" '$1==t{print $2}' | sed 's|^containers/||')" + if [ "${{ inputs.force_rebuild }}" != "true" ] && [ "${{ inputs.dry_run }}" != "true" ] && \ + bash containers/scripts/fleet-status.sh check "${ref}:latest-${ARCH}" "$H" >/dev/null; then + if [ "$TAG" = "latest-${ARCH}" ]; then echo "fresh: ${ref}:${TAG} (unchanged; kept)" + else + docker buildx imagetools create --tag "${ref}:${TAG}" "${ref}:latest-${ARCH}" + echo "carried forward: ${ref}:${TAG} <- :latest-${ARCH}" + fi + exit 0 fi mapfile -t FILES < <(find containers -name docker-bake.hcl -not -name 'combination*') FARGS=(); for f in "${FILES[@]}"; do FARGS+=(-f "$f"); done @@ -357,13 +384,6 @@ jobs: echo "::warning::build attempt $n/$max failed; backing off"; sleep $(( n*30 + (RANDOM % 30) )) done } - # Stamp the leaf's build-input hash (delivery/RULES.md rule 12). The - # label records the repo-computed inputs at this commit; on a frozen- - # bases dev dispatch the leaf may build FROM an older frozen base than - # the hash's base component claims — tagged releases always rebuild - # bases first, so release labels are exact. - H=$(bash containers/scripts/fleet-hash.sh | awk -F'\t' -v t="$T" '$1==t{print $2}') - [ -n "$H" ] || { echo "::error::no input-hash for $T"; exit 1; } # dry_run validates config only (bake --print); else build + attest + push. ACT=(--provenance=mode=max --sbom=true "$T" --push) [ "${{ inputs.dry_run }}" = "true" ] && ACT=("$T" --print) @@ -435,13 +455,24 @@ jobs: tid=$(printf '%s' "$TASK" | tr '[:upper:]' '[:lower:]') ref="${REGISTRY}/benchmarks/${B}-${tid}:${TAG}" cache="${REGISTRY}/buildcache/${B}-${tid}-${ARCH}" - if [ "${{ inputs.skip_published }}" = "true" ] && docker buildx imagetools inspect "$ref" >/dev/null 2>&1; then echo "skip-published: $ref"; continue; fi if [ "$DRY" = "true" ]; then echo "dry-run: $ref ($KIND)"; continue; fi # Per-task build-input hash (delivery/RULES.md rule 12): the raw # task id is part of the preimage; build.sh stamps it via # EVAL_INPUT_HASH, bake via a per-target label --set. H=$(bash containers/scripts/fleet-hash.sh per-task "$B" "$TASK" | cut -f2) [ -n "$H" ] || { echo "::error::no input-hash for $B/$TASK"; fails=$((fails+1)); continue; } + # Carried-forward (rule 13): unchanged inputs vs the prior release + # ⇒ retag its digest; force_rebuild overrides (CVE refresh). + if [ "${{ inputs.force_rebuild }}" != "true" ] && \ + bash containers/scripts/fleet-status.sh check "${REGISTRY}/benchmarks/${B}-${tid}:latest-${ARCH}" "$H" >/dev/null; then + if [ "$TAG" = "latest-${ARCH}" ]; then echo "fresh: $ref (unchanged; kept)" + elif docker buildx imagetools create --tag "$ref" "${REGISTRY}/benchmarks/${B}-${tid}:latest-${ARCH}"; then + echo "carried forward: $ref <- :latest-${ARCH}" + else + echo "::error::retag failed: $ref"; fails=$((fails+1)) + fi + continue + fi echo "::group::$ref ($KIND)" if [ "$KIND" = "script" ]; then # It builds native (per-arch); push only if the built arch matches this @@ -630,16 +661,33 @@ jobs: printf '%s' "$base_raw" | grep -q '"arm64"' && PLAT="${PLAT:+$PLAT,}linux/arm64" PLAT="${PLAT:-linux/amd64}" # fall back to amd64 if inspect is unreadable fi - if [ "${{ inputs.skip_published }}" = "true" ] && docker buildx imagetools inspect "${REGISTRY}/evals/${eb}--${A}:${TAG}" >/dev/null 2>&1; then echo "skip-published: evals/${eb}--${A}"; continue; fi # Combo build-input hashes (delivery/RULES.md rule 12) — per-target, # never `*.labels`: eval and eval-standalone hash differently. HROWS=$(bash containers/scripts/fleet-hash.sh combo "$B" "$A" ${TASK:+"$TASK"}) \ || { echo "::error::no input-hash for evals/${eb}--${A}"; fails=$((fails+1)); continue; } EH=$(sed -n 1p <<< "$HROWS" | cut -f2); SH=$(sed -n 2p <<< "$HROWS" | cut -f2) + # Carried-forward (rule 13), per variant: unchanged inputs vs the + # prior release (:latest — combos push manifest lists directly) + # ⇒ retag its digest; force_rebuild overrides (CVE refresh). The + # variants judge independently: a stale standalone can rebuild + # while the lean combo carries forward. + carry() { # $1=name-suffix ("" | -standalone) $2=expected hash + [ "${{ inputs.force_rebuild }}" != "true" ] && [ "$DRY" != "true" ] || return 1 + bash containers/scripts/fleet-status.sh check "${REGISTRY}/evals/${eb}--${A}$1:latest" "$2" >/dev/null || return 1 + if [ "$TAG" = "latest" ]; then echo "fresh: evals/${eb}--${A}$1:${TAG} (unchanged; kept)" + else + docker buildx imagetools create --tag "${REGISTRY}/evals/${eb}--${A}$1:${TAG}" \ + "${REGISTRY}/evals/${eb}--${A}$1:latest" || return 1 + echo "carried forward: evals/${eb}--${A}$1:${TAG} <- :latest" + fi + } # eval = lean base (sidecar mode); eval-standalone = single-container # bundle (gateway+otelcol+process-compose in-image). bake builds eval # once, then layers standalone on it via the eval-base context. - TGT=(eval); [ "$STANDALONE" != "false" ] && TGT+=(eval-standalone) + TGT=() + carry "" "$EH" || TGT+=(eval) + [ "$STANDALONE" = "false" ] || carry "-standalone" "$SH" || TGT+=(eval-standalone) + [ "${#TGT[@]}" -gt 0 ] || continue ACT=("${TGT[@]}" --provenance=mode=max --sbom=true --push) [ "$DRY" = "true" ] && ACT=("${TGT[@]}" --print) echo "::group::evals/${eb}--${A}" diff --git a/containers/scripts/fleet-status.sh b/containers/scripts/fleet-status.sh index 9839d009..4865ec33 100644 --- a/containers/scripts/fleet-status.sh +++ b/containers/scripts/fleet-status.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# fleet-status — compare every fleet image's recorded build-input hash against -# the repository's computed hash (delivery/RULES.md rules 13–14). +# fleet-status — compare fleet images' recorded build-input hashes against +# the repository's computed hashes (delivery/RULES.md rules 13–14). # # For each static bake target: the registry ref is the graph's context column # minus `containers/` (exact for every target, including dotted model dirs @@ -16,23 +16,23 @@ # # Anything non-fresh MUST be rebuilt or retagged by the next release. # -# Usage: fleet-status.sh [tag] (default: latest) +# Usage: +# fleet-status.sh [tag] # full-fleet report (default: latest) +# fleet-status.sh check # one ref: prints the verdict; +# # exit 0 = fresh, 1 = not fresh +# The `check` form is the release workflow's retag decision (rule 13) — the +# read logic lives only here. # Output (TSV): ref verdict computed-hash recorded-hash # Env: REGISTRY (default ghcr.io/exgentic), REF (default HEAD), # STATUS_JOBS (parallel inspects, default 8) -# Exit: 0 always when the sweep completes — freshness is a report, not a gate. +# Exit (report form): 0 when the sweep completes — a report, not a gate. set -euo pipefail -TAG="${1:-latest}" REGISTRY="${REGISTRY:-ghcr.io/exgentic}" HERE="$(cd "$(dirname "$0")" && pwd)" command -v jq >/dev/null || { echo "fleet-status: jq not found" >&2; exit 2; } -# One fleet-hash run gives both the ref map (graph) and the expected hashes. -GRAPH=$("$HERE/fleet-hash.sh" graph) -ALL=$("$HERE/fleet-hash.sh") - check_one() { local ref=$1 want=$2 img got if ! img=$(docker buildx imagetools inspect "$ref" --format '{{json .Image}}' 2>/dev/null); then @@ -51,6 +51,21 @@ check_one() { } export -f check_one +if [ "${1:-}" = "check" ]; then + { [ $# -eq 3 ] && [ -n "$2" ] && [ -n "$3" ]; } \ + || { echo "fleet-status: usage: fleet-status.sh check " >&2; exit 2; } + out=$(check_one "$2" "$3") + printf '%s\n' "$out" + [ "$(cut -f2 <<< "$out")" = "fresh" ] + exit +fi + +TAG="${1:-latest}" + +# One fleet-hash run gives both the ref map (graph) and the expected hashes. +GRAPH=$("$HERE/fleet-hash.sh" graph) +ALL=$("$HERE/fleet-hash.sh") + # target|context|deps ⋈ targethash… → " " pairs, # fanned out over STATUS_JOBS parallel inspects. # shellcheck disable=SC2016 # $1/$2 belong to the xargs-spawned bash, not this shell diff --git a/docs/README.md b/docs/README.md index 05f06b8f..c028c6bc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -24,6 +24,7 @@ way on a laptop, in CI, or on a Kubernetes cluster. - [Triple-mode](concepts/triple-mode.md) — the same eval as container / compose / k8s job - [Isolation & gateways](concepts/isolation-and-gateways.md) — how trajectories stay honest - [The Helm chart](concepts/the-helm-chart.md) — one chart, `--set benchmark=` to select; optional per-benchmark preset +- [Release freshness](concepts/release-freshness.md) — input-hash labels, the fleet-status report, and why a release retags unchanged images ## Guides — *how to do a task* diff --git a/docs/concepts/release-freshness.md b/docs/concepts/release-freshness.md new file mode 100644 index 00000000..40510c85 --- /dev/null +++ b/docs/concepts/release-freshness.md @@ -0,0 +1,45 @@ +# Release freshness + +*Concept · for operators · derives from [`.agents/delivery/RULES.md`](../../.agents/delivery/RULES.md) rules 11–15, [`.agents/RULES.md`](../../.agents/RULES.md) principle 9.* + +Every fleet image carries a label, `eval.input-hash`, recording a hash of its +build inputs: the git tree of its build context and of every in-repo base it +builds on, plus (for per-task images) the task id. The hash is a pure function +of the repository at a commit — computable offline, identically, by anyone: + +```bash +containers/scripts/fleet-hash.sh # every image's expected hash +containers/scripts/fleet-status.sh v0.2.0 # compare a published tag +``` + +`fleet-status` classifies every image at a tag as **fresh** (recorded hash +matches the repo), **stale**, **unlabeled**, or **absent** — everything +non-fresh counts as *changed*, and absent/unreadable fails dirty rather than +fresh. The dispatchable **Fleet status** workflow runs the same sweep against +any tag and summarizes the fleet's freshness in one page. + +## What a version tag means + +Image builds are not bit-reproducible (package resolution moves under +identical inputs), so rebuilding an *unchanged* image would silently ship +different bits under the new version. The release therefore does the safer +thing: **an image whose inputs are unchanged from the prior release is +retagged from that release's digest — same bits, new tag — instead of being +rebuilt.** A version tag pins a *coherent, tested set of inputs*, not a build +timestamp; two consecutive versions may share digests for images whose inputs +did not change, and each image's SLSA provenance honestly names the run that +actually built it. + +Only images work this way. Artifacts that embed the version in their own bytes +— the per-benchmark `eval-` compose artifacts, the Helm chart, the +CLI — are republished fresh on every release by definition. + +## Forcing a rebuild + +The input hash sees the repository, not the outside world: an upstream base +image (`python:3.12-slim`) or unpinned package moving does not change any +input. To pick up upstream fixes, dispatch **Release the fleet** with +`force_rebuild: true` (or `rebuild_bases: true` for the shared bases alone) — +principle 9 classes such CVE/base refreshes as a patch release. The CVE gate +scans whatever the release tag points to, carried-forward or freshly built, so +a stale-but-carried base cannot slip through a gated release unscanned. From 7bbac86d7eb958388f14df230bc84b527eb9cdfc Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Sun, 9 Aug 2026 18:44:08 +0300 Subject: [PATCH 2/2] polish: retire stale FROZEN prose, hoist carry() out of the combo loop Signed-off-by: Elron Bandel --- .github/.secrets.baseline | 4 +- .github/workflows/release-images.yml | 67 +++++++++++++++------------- 2 files changed, 38 insertions(+), 33 deletions(-) diff --git a/.github/.secrets.baseline b/.github/.secrets.baseline index 1e579f71..ecdb6ba5 100644 --- a/.github/.secrets.baseline +++ b/.github/.secrets.baseline @@ -145,7 +145,7 @@ "filename": ".github/workflows/release-images.yml", "hashed_secret": "6e0da5f85a202cf018708adc9db4b5c04ac093e6", "is_verified": false, - "line_number": 151 + "line_number": 153 } ], ".github/workflows/release.yml": [ @@ -194,5 +194,5 @@ } ] }, - "generated_at": "2026-06-29T11:05:34Z" + "generated_at": "2026-08-09T15:44:02Z" } diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index e5be4ab9..21f9e5be 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -5,9 +5,11 @@ name: Release the fleet # fleet-publish-dev + combos-claude-code + publish-per-task) into a single # pipeline that uses every GitHub-Actions-NATIVE textbook speedup: # -# * FROZEN BASE — shared deps (core + gateways) build ONCE; leaves pull -# the frozen :latest by overriding the bake context to -# docker-image:// (stable digest -> leaf caches hit). +# * CARRIED-FORWARD — every image whose build inputs are unchanged from the +# prior release is retagged from its digest, not rebuilt +# (delivery/RULES.md rule 13); shared bases build ONCE and +# leaves pull them from the registry via docker-image:// +# contexts (stable digest -> leaf caches hit). # * PER-LEAF MATRIX — one job per leaf (not 10-leaf groups), so the wall # clock is the slowest SINGLE benchmark, not the slowest # group, and one leaf's failure can't sink its neighbours @@ -45,7 +47,7 @@ on: description: "Version to publish (e.g. v0.1.0); blank = build at :latest" default: "" rebuild_bases: - description: "Rebuild shared bases (else reuse the frozen :latest digest)" + description: "Rebuild shared bases even when their input-hashes are fresh" type: boolean default: false combo_agents: @@ -118,7 +120,7 @@ jobs: name: cli path: target/release/eval-containers - # ── frozen base: shared deps (core + gateways) build ONCE, per-arch ─────── + # ── shared bases (core + gateways): build stale, carry forward fresh ────── # Native-per-arch: amd64 on a standard runner, arm64 on ubuntu-24.04-arm # (set vars.FLEET_RUNNER_ARM to override). Each arch pushes :TAG-; the # `merge` job stitches them into the :TAG manifest list. No QEMU — the heavy @@ -144,7 +146,7 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build bases (${{ matrix.arch }}) only if missing or forced (else freeze) + - name: Bases (${{ matrix.arch }}) — build stale, carry forward fresh env: HF_TOKEN: ${{ secrets.HF_TOKEN }} # pragma: allowlist secret ARCH: ${{ matrix.arch }} @@ -339,7 +341,7 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Bake ${{ matrix.target }} (${{ matrix.arch }}; frozen base, per-leaf cache) + - name: Bake ${{ matrix.target }} (${{ matrix.arch }}; registry bases, per-leaf cache) timeout-minutes: 25 # cap a stalled leaf — a hung download blocked the matrix ~40m; leaves build in <2m env: T: ${{ matrix.target }} @@ -366,8 +368,8 @@ jobs: fi mapfile -t FILES < <(find containers -name docker-bake.hcl -not -name 'combination*') FARGS=(); for f in "${FILES[@]}"; do FARGS+=(-f "$f"); done - # Override every base context to PULL the frozen registry image, so this - # leaf builds only its own layers FROM the frozen base (never rebuilds it). + # Override every base context to PULL the published registry image, so + # this leaf builds only its own layers FROM it (never rebuilds a base). OV=() for d in $(find containers/core containers/gateways -mindepth 1 -maxdepth 1 -type d | sed -E 's#containers/##'); do [ -f "containers/$d/Dockerfile" ] || continue @@ -487,7 +489,7 @@ jobs: fi else echo "::error::per-task build failed: $ref"; fails=$((fails+1)); fi else - # bake: FROM the Epoch per-task base + EVAL_TASK_ID; pull the frozen entrypoint. + # bake: FROM the Epoch per-task base + EVAL_TASK_ID; pull the published entrypoint. if retry docker buildx bake -f containers/docker-bake.hcl -f "containers/benchmarks/$B/docker-bake.hcl" \ --set "benchmark-${B}.labels.eval.input-hash=${H}" \ --set "benchmark-${B}.args.EVAL_TASK_ID=${TASK}" \ @@ -642,6 +644,20 @@ jobs: echo "::warning::attempt $n/$max failed; backing off"; sleep $(( n*30 + (RANDOM % 30) )) done } + # Carried-forward (rule 13), per variant: unchanged inputs vs the + # prior release (:latest — combos push manifest lists directly) + # ⇒ retag its digest; force_rebuild overrides (CVE refresh). Reads + # the per-iteration eb/A/TAG/DRY at call time. + carry() { # $1=name-suffix ("" | -standalone) $2=expected hash + [ "${{ inputs.force_rebuild }}" != "true" ] && [ "$DRY" != "true" ] || return 1 + bash containers/scripts/fleet-status.sh check "${REGISTRY}/evals/${eb}--${A}$1:latest" "$2" >/dev/null || return 1 + if [ "$TAG" = "latest" ]; then echo "fresh: evals/${eb}--${A}$1:${TAG} (unchanged; kept)" + else + docker buildx imagetools create --tag "${REGISTRY}/evals/${eb}--${A}$1:${TAG}" \ + "${REGISTRY}/evals/${eb}--${A}$1:latest" || return 1 + echo "carried forward: evals/${eb}--${A}$1:${TAG} <- :latest" + fi + } fails=0 while read -r it; do IFS=$'\t' read -r B A TASK < <(jq -r '[.b,.a,.task]|@tsv' <<< "$it") @@ -666,24 +682,11 @@ jobs: HROWS=$(bash containers/scripts/fleet-hash.sh combo "$B" "$A" ${TASK:+"$TASK"}) \ || { echo "::error::no input-hash for evals/${eb}--${A}"; fails=$((fails+1)); continue; } EH=$(sed -n 1p <<< "$HROWS" | cut -f2); SH=$(sed -n 2p <<< "$HROWS" | cut -f2) - # Carried-forward (rule 13), per variant: unchanged inputs vs the - # prior release (:latest — combos push manifest lists directly) - # ⇒ retag its digest; force_rebuild overrides (CVE refresh). The - # variants judge independently: a stale standalone can rebuild - # while the lean combo carries forward. - carry() { # $1=name-suffix ("" | -standalone) $2=expected hash - [ "${{ inputs.force_rebuild }}" != "true" ] && [ "$DRY" != "true" ] || return 1 - bash containers/scripts/fleet-status.sh check "${REGISTRY}/evals/${eb}--${A}$1:latest" "$2" >/dev/null || return 1 - if [ "$TAG" = "latest" ]; then echo "fresh: evals/${eb}--${A}$1:${TAG} (unchanged; kept)" - else - docker buildx imagetools create --tag "${REGISTRY}/evals/${eb}--${A}$1:${TAG}" \ - "${REGISTRY}/evals/${eb}--${A}$1:latest" || return 1 - echo "carried forward: evals/${eb}--${A}$1:${TAG} <- :latest" - fi - } # eval = lean base (sidecar mode); eval-standalone = single-container # bundle (gateway+otelcol+process-compose in-image). bake builds eval - # once, then layers standalone on it via the eval-base context. + # once, then layers standalone on it via the eval-base context. The + # variants judge freshness independently: a stale standalone can + # rebuild while the lean combo carries forward. TGT=() carry "" "$EH" || TGT+=(eval) [ "$STANDALONE" = "false" ] || carry "-standalone" "$SH" || TGT+=(eval-standalone) @@ -705,10 +708,12 @@ jobs: # ── CVE gate on the shared bases, then promote :TAG -> :latest ──────────── # Runs for real releases (tag / explicit version) or whenever the bases were - # rebuilt — i.e. when there's something new to gate + promote. A frozen-base - # dev dispatch re-scanning unchanged bases would only surface CVEs disclosed - # since they were last gated at release, so it's skipped (the dev run isn't - # publishing :latest anyway — the promotion below is tag-only). + # force-rebuilt. The scan covers whatever :TAG points to — carried-forward + # digests included (rule 15) — so a base carried from the prior release is + # re-gated at every release; a failure there means the base accumulated CVEs + # since it was built, and the fix is a force_rebuild/rebuild_bases refresh. + # A dev dispatch skips the gate (it isn't publishing :latest — the promotion + # below is tag-only). release-gate: needs: [merge, per-task, compose, combos] if: always() && needs.merge.result == 'success' && !inputs.dry_run && (github.ref_type == 'tag' || inputs.tag != '' || inputs.rebuild_bases) @@ -831,7 +836,7 @@ jobs: echo "| CVE gate + :latest | ${{ needs.release-gate.result }} |" echo "" echo "## Per-image build time + registry size — leaves, biggest first" - echo "_build = the leaf's own time FROM the frozen base; bases add ${basedur:-?}s, amortized once across all leaves._" + echo "_build = the leaf's own time FROM the published base; bases add ${basedur:-?}s, amortized once across all leaves._" echo "| image | build | size |"; echo "|---|--:|--:|" sort -t$'\t' -k4 -rn /tmp/sizes.tsv \ | awk -F'\t' '{printf "| %s | %ds | %d MB |\n", $1, $3, $4/1048576}'