diff --git a/.github/.secrets.baseline b/.github/.secrets.baseline index 1e579f71..ecdb6ba5 100644 --- a/.github/.secrets.baseline +++ b/.github/.secrets.baseline @@ -145,7 +145,7 @@ "filename": ".github/workflows/release-images.yml", "hashed_secret": "6e0da5f85a202cf018708adc9db4b5c04ac093e6", "is_verified": false, - "line_number": 151 + "line_number": 153 } ], ".github/workflows/release.yml": [ @@ -194,5 +194,5 @@ } ] }, - "generated_at": "2026-06-29T11:05:34Z" + "generated_at": "2026-08-09T15:44:02Z" } diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index c65f7a36..21f9e5be 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -5,9 +5,11 @@ name: Release the fleet # fleet-publish-dev + combos-claude-code + publish-per-task) into a single # pipeline that uses every GitHub-Actions-NATIVE textbook speedup: # -# * FROZEN BASE — shared deps (core + gateways) build ONCE; leaves pull -# the frozen :latest by overriding the bake context to -# docker-image:// (stable digest -> leaf caches hit). +# * CARRIED-FORWARD — every image whose build inputs are unchanged from the +# prior release is retagged from its digest, not rebuilt +# (delivery/RULES.md rule 13); shared bases build ONCE and +# leaves pull them from the registry via docker-image:// +# contexts (stable digest -> leaf caches hit). # * PER-LEAF MATRIX — one job per leaf (not 10-leaf groups), so the wall # clock is the slowest SINGLE benchmark, not the slowest # group, and one leaf's failure can't sink its neighbours @@ -45,7 +47,7 @@ on: description: "Version to publish (e.g. v0.1.0); blank = build at :latest" default: "" rebuild_bases: - description: "Rebuild shared bases (else reuse the frozen :latest digest)" + description: "Rebuild shared bases even when their input-hashes are fresh" type: boolean default: false combo_agents: @@ -69,8 +71,8 @@ on: only: description: "Debug: restrict matrices to these space-separated leaf targets (e.g. 'benchmark-gsm8k agent-claude-code'); blank = whole fleet" default: "" - skip_published: - description: "Incremental: skip building any image already at :TAG (build only missing/failed). Use when re-running and the published images are still current." + force_rebuild: + description: "Rebuild every image even when its recorded input-hash is fresh (use for CVE/base refreshes — upstream base drift is invisible to the input hash)" type: boolean default: false @@ -118,7 +120,7 @@ jobs: name: cli path: target/release/eval-containers - # ── frozen base: shared deps (core + gateways) build ONCE, per-arch ─────── + # ── shared bases (core + gateways): build stale, carry forward fresh ────── # Native-per-arch: amd64 on a standard runner, arm64 on ubuntu-24.04-arm # (set vars.FLEET_RUNNER_ARM to override). Each arch pushes :TAG-; the # `merge` job stitches them into the :TAG manifest list. No QEMU — the heavy @@ -144,35 +146,50 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build bases (${{ matrix.arch }}) only if missing or forced (else freeze) + - name: Bases (${{ matrix.arch }}) — build stale, carry forward fresh env: HF_TOKEN: ${{ secrets.HF_TOKEN }} # pragma: allowlist secret ARCH: ${{ matrix.arch }} run: | export TAG="${TAG}-${ARCH}" # per-arch tag; merge builds the :TAG manifest list - if [ "${{ inputs.rebuild_bases }}" != "true" ] && \ - docker buildx imagetools inspect "${REGISTRY}/core/entrypoint:${TAG}" >/dev/null 2>&1; then - echo "Bases present at :${TAG} — FROZEN (skip; stable digest → leaf caches hit)." - exit 0 - fi mapfile -t FILES < <(find containers -name docker-bake.hcl -not -name 'combination*') FARGS=(); for f in "${FILES[@]}"; do FARGS+=(-f "$f"); done mapfile -t DEPS < <(find containers/core containers/gateways -name docker-bake.hcl \ -exec grep -hoE '^target "[^"]+"' {} + | sed -E 's/^target "([^"]+)"/\1/' | sort -u) - echo "Building ${#DEPS[@]} shared deps (${ARCH}): ${DEPS[*]}" # Stamp each base's build-input hash (delivery/RULES.md rule 12) — # per-target, never `*.labels`: this one invocation bakes ~19 targets # with ~19 different hashes. The label records the repo-computed # inputs at this commit; arch-independent by construction. HASHES=$(bash containers/scripts/fleet-hash.sh) - LBL=() + GRAPH=$(bash containers/scripts/fleet-hash.sh graph) + # Carried-forward (rule 13): a base whose inputs are unchanged from + # the prior release (:latest-) is retagged, never rebuilt — + # digest-stable, so leaf caches keep hitting; this subsumes the old + # whole-set FROZEN sentinel with a per-target judgment. rebuild_bases + # / force_rebuild override it (CVE refreshes — upstream base drift is + # invisible to the input hash). The CVE gate scans whatever :TAG + # points to, carried or fresh (rule 15). + BUILD=(); LBL=() for d in "${DEPS[@]}"; do h=$(awk -F'\t' -v t="$d" '$1==t{print $2}' <<< "$HASHES") [ -n "$h" ] || { echo "::error::no input-hash for $d"; exit 1; } - LBL+=(--set "${d}.labels.eval.input-hash=${h}") + ref="${REGISTRY}/$(awk -F'|' -v t="$d" '$1==t{print $2}' <<< "$GRAPH" | sed 's|^containers/||')" + if [ "${{ inputs.rebuild_bases }}" != "true" ] && [ "${{ inputs.force_rebuild }}" != "true" ] && \ + [ "${{ inputs.dry_run }}" != "true" ] && \ + bash containers/scripts/fleet-status.sh check "${ref}:latest-${ARCH}" "$h" >/dev/null; then + if [ "$TAG" = "latest-${ARCH}" ]; then echo "fresh: ${ref}:${TAG} (unchanged; kept)" + else + docker buildx imagetools create --tag "${ref}:${TAG}" "${ref}:latest-${ARCH}" + echo "carried forward: ${ref}:${TAG} <- :latest-${ARCH}" + fi + continue + fi + BUILD+=("$d"); LBL+=(--set "${d}.labels.eval.input-hash=${h}") done - ACT=(--provenance=mode=max --sbom=true "${DEPS[@]}" --push) - [ "${{ inputs.dry_run }}" = "true" ] && ACT=("${DEPS[@]}" --print) + [ "${#BUILD[@]}" -gt 0 ] || { echo "All ${#DEPS[@]} bases fresh — nothing to build."; exit 0; } + echo "Building ${#BUILD[@]}/${#DEPS[@]} shared deps (${ARCH}): ${BUILD[*]}" + ACT=(--provenance=mode=max --sbom=true "${BUILD[@]}" --push) + [ "${{ inputs.dry_run }}" = "true" ] && ACT=("${BUILD[@]}" --print) docker buildx bake "${FARGS[@]}" "${LBL[@]}" \ --set "*.args.REGISTRY=${REGISTRY}" \ --set "*.labels.org.opencontainers.image.source=https://github.com/Exgentic/eval-containers" \ @@ -324,7 +341,7 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Bake ${{ matrix.target }} (${{ matrix.arch }}; frozen base, per-leaf cache) + - name: Bake ${{ matrix.target }} (${{ matrix.arch }}; registry bases, per-leaf cache) timeout-minutes: 25 # cap a stalled leaf — a hung download blocked the matrix ~40m; leaves build in <2m env: T: ${{ matrix.target }} @@ -332,15 +349,27 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} # pragma: allowlist secret run: | export TAG="${TAG}-${ARCH}" # per-arch tag; merge builds the :TAG manifest list - # Incremental (skip_published): if this leaf is already at :TAG-arch, skip it. - if [ "${{ inputs.skip_published }}" = "true" ] && \ - docker buildx imagetools inspect "${REGISTRY}/${T%%-*}s/${T#*-}:${TAG}" >/dev/null 2>&1; then - echo "skip-published: $T already at :$TAG"; exit 0 + # Stamp + carry forward (delivery/RULES.md rules 12–13): the hash is + # the repo-computed inputs at this commit; the ref comes from the + # graph's context column (dot-safe — target names mangle gpt-5.4). + # Unchanged inputs vs the prior release (:latest-) ⇒ retag its + # digest instead of rebuilding; force_rebuild overrides (CVE refresh). + H=$(bash containers/scripts/fleet-hash.sh | awk -F'\t' -v t="$T" '$1==t{print $2}') + [ -n "$H" ] || { echo "::error::no input-hash for $T"; exit 1; } + ref="${REGISTRY}/$(bash containers/scripts/fleet-hash.sh graph | awk -F'|' -v t="$T" '$1==t{print $2}' | sed 's|^containers/||')" + if [ "${{ inputs.force_rebuild }}" != "true" ] && [ "${{ inputs.dry_run }}" != "true" ] && \ + bash containers/scripts/fleet-status.sh check "${ref}:latest-${ARCH}" "$H" >/dev/null; then + if [ "$TAG" = "latest-${ARCH}" ]; then echo "fresh: ${ref}:${TAG} (unchanged; kept)" + else + docker buildx imagetools create --tag "${ref}:${TAG}" "${ref}:latest-${ARCH}" + echo "carried forward: ${ref}:${TAG} <- :latest-${ARCH}" + fi + exit 0 fi mapfile -t FILES < <(find containers -name docker-bake.hcl -not -name 'combination*') FARGS=(); for f in "${FILES[@]}"; do FARGS+=(-f "$f"); done - # Override every base context to PULL the frozen registry image, so this - # leaf builds only its own layers FROM the frozen base (never rebuilds it). + # Override every base context to PULL the published registry image, so + # this leaf builds only its own layers FROM it (never rebuilds a base). OV=() for d in $(find containers/core containers/gateways -mindepth 1 -maxdepth 1 -type d | sed -E 's#containers/##'); do [ -f "containers/$d/Dockerfile" ] || continue @@ -357,13 +386,6 @@ jobs: echo "::warning::build attempt $n/$max failed; backing off"; sleep $(( n*30 + (RANDOM % 30) )) done } - # Stamp the leaf's build-input hash (delivery/RULES.md rule 12). The - # label records the repo-computed inputs at this commit; on a frozen- - # bases dev dispatch the leaf may build FROM an older frozen base than - # the hash's base component claims — tagged releases always rebuild - # bases first, so release labels are exact. - H=$(bash containers/scripts/fleet-hash.sh | awk -F'\t' -v t="$T" '$1==t{print $2}') - [ -n "$H" ] || { echo "::error::no input-hash for $T"; exit 1; } # dry_run validates config only (bake --print); else build + attest + push. ACT=(--provenance=mode=max --sbom=true "$T" --push) [ "${{ inputs.dry_run }}" = "true" ] && ACT=("$T" --print) @@ -435,13 +457,24 @@ jobs: tid=$(printf '%s' "$TASK" | tr '[:upper:]' '[:lower:]') ref="${REGISTRY}/benchmarks/${B}-${tid}:${TAG}" cache="${REGISTRY}/buildcache/${B}-${tid}-${ARCH}" - if [ "${{ inputs.skip_published }}" = "true" ] && docker buildx imagetools inspect "$ref" >/dev/null 2>&1; then echo "skip-published: $ref"; continue; fi if [ "$DRY" = "true" ]; then echo "dry-run: $ref ($KIND)"; continue; fi # Per-task build-input hash (delivery/RULES.md rule 12): the raw # task id is part of the preimage; build.sh stamps it via # EVAL_INPUT_HASH, bake via a per-target label --set. H=$(bash containers/scripts/fleet-hash.sh per-task "$B" "$TASK" | cut -f2) [ -n "$H" ] || { echo "::error::no input-hash for $B/$TASK"; fails=$((fails+1)); continue; } + # Carried-forward (rule 13): unchanged inputs vs the prior release + # ⇒ retag its digest; force_rebuild overrides (CVE refresh). + if [ "${{ inputs.force_rebuild }}" != "true" ] && \ + bash containers/scripts/fleet-status.sh check "${REGISTRY}/benchmarks/${B}-${tid}:latest-${ARCH}" "$H" >/dev/null; then + if [ "$TAG" = "latest-${ARCH}" ]; then echo "fresh: $ref (unchanged; kept)" + elif docker buildx imagetools create --tag "$ref" "${REGISTRY}/benchmarks/${B}-${tid}:latest-${ARCH}"; then + echo "carried forward: $ref <- :latest-${ARCH}" + else + echo "::error::retag failed: $ref"; fails=$((fails+1)) + fi + continue + fi echo "::group::$ref ($KIND)" if [ "$KIND" = "script" ]; then # It builds native (per-arch); push only if the built arch matches this @@ -456,7 +489,7 @@ jobs: fi else echo "::error::per-task build failed: $ref"; fails=$((fails+1)); fi else - # bake: FROM the Epoch per-task base + EVAL_TASK_ID; pull the frozen entrypoint. + # bake: FROM the Epoch per-task base + EVAL_TASK_ID; pull the published entrypoint. if retry docker buildx bake -f containers/docker-bake.hcl -f "containers/benchmarks/$B/docker-bake.hcl" \ --set "benchmark-${B}.labels.eval.input-hash=${H}" \ --set "benchmark-${B}.args.EVAL_TASK_ID=${TASK}" \ @@ -611,6 +644,20 @@ jobs: echo "::warning::attempt $n/$max failed; backing off"; sleep $(( n*30 + (RANDOM % 30) )) done } + # Carried-forward (rule 13), per variant: unchanged inputs vs the + # prior release (:latest — combos push manifest lists directly) + # ⇒ retag its digest; force_rebuild overrides (CVE refresh). Reads + # the per-iteration eb/A/TAG/DRY at call time. + carry() { # $1=name-suffix ("" | -standalone) $2=expected hash + [ "${{ inputs.force_rebuild }}" != "true" ] && [ "$DRY" != "true" ] || return 1 + bash containers/scripts/fleet-status.sh check "${REGISTRY}/evals/${eb}--${A}$1:latest" "$2" >/dev/null || return 1 + if [ "$TAG" = "latest" ]; then echo "fresh: evals/${eb}--${A}$1:${TAG} (unchanged; kept)" + else + docker buildx imagetools create --tag "${REGISTRY}/evals/${eb}--${A}$1:${TAG}" \ + "${REGISTRY}/evals/${eb}--${A}$1:latest" || return 1 + echo "carried forward: evals/${eb}--${A}$1:${TAG} <- :latest" + fi + } fails=0 while read -r it; do IFS=$'\t' read -r B A TASK < <(jq -r '[.b,.a,.task]|@tsv' <<< "$it") @@ -630,7 +677,6 @@ jobs: printf '%s' "$base_raw" | grep -q '"arm64"' && PLAT="${PLAT:+$PLAT,}linux/arm64" PLAT="${PLAT:-linux/amd64}" # fall back to amd64 if inspect is unreadable fi - if [ "${{ inputs.skip_published }}" = "true" ] && docker buildx imagetools inspect "${REGISTRY}/evals/${eb}--${A}:${TAG}" >/dev/null 2>&1; then echo "skip-published: evals/${eb}--${A}"; continue; fi # Combo build-input hashes (delivery/RULES.md rule 12) — per-target, # never `*.labels`: eval and eval-standalone hash differently. HROWS=$(bash containers/scripts/fleet-hash.sh combo "$B" "$A" ${TASK:+"$TASK"}) \ @@ -638,8 +684,13 @@ jobs: EH=$(sed -n 1p <<< "$HROWS" | cut -f2); SH=$(sed -n 2p <<< "$HROWS" | cut -f2) # eval = lean base (sidecar mode); eval-standalone = single-container # bundle (gateway+otelcol+process-compose in-image). bake builds eval - # once, then layers standalone on it via the eval-base context. - TGT=(eval); [ "$STANDALONE" != "false" ] && TGT+=(eval-standalone) + # once, then layers standalone on it via the eval-base context. The + # variants judge freshness independently: a stale standalone can + # rebuild while the lean combo carries forward. + TGT=() + carry "" "$EH" || TGT+=(eval) + [ "$STANDALONE" = "false" ] || carry "-standalone" "$SH" || TGT+=(eval-standalone) + [ "${#TGT[@]}" -gt 0 ] || continue ACT=("${TGT[@]}" --provenance=mode=max --sbom=true --push) [ "$DRY" = "true" ] && ACT=("${TGT[@]}" --print) echo "::group::evals/${eb}--${A}" @@ -657,10 +708,12 @@ jobs: # ── CVE gate on the shared bases, then promote :TAG -> :latest ──────────── # Runs for real releases (tag / explicit version) or whenever the bases were - # rebuilt — i.e. when there's something new to gate + promote. A frozen-base - # dev dispatch re-scanning unchanged bases would only surface CVEs disclosed - # since they were last gated at release, so it's skipped (the dev run isn't - # publishing :latest anyway — the promotion below is tag-only). + # force-rebuilt. The scan covers whatever :TAG points to — carried-forward + # digests included (rule 15) — so a base carried from the prior release is + # re-gated at every release; a failure there means the base accumulated CVEs + # since it was built, and the fix is a force_rebuild/rebuild_bases refresh. + # A dev dispatch skips the gate (it isn't publishing :latest — the promotion + # below is tag-only). release-gate: needs: [merge, per-task, compose, combos] if: always() && needs.merge.result == 'success' && !inputs.dry_run && (github.ref_type == 'tag' || inputs.tag != '' || inputs.rebuild_bases) @@ -783,7 +836,7 @@ jobs: echo "| CVE gate + :latest | ${{ needs.release-gate.result }} |" echo "" echo "## Per-image build time + registry size — leaves, biggest first" - echo "_build = the leaf's own time FROM the frozen base; bases add ${basedur:-?}s, amortized once across all leaves._" + echo "_build = the leaf's own time FROM the published base; bases add ${basedur:-?}s, amortized once across all leaves._" echo "| image | build | size |"; echo "|---|--:|--:|" sort -t$'\t' -k4 -rn /tmp/sizes.tsv \ | awk -F'\t' '{printf "| %s | %ds | %d MB |\n", $1, $3, $4/1048576}' diff --git a/containers/scripts/fleet-status.sh b/containers/scripts/fleet-status.sh index 9839d009..4865ec33 100644 --- a/containers/scripts/fleet-status.sh +++ b/containers/scripts/fleet-status.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# fleet-status — compare every fleet image's recorded build-input hash against -# the repository's computed hash (delivery/RULES.md rules 13–14). +# fleet-status — compare fleet images' recorded build-input hashes against +# the repository's computed hashes (delivery/RULES.md rules 13–14). # # For each static bake target: the registry ref is the graph's context column # minus `containers/` (exact for every target, including dotted model dirs @@ -16,23 +16,23 @@ # # Anything non-fresh MUST be rebuilt or retagged by the next release. # -# Usage: fleet-status.sh [tag] (default: latest) +# Usage: +# fleet-status.sh [tag] # full-fleet report (default: latest) +# fleet-status.sh check # one ref: prints the verdict; +# # exit 0 = fresh, 1 = not fresh +# The `check` form is the release workflow's retag decision (rule 13) — the +# read logic lives only here. # Output (TSV): ref verdict computed-hash recorded-hash # Env: REGISTRY (default ghcr.io/exgentic), REF (default HEAD), # STATUS_JOBS (parallel inspects, default 8) -# Exit: 0 always when the sweep completes — freshness is a report, not a gate. +# Exit (report form): 0 when the sweep completes — a report, not a gate. set -euo pipefail -TAG="${1:-latest}" REGISTRY="${REGISTRY:-ghcr.io/exgentic}" HERE="$(cd "$(dirname "$0")" && pwd)" command -v jq >/dev/null || { echo "fleet-status: jq not found" >&2; exit 2; } -# One fleet-hash run gives both the ref map (graph) and the expected hashes. -GRAPH=$("$HERE/fleet-hash.sh" graph) -ALL=$("$HERE/fleet-hash.sh") - check_one() { local ref=$1 want=$2 img got if ! img=$(docker buildx imagetools inspect "$ref" --format '{{json .Image}}' 2>/dev/null); then @@ -51,6 +51,21 @@ check_one() { } export -f check_one +if [ "${1:-}" = "check" ]; then + { [ $# -eq 3 ] && [ -n "$2" ] && [ -n "$3" ]; } \ + || { echo "fleet-status: usage: fleet-status.sh check " >&2; exit 2; } + out=$(check_one "$2" "$3") + printf '%s\n' "$out" + [ "$(cut -f2 <<< "$out")" = "fresh" ] + exit +fi + +TAG="${1:-latest}" + +# One fleet-hash run gives both the ref map (graph) and the expected hashes. +GRAPH=$("$HERE/fleet-hash.sh" graph) +ALL=$("$HERE/fleet-hash.sh") + # target|context|deps ⋈ targethash… → " " pairs, # fanned out over STATUS_JOBS parallel inspects. # shellcheck disable=SC2016 # $1/$2 belong to the xargs-spawned bash, not this shell diff --git a/docs/README.md b/docs/README.md index 05f06b8f..c028c6bc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -24,6 +24,7 @@ way on a laptop, in CI, or on a Kubernetes cluster. - [Triple-mode](concepts/triple-mode.md) — the same eval as container / compose / k8s job - [Isolation & gateways](concepts/isolation-and-gateways.md) — how trajectories stay honest - [The Helm chart](concepts/the-helm-chart.md) — one chart, `--set benchmark=` to select; optional per-benchmark preset +- [Release freshness](concepts/release-freshness.md) — input-hash labels, the fleet-status report, and why a release retags unchanged images ## Guides — *how to do a task* diff --git a/docs/concepts/release-freshness.md b/docs/concepts/release-freshness.md new file mode 100644 index 00000000..40510c85 --- /dev/null +++ b/docs/concepts/release-freshness.md @@ -0,0 +1,45 @@ +# Release freshness + +*Concept · for operators · derives from [`.agents/delivery/RULES.md`](../../.agents/delivery/RULES.md) rules 11–15, [`.agents/RULES.md`](../../.agents/RULES.md) principle 9.* + +Every fleet image carries a label, `eval.input-hash`, recording a hash of its +build inputs: the git tree of its build context and of every in-repo base it +builds on, plus (for per-task images) the task id. The hash is a pure function +of the repository at a commit — computable offline, identically, by anyone: + +```bash +containers/scripts/fleet-hash.sh # every image's expected hash +containers/scripts/fleet-status.sh v0.2.0 # compare a published tag +``` + +`fleet-status` classifies every image at a tag as **fresh** (recorded hash +matches the repo), **stale**, **unlabeled**, or **absent** — everything +non-fresh counts as *changed*, and absent/unreadable fails dirty rather than +fresh. The dispatchable **Fleet status** workflow runs the same sweep against +any tag and summarizes the fleet's freshness in one page. + +## What a version tag means + +Image builds are not bit-reproducible (package resolution moves under +identical inputs), so rebuilding an *unchanged* image would silently ship +different bits under the new version. The release therefore does the safer +thing: **an image whose inputs are unchanged from the prior release is +retagged from that release's digest — same bits, new tag — instead of being +rebuilt.** A version tag pins a *coherent, tested set of inputs*, not a build +timestamp; two consecutive versions may share digests for images whose inputs +did not change, and each image's SLSA provenance honestly names the run that +actually built it. + +Only images work this way. Artifacts that embed the version in their own bytes +— the per-benchmark `eval-` compose artifacts, the Helm chart, the +CLI — are republished fresh on every release by definition. + +## Forcing a rebuild + +The input hash sees the repository, not the outside world: an upstream base +image (`python:3.12-slim`) or unpinned package moving does not change any +input. To pick up upstream fixes, dispatch **Release the fleet** with +`force_rebuild: true` (or `rebuild_bases: true` for the shared bases alone) — +principle 9 classes such CVE/base refreshes as a patch release. The CVE gate +scans whatever the release tag points to, carried-forward or freshly built, so +a stale-but-carried base cannot slip through a gated release unscanned.