From 6e6a4ba42c5394ecefc711dd268c2eb3788c1f13 Mon Sep 17 00:00:00 2001 From: wangnov <48670012+Wangnov@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:30:42 +0800 Subject: [PATCH 1/8] chore: reserve implementation for #170 From ea6c0c95f5f9fe0c1955d178adbbeb83bfe75578 Mon Sep 17 00:00:00 2001 From: wangnov <48670012+Wangnov@users.noreply.github.com> Date: Fri, 10 Jul 2026 21:32:38 +0800 Subject: [PATCH 2/8] fix(release): make mirror promotion monotonic --- .github/workflows/release.yml | 548 ++++- cloudflare/manager-download-router/README.md | 30 +- .../manager-download-router/src/index.js | 38 +- .../test/index.test.js | 40 + docs/release.md | 114 +- package.json | 1 + scripts/check-immutable-releases.mjs | 75 + scripts/check-release-reuse.mjs | 119 ++ scripts/mirror-release.mjs | 1800 +++++++++++++++++ scripts/mirror-release.test.mjs | 1345 ++++++++++++ scripts/release-workflow.test.mjs | 164 ++ scripts/sync-mirror.sh | 245 +-- scripts/validate-release-manifest.mjs | 37 + scripts/verify-release-artifacts.mjs | 63 + scripts/write-release-summary.mjs | 95 +- 15 files changed, 4413 insertions(+), 301 deletions(-) create mode 100644 scripts/check-immutable-releases.mjs create mode 100644 scripts/check-release-reuse.mjs create mode 100755 scripts/mirror-release.mjs create mode 100644 scripts/mirror-release.test.mjs create mode 100644 scripts/release-workflow.test.mjs create mode 100644 scripts/validate-release-manifest.mjs create mode 100644 scripts/verify-release-artifacts.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 64e88d4..d4ba4e6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,23 @@ name: Release # AC_API_KEY_BASE64 base64 of the AuthKey_XXXX.p8 # TAURI_SIGNING_PRIVATE_KEY updater private key (string) # TAURI_SIGNING_PRIVATE_KEY_PASSWORD its password (empty if none) +# IMMUTABLE_RELEASES_READ_TOKEN fine-grained token with repository +# Administration: read-only +# MANAGER_R2_S3_ENDPOINT / MANAGER_R2_PROMOTION_ACCESS_KEY_ID / +# MANAGER_R2_PROMOTION_SECRET_ACCESS_KEY +# R2 S3-compatible read/write access +# MANAGER_IHEP_S3_ENDPOINT / MANAGER_IHEP_S3_BUCKET / +# MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID / +# MANAGER_IHEP_S3_PROMOTION_SECRET_ACCESS_KEY +# IHEP S3-compatible read/write access +# Both mirror backends are required for stable publication. They must already +# contain a valid latest.json baseline and support conditional PutObject +# (If-Match / If-None-Match) plus HeadObject user-metadata round trips. +# GitHub Immutable Releases must be enabled before any release starts; the +# workflow verifies the repository setting with IMMUTABLE_RELEASES_READ_TOKEN. +# Every reused asset is also pinned to the API-provided sha256 digest. +# The old MANAGER_*_ACCESS_KEY_ID / SECRET_ACCESS_KEY names must stay deleted; +# otherwise a historical workflow revision could still perform unconditional writes. # # Optional Windows Authenticode (non-blocking until configured; see docs/windows-signing.md): # WINDOWS_CERTIFICATE base64 of OV/EV code-signing .pfx @@ -26,13 +43,99 @@ on: push: tags: ["v*"] workflow_dispatch: + inputs: + target_tag: + description: "Existing vX.Y.Z release to republish through the current protected workflow" + required: false + default: "" + type: string + allow_mirror_downgrade: + description: "Emergency only: allow this tag to replace a newer mirror latest.json" + required: false + default: false + type: boolean + mirror_downgrade_reason: + description: "Required audit reason when emergency mirror downgrade is enabled" + required: false + default: "" + type: string permissions: contents: read +env: + RELEASE_TAG: ${{ inputs.target_tag || github.ref_name }} + +# Serialize every tag in the repository through one release lane. The mirror +# promotion also uses per-object ETag conditions, so an external/manual writer +# cannot bypass monotonicity merely by racing this workflow. +concurrency: + group: release-latest-${{ github.repository }} + cancel-in-progress: false + queue: max + jobs: + prepare: + if: ${{ startsWith(inputs.target_tag || github.ref_name, 'v') }} + runs-on: ubuntu-latest + environment: release + permissions: + contents: read + outputs: + release_reusable: ${{ steps.lookup.outputs.release_reusable }} + release_asset_digests: ${{ steps.lookup.outputs.release_asset_digests }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: 20 + + - name: Require GitHub Immutable Releases + env: + GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }} + run: node scripts/check-immutable-releases.mjs + + - name: Check for an existing immutable release + id: lookup + shell: bash + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + DISPATCH_REF_NAME: ${{ github.ref_name }} + GH_TOKEN: ${{ github.token }} + REQUESTED_TARGET_TAG: ${{ inputs.target_tag || '' }} + run: | + set -euo pipefail + if [[ -n "$REQUESTED_TARGET_TAG" && "$DISPATCH_REF_NAME" != "$DEFAULT_BRANCH" ]]; then + echo "::error::target_tag must be dispatched from the default branch ($DEFAULT_BRANCH), not $DISPATCH_REF_NAME" + exit 1 + fi + if [[ ! "$RELEASE_TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then + echo "::error::release tag must be a semantic vX.Y.Z tag: $RELEASE_TAG" + exit 1 + fi + lookup_error="$RUNNER_TEMP/release-lookup-error.txt" + release_json="$RUNNER_TEMP/release.json" + if gh api "repos/$GITHUB_REPOSITORY/releases/tags/$RELEASE_TAG" >"$release_json" 2>"$lookup_error"; then + REQUESTED_TARGET_TAG="$REQUESTED_TARGET_TAG" \ + node scripts/check-release-reuse.mjs "$RELEASE_TAG" "$release_json" + elif grep -q 'HTTP 404' "$lookup_error"; then + if [[ -n "$REQUESTED_TARGET_TAG" ]]; then + echo "::error::target_tag $REQUESTED_TARGET_TAG must already exist as a GitHub Release" + exit 1 + fi + echo "release_reusable=false" >> "$GITHUB_OUTPUT" + else + cat "$lookup_error" >&2 + echo "::error::could not determine whether release $RELEASE_TAG already exists" + exit 1 + fi + build: - if: ${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + needs: prepare + if: ${{ needs.prepare.result == 'success' && needs.prepare.outputs.release_reusable != 'true' && github.ref_type == 'tag' && startsWith(github.ref_name, 'v') && !(github.event_name == 'workflow_dispatch' && inputs.target_tag != '') }} # Repo admins should configure the release environment and copy signing # secrets there before moving them off repo-level Actions secrets. environment: release @@ -281,13 +384,75 @@ jobs: - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: bundle-${{ matrix.target }} + name: bundle-${{ matrix.target }}-${{ github.run_id }}-${{ github.run_attempt }} path: dist-artifacts/* if-no-files-found: error + # Signing and notarization are intentionally non-deterministic. Keep every + # successful matrix artifact under an attempt-qualified name, then select the + # earliest successful artifact for each target. A release/stage rerun therefore + # reuses the exact first bytes instead of creating a second immutable payload. + select_artifacts: + needs: [prepare, build] + if: ${{ always() && needs.prepare.result == 'success' }} + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + outputs: + artifact_names: ${{ steps.select.outputs.artifact_names }} + steps: + - name: Select canonical build artifacts + id: select + if: ${{ needs.prepare.outputs.release_reusable != 'true' }} + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + targets=( + aarch64-apple-darwin + x86_64-apple-darwin + x86_64-pc-windows-msvc + aarch64-pc-windows-msvc + ) + names_file="$RUNNER_TEMP/canonical-artifact-names.txt" + : > "$names_file" + for poll in 1 2 3 4 5; do + gh api --paginate \ + "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100" \ + --jq '.artifacts[].name' > "$RUNNER_TEMP/run-artifacts.txt" + : > "$names_file" + complete=true + for target in "${targets[@]}"; do + selected="" + for attempt in $(seq 1 "$GITHUB_RUN_ATTEMPT"); do + candidate="bundle-${target}-${GITHUB_RUN_ID}-${attempt}" + if grep -Fxq "$candidate" "$RUNNER_TEMP/run-artifacts.txt"; then + selected="$candidate" + break + fi + done + if [[ -z "$selected" ]]; then + complete=false + break + fi + echo "$selected" >> "$names_file" + done + [[ "$complete" == "true" ]] && break + sleep 5 + done + [[ "$complete" == "true" ]] || { + echo "::error::no complete canonical release artifact set exists for this run" + exit 1 + } + artifact_names="$(jq -Rsc 'split("\n") | map(select(length > 0))' "$names_file")" + echo "artifact_names=$artifact_names" >> "$GITHUB_OUTPUT" + echo "Canonical artifacts: $artifact_names" + release: - if: ${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} - needs: build + if: ${{ always() && needs.prepare.result == 'success' && needs.select_artifacts.result == 'success' && startsWith(inputs.target_tag || github.ref_name, 'v') }} + needs: [prepare, build, select_artifacts] # Repo admins should configure the release environment and copy mirror # secrets there before moving them off repo-level Actions secrets. environment: release @@ -302,14 +467,167 @@ jobs: with: persist-credentials: false + # Historical workflow revisions used the legacy credential names and wrote + # latest.json unconditionally. Keeping those secrets deleted is the storage + # authorization boundary that makes an old run's "Re-run all jobs" fail + # before it can mutate either mirror. + - name: Enforce legacy mirror credential revocation + shell: bash + env: + LEGACY_R2_ACCESS_KEY_ID: ${{ secrets.MANAGER_R2_ACCESS_KEY_ID }} + LEGACY_R2_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_R2_SECRET_ACCESS_KEY }} + LEGACY_IHEP_ACCESS_KEY_ID: ${{ secrets.MANAGER_IHEP_S3_ACCESS_KEY_ID }} + LEGACY_IHEP_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_IHEP_S3_SECRET_ACCESS_KEY }} + run: | + if [[ -n "$LEGACY_R2_ACCESS_KEY_ID$LEGACY_R2_SECRET_ACCESS_KEY$LEGACY_IHEP_ACCESS_KEY_ID$LEGACY_IHEP_SECRET_ACCESS_KEY" ]]; then + echo "::error::legacy mirror write secrets are still configured; move credentials to the *_PROMOTION_* names and delete all four legacy access-key secrets" + exit 1 + fi + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: 20 - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - path: dist - merge-multiple: true + # Re-check on every release-job attempt. `prepare` is not rerun by + # "Re-run failed jobs", and publishing a resumed draft while the setting is + # disabled would create a mutable canonical source. + - name: Re-check GitHub Immutable Releases + env: + GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }} + run: node scripts/check-immutable-releases.mjs + + # `prepare` may belong to an earlier run attempt when the operator chooses + # "Re-run failed jobs". Refresh the release state inside this job so a + # GitHub Release published by the failed attempt becomes the canonical, + # immutable byte source instead of being uploaded a second time. + - name: Refresh immutable release state + id: live_release + shell: bash + env: + GH_TOKEN: ${{ github.token }} + REQUESTED_TARGET_TAG: ${{ inputs.target_tag || '' }} + run: | + set -euo pipefail + lookup_error="$RUNNER_TEMP/live-release-lookup-error.txt" + release_json="$RUNNER_TEMP/live-release.json" + if gh api "repos/$GITHUB_REPOSITORY/releases/tags/$RELEASE_TAG" >"$release_json" 2>"$lookup_error"; then + REQUESTED_TARGET_TAG="$REQUESTED_TARGET_TAG" \ + node scripts/check-release-reuse.mjs "$RELEASE_TAG" "$release_json" + elif grep -q 'HTTP 404' "$lookup_error"; then + if [[ -n "$REQUESTED_TARGET_TAG" ]]; then + echo "::error::target_tag $REQUESTED_TARGET_TAG must already exist as a GitHub Release" + exit 1 + fi + echo "release_reusable=false" >> "$GITHUB_OUTPUT" + echo "release_asset_digests={}" >> "$GITHUB_OUTPUT" + else + cat "$lookup_error" >&2 + echo "::error::could not refresh release state for $RELEASE_TAG" + exit 1 + fi + + # A target_tag dispatch intentionally checks out the protected default + # branch so historical workflow code cannot run. The updater trust root is + # release data, however, and must come from the target tag: a later key + # rotation must not make valid immutable historical artifacts unverifiable. + - name: Resolve updater trust root for release tag + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + release_config="$RUNNER_TEMP/release-tauri.conf.json" + gh api --method GET \ + -H "Accept: application/vnd.github.raw+json" \ + "repos/$GITHUB_REPOSITORY/contents/src-tauri/tauri.conf.json" \ + -f ref="$RELEASE_TAG" >"$release_config" + if ! updater_public_key="$( + jq -er ' + .plugins.updater.pubkey + | select(type == "string" and length > 0) + | select((contains("\n") or contains("\r")) | not) + | select(test("^[A-Za-z0-9+/]+={0,2}$")) + ' "$release_config" + )"; then + echo "::error::release tag $RELEASE_TAG has no valid single-line updater public key" + exit 1 + fi + printf 'RELEASE_TAURI_CONFIG=%s\n' "$release_config" >> "$GITHUB_ENV" + printf 'MIRROR_UPDATER_PUBLIC_KEY=%s\n' "$updater_public_key" >> "$GITHUB_ENV" + + - name: Download canonical build artifacts + if: ${{ steps.live_release.outputs.release_reusable != 'true' }} + shell: bash + env: + ARTIFACT_NAMES_JSON: ${{ needs.select_artifacts.outputs.artifact_names }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + rm -rf dist + mkdir -p dist + jq -er '.[]' <<<"$ARTIFACT_NAMES_JSON" | while IFS= read -r artifact; do + gh run download "$GITHUB_RUN_ID" \ + --repo "$GITHUB_REPOSITORY" \ + --name "$artifact" \ + --dir dist + done + + # A same-tag rerun or emergency downgrade must use the bytes already + # published for that tag. Rebuilding signed/notarized artifacts is + # intentionally non-reproducible and would conflict with immutable keys. + - name: Resolve immutable release artifact source + id: release_source + shell: bash + env: + GH_TOKEN: ${{ github.token }} + RELEASE_ASSET_DIGESTS: ${{ steps.live_release.outputs.release_asset_digests }} + run: | + set -euo pipefail + if [[ "${{ steps.live_release.outputs.release_reusable }}" == "true" ]]; then + published="$RUNNER_TEMP/published-release-artifacts" + rm -rf "$published" + mkdir -p "$published" + gh release download "$RELEASE_TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --dir "$published" \ + --pattern 'CodexAppManager*' \ + --pattern 'latest.json' + test -f "$published/latest.json" || { + echo "::error::published $RELEASE_TAG has no latest.json" + exit 1 + } + jq -e 'type == "object" and length > 0' <<<"$RELEASE_ASSET_DIGESTS" >/dev/null || { + echo "::error::immutable release digest evidence is missing" + exit 1 + } + while IFS=$'\t' read -r name expected_digest; do + file="$published/$name" + test -f "$file" || { + echo "::error::immutable release asset is missing after download: $name" + exit 1 + } + actual_digest="sha256:$(sha256sum "$file" | cut -d' ' -f1)" + if [[ "$actual_digest" != "$expected_digest" ]]; then + echo "::error::immutable release digest mismatch for $name" + exit 1 + fi + echo "Verified immutable release digest: $name" + done < <(jq -r 'to_entries[] | [.key, .value] | @tsv' <<<"$RELEASE_ASSET_DIGESTS") + cp "$published/latest.json" "$RUNNER_TEMP/published-latest.json" + rm "$published/latest.json" + rm -rf dist + mv "$published" dist + echo "existing=true" >> "$GITHUB_OUTPUT" + echo "source=github-release" >> "$GITHUB_OUTPUT" + echo "Reusing immutable artifacts from existing release $RELEASE_TAG" + else + test -d dist || { + echo "::error::no build artifacts are available for new release $RELEASE_TAG" + exit 1 + } + echo "existing=false" >> "$GITHUB_OUTPUT" + echo "source=canonical-actions-artifacts" >> "$GITHUB_OUTPUT" + fi # Validate the merged *publishable* tree (renamed final names), not the # per-target intermediate build directories from the matrix jobs. @@ -322,12 +640,20 @@ jobs: missing=0 require() { local pattern="$1" + local matches # shellcheck disable=SC2086 - if ! compgen -G "dist/$pattern" > /dev/null; then + matches="$(compgen -G "dist/$pattern" || true)" + if [[ -z "$matches" ]]; then echo "::error::[build] missing final artifact matching: $pattern" missing=1 else - echo "OK $pattern → $(compgen -G "dist/$pattern" | tr '\n' ' ')" + while IFS= read -r file; do + if [[ ! -s "$file" ]]; then + echo "::error::[build] empty final artifact: $file" + missing=1 + fi + done <<<"$matches" + echo "OK $pattern → $(tr '\n' ' ' <<<"$matches")" fi } require "CodexAppManager_aarch64.dmg" @@ -350,29 +676,91 @@ jobs: - name: Generate updater manifest (latest.json) env: ALLOW_PARTIAL_RELEASE: ${{ vars.ALLOW_PARTIAL_RELEASE }} - run: node scripts/gen-updater-manifest.mjs "${{ github.ref_name }}" dist + run: | + node scripts/gen-updater-manifest.mjs "$RELEASE_TAG" dist + cp latest.json "$RUNNER_TEMP/artifact-derived-latest.json" + if [[ "${{ steps.release_source.outputs.existing }}" == "true" ]]; then + cp "$RUNNER_TEMP/published-latest.json" latest.json + fi + if [[ "$RELEASE_TAG" != *-* ]]; then + node scripts/validate-release-manifest.mjs \ + "$RELEASE_TAG" \ + latest.json \ + "$RUNNER_TEMP/artifact-derived-latest.json" + fi + + # A wrong private/public updater-key pairing cannot be repaired after an + # immutable GitHub Release is published. Verify the exact local payloads + # (including manifest/sidecar equality) before either mirror staging or + # draft publication. This also protects prereleases, which skip mirrors. + - name: Verify local updater signatures before immutable publication + run: | + node scripts/verify-release-artifacts.mjs \ + latest.json \ + dist \ + "$RELEASE_TAURI_CONFIG" - # Stable releases mirror in two phases: stage immutable versioned assets - # before GitHub Release publication, then promote latest.json only after + # Stable releases stage immutable versioned assets before GitHub Release + # publication, verify them separately, then promote latest.json only after # the GitHub Release succeeds. Pre-releases still skip the mirror entirely. - name: Stage CDN mirror candidate (R2 + IHEP S3) id: mirror_stage - if: ${{ !contains(github.ref_name, '-') }} + if: ${{ !contains(env.RELEASE_TAG, '-') }} env: MIRROR_PHASE: stage + MIRROR_REQUIRED_BACKENDS: r2,ihep + MIRROR_CANDIDATE_ID: ${{ github.run_id }}-${{ github.run_attempt }} + MIRROR_ALLOW_DOWNGRADE: ${{ inputs.allow_mirror_downgrade && '1' || '0' }} + MIRROR_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + MIRROR_DOWNGRADE_REASON: ${{ inputs.mirror_downgrade_reason || '' }} + MIRROR_WORKFLOW_REF_NAME: ${{ github.ref_name }} + MANAGER_R2_S3_ENDPOINT: ${{ secrets.MANAGER_R2_S3_ENDPOINT }} + MANAGER_R2_BUCKET: ${{ vars.MANAGER_R2_BUCKET }} + MANAGER_R2_ACCESS_KEY_ID: ${{ secrets.MANAGER_R2_PROMOTION_ACCESS_KEY_ID }} + MANAGER_R2_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_R2_PROMOTION_SECRET_ACCESS_KEY }} + MANAGER_IHEP_S3_ENDPOINT: ${{ secrets.MANAGER_IHEP_S3_ENDPOINT }} + MANAGER_IHEP_S3_BUCKET: ${{ secrets.MANAGER_IHEP_S3_BUCKET }} + MANAGER_IHEP_S3_REGION: ${{ secrets.MANAGER_IHEP_S3_REGION }} + MANAGER_IHEP_S3_PREFIX: ${{ secrets.MANAGER_IHEP_S3_PREFIX }} + MANAGER_IHEP_S3_ACCESS_KEY_ID: ${{ secrets.MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID }} + MANAGER_IHEP_S3_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_IHEP_S3_PROMOTION_SECRET_ACCESS_KEY }} + run: | + [ -f dist/latest.json ] || cp latest.json dist/latest.json + bash scripts/sync-mirror.sh dist + # The GitHub Release uploads the root latest.json exactly once. Recreate + # dist/latest.json later for promotion instead of giving the upload + # action two paths with the same asset basename. + rm -f dist/latest.mirror.json dist/latest.json + + # Complete a direct readback from both object stores and force separate + # readbacks through the Worker's R2 and mainland-China IHEP branches. This + # is a verify-only phase: latest.json cannot be changed here. Both public + # branches must pass before GitHub publication becomes immutable. + - name: Verify staged CDN mirror before immutable publication + id: mirror_verify + if: ${{ !contains(env.RELEASE_TAG, '-') }} + env: + MIRROR_PHASE: verify + MIRROR_REQUIRED_BACKENDS: r2,ihep + MIRROR_CANDIDATE_ID: ${{ github.run_id }}-${{ github.run_attempt }} + MIRROR_ALLOW_DOWNGRADE: ${{ inputs.allow_mirror_downgrade && '1' || '0' }} + MIRROR_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + MIRROR_DOWNGRADE_REASON: ${{ inputs.mirror_downgrade_reason || '' }} + MIRROR_WORKFLOW_REF_NAME: ${{ github.ref_name }} MANAGER_R2_S3_ENDPOINT: ${{ secrets.MANAGER_R2_S3_ENDPOINT }} - MANAGER_R2_ACCESS_KEY_ID: ${{ secrets.MANAGER_R2_ACCESS_KEY_ID }} - MANAGER_R2_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_R2_SECRET_ACCESS_KEY }} + MANAGER_R2_BUCKET: ${{ vars.MANAGER_R2_BUCKET }} + MANAGER_R2_ACCESS_KEY_ID: ${{ secrets.MANAGER_R2_PROMOTION_ACCESS_KEY_ID }} + MANAGER_R2_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_R2_PROMOTION_SECRET_ACCESS_KEY }} MANAGER_IHEP_S3_ENDPOINT: ${{ secrets.MANAGER_IHEP_S3_ENDPOINT }} MANAGER_IHEP_S3_BUCKET: ${{ secrets.MANAGER_IHEP_S3_BUCKET }} MANAGER_IHEP_S3_REGION: ${{ secrets.MANAGER_IHEP_S3_REGION }} MANAGER_IHEP_S3_PREFIX: ${{ secrets.MANAGER_IHEP_S3_PREFIX }} - MANAGER_IHEP_S3_ACCESS_KEY_ID: ${{ secrets.MANAGER_IHEP_S3_ACCESS_KEY_ID }} - MANAGER_IHEP_S3_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_IHEP_S3_SECRET_ACCESS_KEY }} + MANAGER_IHEP_S3_ACCESS_KEY_ID: ${{ secrets.MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID }} + MANAGER_IHEP_S3_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_IHEP_S3_PROMOTION_SECRET_ACCESS_KEY }} run: | [ -f dist/latest.json ] || cp latest.json dist/latest.json bash scripts/sync-mirror.sh dist - rm -f dist/latest.mirror.json + rm -f dist/latest.mirror.json dist/latest.json # ── release notes: the hand-written file from the version-bump PR # (docs/releases/.md, see docs/releases/TEMPLATE.md) wins; @@ -380,7 +768,7 @@ jobs: # empty. GitHub's auto "What's Changed" section is appended either way. - name: Prepare release notes run: | - NOTES="docs/releases/${{ github.ref_name }}.md" + NOTES="docs/releases/$RELEASE_TAG.md" if [ -f "$NOTES" ]; then cp "$NOTES" "$RUNNER_TEMP/release-notes.md" else @@ -432,17 +820,21 @@ jobs: exit "$status" - - name: Publish GitHub Release - id: publish_release + # Explicit draft-first publication keeps asset upload ahead of publication + # for both stable and prerelease tags. This is required when GitHub immutable + # releases are enabled: a published release can no longer accept assets. + - name: Upload GitHub Release draft + id: upload_release_draft + if: ${{ steps.release_source.outputs.existing != 'true' }} uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3 with: - tag_name: ${{ github.ref_name }} - draft: false + tag_name: ${{ env.RELEASE_TAG }} + draft: true # Tags carrying a pre-release identifier (e.g. v0.1.2-rc1) publish as a # prerelease, so GitHub's `releases/latest` — and the updater endpoint # pointing at it — keep resolving to the last stable build instead of # serving a release-candidate to real users. - prerelease: ${{ contains(github.ref_name, '-') }} + prerelease: ${{ contains(env.RELEASE_TAG, '-') }} body_path: ${{ runner.temp }}/release-notes.md generate_release_notes: true files: | @@ -451,36 +843,86 @@ jobs: SHA256SUMS release-assets/* + # Omitting `draft` tells action-gh-release v3 to publish the existing draft + # after all uploads succeeded while retaining its stable/prerelease metadata. + - name: Publish GitHub Release + id: publish_release + if: ${{ steps.release_source.outputs.existing != 'true' }} + uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3 + with: + tag_name: ${{ env.RELEASE_TAG }} + + - name: Verify published immutable Release and asset digests + if: ${{ steps.release_source.outputs.existing != 'true' }} + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + release_json="$RUNNER_TEMP/published-release.json" + verification_log="$RUNNER_TEMP/published-release-verification.log" + verification_output="$RUNNER_TEMP/published-release-verification-output.txt" + verified=false + for attempt in 1 2 3 4 5; do + : > "$verification_log" + : > "$verification_output" + if gh api "repos/$GITHUB_REPOSITORY/releases/tags/$RELEASE_TAG" >"$release_json" 2>"$verification_log" \ + && REQUESTED_TARGET_TAG="$RELEASE_TAG" GITHUB_OUTPUT="$verification_output" \ + node scripts/check-release-reuse.mjs "$RELEASE_TAG" "$release_json" >>"$verification_log" 2>&1; then + verified=true + break + fi + sleep 3 + done + if [[ "$verified" != "true" ]]; then + cat "$verification_log" >&2 + echo "::error::published $RELEASE_TAG did not become immutable with canonical asset digests" + exit 1 + fi + cat "$verification_log" + + # Attest immediately after GitHub confirms the canonical immutable bytes, + # before mirror promotion can fail the job. Keep the subjects limited to + # updater payloads and latest.json: on a failed-job rerun those are fetched + # from the immutable Release and checked against GitHub's canonical SHA-256 + # digests. SHA256SUMS and SBOMs are intentionally excluded because they are + # regenerated locally and are not necessarily the bytes already published. + # A failed attestation must fail the job so the release stays retryable and + # neither mirror promotion nor winget dispatch can proceed without proof. + - name: Attest build provenance + id: attest + if: ${{ steps.publish_release.outcome == 'success' || steps.release_source.outputs.existing == 'true' }} + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + with: + subject-path: | + dist/* + latest.json + - name: Promote CDN mirror latest (R2 + IHEP S3) id: mirror_promote - if: ${{ !contains(github.ref_name, '-') }} + if: ${{ success() && steps.attest.outcome == 'success' && !contains(env.RELEASE_TAG, '-') }} env: MIRROR_PHASE: promote + MIRROR_REQUIRED_BACKENDS: r2,ihep + MIRROR_CANDIDATE_ID: ${{ github.run_id }}-${{ github.run_attempt }} + MIRROR_ALLOW_DOWNGRADE: ${{ inputs.allow_mirror_downgrade && '1' || '0' }} + MIRROR_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + MIRROR_DOWNGRADE_REASON: ${{ inputs.mirror_downgrade_reason || '' }} + MIRROR_WORKFLOW_REF_NAME: ${{ github.ref_name }} MANAGER_R2_S3_ENDPOINT: ${{ secrets.MANAGER_R2_S3_ENDPOINT }} - MANAGER_R2_ACCESS_KEY_ID: ${{ secrets.MANAGER_R2_ACCESS_KEY_ID }} - MANAGER_R2_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_R2_SECRET_ACCESS_KEY }} + MANAGER_R2_BUCKET: ${{ vars.MANAGER_R2_BUCKET }} + MANAGER_R2_ACCESS_KEY_ID: ${{ secrets.MANAGER_R2_PROMOTION_ACCESS_KEY_ID }} + MANAGER_R2_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_R2_PROMOTION_SECRET_ACCESS_KEY }} MANAGER_IHEP_S3_ENDPOINT: ${{ secrets.MANAGER_IHEP_S3_ENDPOINT }} MANAGER_IHEP_S3_BUCKET: ${{ secrets.MANAGER_IHEP_S3_BUCKET }} MANAGER_IHEP_S3_REGION: ${{ secrets.MANAGER_IHEP_S3_REGION }} MANAGER_IHEP_S3_PREFIX: ${{ secrets.MANAGER_IHEP_S3_PREFIX }} - MANAGER_IHEP_S3_ACCESS_KEY_ID: ${{ secrets.MANAGER_IHEP_S3_ACCESS_KEY_ID }} - MANAGER_IHEP_S3_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_IHEP_S3_SECRET_ACCESS_KEY }} + MANAGER_IHEP_S3_ACCESS_KEY_ID: ${{ secrets.MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID }} + MANAGER_IHEP_S3_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_IHEP_S3_PROMOTION_SECRET_ACCESS_KEY }} run: | [ -f dist/latest.json ] || cp latest.json dist/latest.json bash scripts/sync-mirror.sh dist - - name: Attest build provenance - id: attest - if: ${{ steps.publish_release.outcome == 'success' }} - continue-on-error: true - uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 - with: - subject-path: | - dist/* - latest.json - SHA256SUMS - release-assets/* - # The release above is created with GITHUB_TOKEN, which by design does not # trigger other workflows — so winget.yml's `on: release` never fires. # Dispatch it explicitly (stable releases only). winget.yml no-ops cleanly @@ -488,11 +930,11 @@ jobs: # so a re-run is harmless. - name: Trigger winget submission id: winget - if: ${{ !contains(github.ref_name, '-') }} + if: ${{ success() && steps.attest.outcome == 'success' && !contains(env.RELEASE_TAG, '-') }} env: GH_TOKEN: ${{ github.token }} run: | - if gh workflow run winget.yml -f release-tag="${{ github.ref_name }}" --repo "${{ github.repository }}"; then + if gh workflow run winget.yml -f release-tag="$RELEASE_TAG" --repo "${{ github.repository }}"; then echo "status=dispatched" >> "$GITHUB_OUTPUT" else echo "::warning::winget dispatch failed" @@ -502,10 +944,26 @@ jobs: - name: Write release summary if: always() env: - PUBLISH_OUTCOME: ${{ steps.publish_release.outcome }} + PUBLISH_OUTCOME: ${{ steps.release_source.outputs.existing == 'true' && 'reused' || steps.publish_release.outcome }} MIRROR_STAGE_OUTCOME: ${{ steps.mirror_stage.outcome }} + MIRROR_VERIFY_OUTCOME: ${{ steps.mirror_verify.outcome }} MIRROR_PROMOTE_OUTCOME: ${{ steps.mirror_promote.outcome }} WINGET_OUTCOME: ${{ steps.winget.outputs.status || steps.winget.outcome }} SBOM_OUTCOME: ${{ steps.sbom.outcome }} ATTESTATION_OUTCOME: ${{ steps.attest.outcome }} run: node scripts/write-release-summary.mjs + + - name: Upload release audit records + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: release-audit-${{ env.RELEASE_TAG }}-${{ github.run_attempt }} + path: | + manifest-summary.json + updater-signature-verification.json + mirror-stage-summary.json + mirror-verification-summary.json + mirror-promotion-summary.json + SHA256SUMS + if-no-files-found: warn + retention-days: 90 diff --git a/cloudflare/manager-download-router/README.md b/cloudflare/manager-download-router/README.md index 920e514..bb08eea 100644 --- a/cloudflare/manager-download-router/README.md +++ b/cloudflare/manager-download-router/README.md @@ -18,8 +18,12 @@ The updater (`src-tauri/tauri.conf.json`) already checks ## Why a separate latest.json on the mirror `latest.json`'s embedded signatures sign the artifact **bytes**, not the URL, so re-hosting the same files keeps them valid — we only rewrite the download URLs to -`…/manager//`. `scripts/sync-mirror.sh` does this on every -(non-pre-)release. +`…/manager//`. `scripts/sync-mirror.sh` stages stable releases, +then verifies both storage endpoints and separately forces this Worker's R2 and +IHEP branches before the GitHub Release becomes immutable. Probe responses identify +their backend with `X-Codex-Mirror-Backend`; an explicit IHEP probe fails instead +of falling back when its Worker secrets are incomplete. The release repeats that +readback immediately before advancing the mirror's `latest.json` pointer. Installers are uploaded under a **per-version** key (`/`) and `latest.json` stays at the fixed root the updater polls. macOS updater tarballs @@ -31,7 +35,7 @@ self-consistent; v0.1.9+ use the versioned layout.) > ⚠️ The mirror's `latest.json` MUST be refreshed on every stable release, or the > first endpoint serves a stale version and **blocks** updates. That's why the -> `release.yml` "Sync to CDN mirror" step exists. +> `release.yml` stage, verify, and promote steps exist. ## Already provisioned (done) - R2 bucket `codex-app-manager` created. @@ -59,17 +63,27 @@ So each release uploads to the mirror (`release.yml` → `scripts/sync-mirror.sh | Secret | Notes | | --- | --- | | `MANAGER_R2_S3_ENDPOINT` | `https://d39dc6c92d1c4cfde580bf13e946b616.r2.cloudflarestorage.com` | -| `MANAGER_R2_ACCESS_KEY_ID` / `MANAGER_R2_SECRET_ACCESS_KEY` | R2 S3 API token (can reuse the mirror's; account-scoped) | -| `MANAGER_IHEP_S3_ENDPOINT` / `_BUCKET` / `_ACCESS_KEY_ID` / `_SECRET_ACCESS_KEY` | IHEP creds (optional; `_REGION`/`_PREFIX` optional) | +| `MANAGER_R2_PROMOTION_ACCESS_KEY_ID` / `MANAGER_R2_PROMOTION_SECRET_ACCESS_KEY` | R2 S3 API token for the current protected release workflow | +| `MANAGER_IHEP_S3_ENDPOINT` / `_BUCKET` | IHEP endpoint and bucket (`_REGION`/`_PREFIX` optional) | +| `MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID` / `MANAGER_IHEP_S3_PROMOTION_SECRET_ACCESS_KEY` | IHEP token for the current protected release workflow | -R2 secrets missing → step warns and skips (mirror goes stale). IHEP missing → -CN just falls back to R2 via the worker. +Both backends must already contain the same valid `latest.json` baseline and must +support conditional `PutObject` (`If-Match` / `If-None-Match`) plus custom +user-metadata round trips through `HeadObject`. Delete the legacy access-key secret +names after migration so historical workflow revisions cannot perform their old +unconditional write. + +Deploy this Worker revision before enabling the protected release workflow. The +release gate intentionally rejects an older Worker that cannot force and identify +both public backend branches. ## Deploy / manual sync ```bash wrangler deploy # from this directory # manual re-sync of a tag's assets (download release → rewrite → upload): -# gh release download vX.Y.Z -D dist && cp -f dist/latest.json dist/ && \ +# rm -rf dist && gh release download vX.Y.Z -D dist && \ # MANAGER_R2_S3_ENDPOINT=… MANAGER_R2_ACCESS_KEY_ID=… MANAGER_R2_SECRET_ACCESS_KEY=… \ +# MANAGER_IHEP_S3_ENDPOINT=… MANAGER_IHEP_S3_BUCKET=… \ +# MANAGER_IHEP_S3_ACCESS_KEY_ID=… MANAGER_IHEP_S3_SECRET_ACCESS_KEY=… \ # bash ../../scripts/sync-mirror.sh dist ``` diff --git a/cloudflare/manager-download-router/src/index.js b/cloudflare/manager-download-router/src/index.js index 7856a06..34e1717 100644 --- a/cloudflare/manager-download-router/src/index.js +++ b/cloudflare/manager-download-router/src/index.js @@ -38,6 +38,7 @@ export default { } const country = request.cf?.country || request.headers.get("CF-IPCountry") || ""; + const probeBackend = requestedProbeBackend(url); const secondaryCountryCodes = new Set( (env.SECONDARY_COUNTRY_CODES || DEFAULT_SECONDARY_COUNTRY_CODES) .split(",") @@ -46,7 +47,19 @@ export default { ); // ── Mainland China: presign the IHEP S3 object and redirect ────────────── - if (secondaryCountryCodes.has(country.toUpperCase()) && hasSecondaryS3Config(env)) { + const secondaryConfigured = hasSecondaryS3Config(env); + if (probeBackend === "ihep" && !secondaryConfigured) { + return new Response("Secondary mirror is not configured", { + status: 503, + headers: { "X-Codex-Mirror-Backend": "ihep" }, + }); + } + const useSecondary = + probeBackend === "ihep" || + (probeBackend !== "r2" && + secondaryCountryCodes.has(country.toUpperCase()) && + secondaryConfigured); + if (useSecondary) { const objectKey = objectKeyForKey(key, env.SECONDARY_S3_PREFIX || ""); const signedUrl = await presignS3Url({ // Sign for the actual method: the redirect preserves it, and an @@ -62,7 +75,7 @@ export default { expiresInSeconds: ttlSeconds(env.SECONDARY_S3_SIGNED_URL_TTL_SECONDS), responseHeaders: {}, }); - return redirect(signedUrl); + return redirect(signedUrl, "ihep"); } // ── Everyone else: stream straight from the bound R2 bucket ────────────── @@ -74,6 +87,7 @@ export default { object.writeHttpMetadata(headers); // Content-Type etc. from R2 metadata headers.set("ETag", object.httpEtag); headers.set("Cache-Control", cacheControlForKey(key)); + headers.set("X-Codex-Mirror-Backend", "r2"); if (request.method === "HEAD") { return new Response(null, { headers }); } @@ -97,16 +111,32 @@ function hasSecondaryS3Config(env) { ); } +// Release verification must exercise both geographic branches from a single +// GitHub runner. This is a routing selector, not an authorization mechanism: +// every addressed object is already public through this Worker. Requiring the +// run-specific probe token shape prevents ordinary download links from changing +// backend accidentally while allowing CI to force and identify each branch. +function requestedProbeBackend(url) { + const token = url.searchParams.get("cam_probe") || ""; + const backend = url.searchParams.get("cam_backend") || ""; + if (!/^[0-9a-f]{24}$/.test(token)) return null; + return ["r2", "ihep"].includes(backend) ? backend : null; +} + function ttlSeconds(value) { const parsed = Number.parseInt(value || DEFAULT_SIGNED_URL_TTL_SECONDS, 10); if (!Number.isFinite(parsed)) return DEFAULT_SIGNED_URL_TTL_SECONDS; return Math.min(Math.max(parsed, 1), 604800); } -function redirect(location) { +function redirect(location, backend) { return new Response(null, { status: 302, - headers: { Location: location, "Cache-Control": "private, no-store" }, + headers: { + Location: location, + "Cache-Control": "private, no-store", + "X-Codex-Mirror-Backend": backend, + }, }); } diff --git a/cloudflare/manager-download-router/test/index.test.js b/cloudflare/manager-download-router/test/index.test.js index e49431a..fefd681 100644 --- a/cloudflare/manager-download-router/test/index.test.js +++ b/cloudflare/manager-download-router/test/index.test.js @@ -1,6 +1,8 @@ import { describe, expect, it } from "vitest"; import worker from "../src/index.js"; +const PROBE_TOKEN = "0123456789abcdef01234567"; + const latestManifest = JSON.stringify({ version: "0.1.18", platforms: { @@ -140,6 +142,7 @@ describe("manager download router", () => { const jsonRes = await worker.fetch(request("/manager/0.1.18/latest.json"), jsonEnv); expect(jsonRes.status).toBe(200); expect(jsonRes.headers.get("Cache-Control")).toBe("public, max-age=120, s-maxage=120"); + expect(jsonRes.headers.get("X-Codex-Mirror-Backend")).toBe("r2"); const installerEnv = { BUCKET: bucket({ @@ -166,6 +169,7 @@ describe("manager download router", () => { ); expect(res.status).toBe(302); + expect(res.headers.get("X-Codex-Mirror-Backend")).toBe("ihep"); const location = new URL(res.headers.get("Location")); expect(location.origin).toBe("https://s3.example.test"); expect(location.pathname).toBe("/root/mirror-bucket/manager/0.1.18/CodexAppManager_0.1.18_x64-setup.exe"); @@ -195,6 +199,42 @@ describe("manager download router", () => { expect(calls).toEqual(["0.1.18/CodexAppManager_0.1.18_x64-setup.exe"]); }); + it("lets release probes force and identify both geographic backends", async () => { + const calls = []; + const path = "/manager/0.1.18/CodexAppManager_0.1.18_x64-setup.exe"; + const env = { + BUCKET: bucket( + { "0.1.18/CodexAppManager_0.1.18_x64-setup.exe": r2Object("installer") }, + calls, + ), + ...secondaryEnv(), + }; + + const forcedR2 = await worker.fetch( + request(`${path}?cam_probe=${PROBE_TOKEN}&cam_backend=r2`, {}, { country: "CN" }), + env, + ); + expect(forcedR2.status).toBe(200); + expect(forcedR2.headers.get("X-Codex-Mirror-Backend")).toBe("r2"); + expect(await forcedR2.text()).toBe("installer"); + + const forcedIhep = await worker.fetch( + request(`${path}?cam_probe=${PROBE_TOKEN}&cam_backend=ihep`, {}, { country: "US" }), + env, + ); + expect(forcedIhep.status).toBe(302); + expect(forcedIhep.headers.get("X-Codex-Mirror-Backend")).toBe("ihep"); + expect(new URL(forcedIhep.headers.get("Location")).origin).toBe("https://s3.example.test"); + expect(calls).toEqual(["0.1.18/CodexAppManager_0.1.18_x64-setup.exe"]); + + const missingSecondary = await worker.fetch( + request(`${path}?cam_probe=${PROBE_TOKEN}&cam_backend=ihep`), + { BUCKET: bucket({}), ...secondaryEnv({ SECONDARY_S3_SECRET_ACCESS_KEY: "" }) }, + ); + expect(missingSecondary.status).toBe(503); + expect(missingSecondary.headers.get("X-Codex-Mirror-Backend")).toBe("ihep"); + }); + it("rejects empty keys, directory keys, and traversal-looking keys", async () => { const calls = []; const env = { BUCKET: bucket({}, calls) }; diff --git a/docs/release.md b/docs/release.md index 5e6a7e3..ff9369a 100644 --- a/docs/release.md +++ b/docs/release.md @@ -67,6 +67,93 @@ PR-time x64 packaged smoke lives in Required CI (`ci.yml`) also runs standalone engine crate tests for `codex-mac-engine` and `codex-win-engine`. +## Mirror promotion safety + +Stable releases use a stage/verify/promote, dual-backend protocol implemented by +[`scripts/mirror-release.mjs`](../scripts/mirror-release.mjs): + +1. Before uploading anything immutable, cryptographically verify every local + updater payload against its manifest signature and the updater public key in + `tauri.conf.json`. The manifest signature must also equal the local `.sig` + sidecar. This gate runs for prereleases as well as stable releases. +2. Upload immutable, versioned artifacts and a run-specific candidate manifest + to both R2 and IHEP. A versioned object is never overwritten. +3. Before publishing the GitHub Release, download the candidate and **every staged + artifact** directly from each S3 endpoint. Verify every file's byte size and + SHA-256, plus each updater bundle's embedded Tauri/minisign signature with the + public key in `tauri.conf.json`. This includes both DMG installers even though + they are not referenced by the updater manifest. Stable promotion requires all + four updater platforms and requires every manifest signature to match its + downloaded `.sig` sidecar; `ALLOW_PARTIAL_RELEASE` cannot weaken the + mirror gate. The same verify-only phase forces separate downloads of the + run-specific candidate and all four updater payloads through the public + Worker's R2 and mainland-China IHEP branches. Each response must identify the + requested backend, catching bad routes, bucket bindings, Worker secondary + credentials, and presigned redirects before immutable publication. +4. Publish the draft GitHub Release and require GitHub to report that it is + immutable with canonical asset digests. +5. Repeat the complete direct-backend and public-route readback immediately before + promotion, then read each backend's current `latest.json` and compare semantic + versions. Newer candidates advance, same-version reruns are + no-write/idempotent, and older tags fail closed. +6. Publish with ETag (`If-Match` / `If-None-Match`) conditions. If the second + backend or final verification fails, restore every backend already changed, + also using ETag conditions so rollback cannot overwrite a concurrent writer. + +The release workflow has one repository-wide `release-latest-*` concurrency lane +with `queue: max`, so every pending tag remains queued instead of a third tag +replacing the second. The object-store conditions remain the independent backstop +for manual/external writes. `mirror-stage-summary.json`, +`mirror-verification-summary.json`, and `mirror-promotion-summary.json` are shown +in the job summary and retained as workflow artifacts for 90 days. If a runner is +hard-killed after only one +`latest.json` write, a fresh run recognizes the exact candidate identity, leaves +that backend untouched, and conditionally advances only the lagging backend. + +Before `prepare` permits any build, and again at the start of every release-job +attempt, the workflow queries the repository Immutable Releases setting with a +dedicated fine-grained, read-only token. A missing token, failed query, or +`enabled: false` response fails closed before draft upload or mirror publication. + +Same-tag reruns reuse the artifacts and `latest.json` attached to a complete, +published GitHub Release only when GitHub reports `immutable: true` and a canonical +`sha256:` digest for every required asset. The workflow re-hashes every downloaded +asset against those API digests, and rejects a mutable release instead of calling +its current bytes canonical. Only a draft is repairable; a published mutable or +incomplete Release fails closed before any asset upload. The release job performs +this lookup live rather than +trusting `prepare` outputs from an earlier attempt, so **Re-run failed jobs** after +a mirror failure reuses the already-published immutable bytes. While a new or +partial release is still being repaired, the workflow selects the earliest +successful, attempt-qualified Actions artifact for each platform so every rerun +stages the same signed/notarized bytes. It never creates a second byte sequence for +an immutable version key. New releases are uploaded as drafts first and published +only after all assets succeed, including prereleases. Immediately after publish, +the workflow requires the Release itself to report `immutable: true` and canonical +digests for every required asset before any mirror pointer can advance. A +historical Actions run still executes its historical workflow revision, so the +release environment intentionally uses new `*_PROMOTION_*` credential names. The +four legacy access-key secrets listed below must be deleted; the current workflow +has a hard gate that rejects them if they are reintroduced. + +### Emergency mirror downgrade + +A normal old-tag rerun can never downgrade the mirror. For an incident rollback: + +1. Open the current `Release` workflow and dispatch it from the repository's + **default branch** (never select the historical tag's workflow revision). +2. Set `target_tag` to the already-published `vX.Y.Z` release to restore. +3. Enable `allow_mirror_downgrade`. +4. Enter a concrete `mirror_downgrade_reason` (at least 10 characters). + +The workflow downloads the target release's original assets instead of rebuilding +them. The target GitHub Release must be immutable and expose canonical SHA-256 +digests; older mutable releases must first be migrated through an explicitly +reviewed process and cannot be used directly. The script rejects the override on +tag-push events or without GitHub actor/run metadata. The triggering actor (plus +the original workflow actor on a rerun), reason, whether the override was actually +used, and the run URL are written to the promotion audit and job summary. + ## Required GitHub Actions secrets | Secret | What | @@ -80,8 +167,24 @@ Required CI (`ci.yml`) also runs standalone engine crate tests for | `AC_API_KEY_BASE64` | base64 of the `AuthKey_XXXX.p8` | | `TAURI_SIGNING_PRIVATE_KEY` | updater private key | | `TAURI_SIGNING_PRIVATE_KEY_PASSWORD` | its password (empty if none) | - -### Optional Windows Authenticode secrets / vars +| `IMMUTABLE_RELEASES_READ_TOKEN` | fine-grained token used only to read this repository's Immutable Releases setting; grant **Administration: read-only** and store it in the `release` environment | +| `MANAGER_R2_S3_ENDPOINT` | R2 S3-compatible endpoint | +| `MANAGER_R2_PROMOTION_ACCESS_KEY_ID` | R2 write/read credential used only by the protected workflow | +| `MANAGER_R2_PROMOTION_SECRET_ACCESS_KEY` | R2 write/read secret used only by the protected workflow | +| `MANAGER_IHEP_S3_ENDPOINT` | IHEP S3-compatible endpoint | +| `MANAGER_IHEP_S3_BUCKET` | IHEP bucket | +| `MANAGER_IHEP_S3_REGION` | IHEP region; defaults to `auto` when empty | +| `MANAGER_IHEP_S3_PREFIX` | optional object-key prefix for IHEP | +| `MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID` | IHEP write/read credential used only by the protected workflow | +| `MANAGER_IHEP_S3_PROMOTION_SECRET_ACCESS_KEY` | IHEP write/read secret used only by the protected workflow | + +After creating the promotion credentials, delete (and preferably revoke/rotate) +`MANAGER_R2_ACCESS_KEY_ID`, `MANAGER_R2_SECRET_ACCESS_KEY`, +`MANAGER_IHEP_S3_ACCESS_KEY_ID`, and `MANAGER_IHEP_S3_SECRET_ACCESS_KEY` from the +repository and `release` environment. Historical workflow revisions reference +those exact names; leaving any of them available defeats old-run isolation. + +### Optional signing secrets and release variables | Name | What | |---|---| @@ -89,6 +192,7 @@ Required CI (`ci.yml`) also runs standalone engine crate tests for | `WINDOWS_CERTIFICATE_PASSWORD` | password for that .pfx | | `AUTHENTICODE_REQUIRED` (repo **variable**) | `true` → fail release when PE is not `Valid` | | `WINDOWS_TIMESTAMP_URL` (repo **variable**) | optional RFC3161 timestamp URL | +| `MANAGER_R2_BUCKET` (repo **variable**) | R2 bucket; defaults to `codex-app-manager` | Export your local .p12 / .p8 / .pfx to base64 with `base64 -i file -o -`. @@ -98,3 +202,9 @@ Export your local .p12 / .p8 / .pfx to base64 with `base64 -i file -o -`. > > Keep **updater signature**, **Authenticode**, and **SmartScreen reputation** > conceptually separate — see [`docs/windows-signing.md`](./windows-signing.md). +> +> Before enabling promotion, seed both S3-compatible endpoints with a valid +> `latest.json` baseline. Both endpoints must support conditional `PutObject` +> requests (`If-Match` / `If-None-Match`) and must preserve custom user metadata +> through `HeadObject`. Promotion fails closed if either baseline is absent or an +> endpoint cannot enforce those requirements. diff --git a/package.json b/package.json index 89e1ada..8cabff7 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "check": "tsc --noEmit", "lint": "eslint . --max-warnings=0", "test": "vitest run", + "test:release": "vitest run scripts/mirror-release.test.mjs scripts/release-workflow.test.mjs", "tauri": "tauri", "tauri:dev": "tauri dev", "tauri:build": "tauri build", diff --git a/scripts/check-immutable-releases.mjs b/scripts/check-immutable-releases.mjs new file mode 100644 index 0000000..3ab5950 --- /dev/null +++ b/scripts/check-immutable-releases.mjs @@ -0,0 +1,75 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +const API_VERSION = "2026-03-10"; +const REPOSITORY_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; + +export function assertImmutableReleasesEnabled(settings) { + if (!settings || typeof settings !== "object" || Array.isArray(settings)) { + throw new Error("GitHub Immutable Releases response must be an object"); + } + if (settings.enabled !== true) { + throw new Error( + "GitHub Immutable Releases are disabled; enable them before starting a release", + ); + } + return { enabled: true }; +} + +export function verifyImmutableReleases({ + repository = process.env.GITHUB_REPOSITORY || "", + token = process.env.GH_TOKEN || "", + runner = spawnSync, +} = {}) { + if (!REPOSITORY_PATTERN.test(repository)) { + throw new Error("GITHUB_REPOSITORY must be an owner/repository name"); + } + if (!token.trim()) { + throw new Error( + "IMMUTABLE_RELEASES_READ_TOKEN is missing; release preflight cannot verify repository settings", + ); + } + + const result = runner( + "gh", + [ + "api", + "-H", + `X-GitHub-Api-Version: ${API_VERSION}`, + `repos/${repository}/immutable-releases`, + ], + { + encoding: "utf8", + env: { ...process.env, GH_TOKEN: token }, + }, + ); + if (result.error || result.status !== 0) { + const detail = String(result.error?.message || result.stderr || result.stdout || "unknown error") + .replace(/\s+/g, " ") + .trim() + .slice(0, 500); + throw new Error( + `could not verify GitHub Immutable Releases with the read-only token: ${detail}`, + ); + } + + let settings; + try { + settings = JSON.parse(result.stdout); + } catch (error) { + throw new Error(`GitHub Immutable Releases response was not valid JSON: ${error.message}`); + } + return assertImmutableReleasesEnabled(settings); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + try { + verifyImmutableReleases(); + console.log("GitHub Immutable Releases are enabled"); + } catch (error) { + console.error(`::error::${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + } +} diff --git a/scripts/check-release-reuse.mjs b/scripts/check-release-reuse.mjs new file mode 100644 index 0000000..fa84e54 --- /dev/null +++ b/scripts/check-release-reuse.mjs @@ -0,0 +1,119 @@ +#!/usr/bin/env node + +import { appendFileSync, readFileSync } from "node:fs"; + +const RELEASE_TAG_PATTERN = + /^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/; + +export function requiredReleaseAssetNames(releaseTag) { + const tag = String(releaseTag); + if (!RELEASE_TAG_PATTERN.test(tag)) { + throw new Error(`release tag must be a semantic vX.Y.Z tag: ${releaseTag}`); + } + const version = tag.slice(1); + return [ + "latest.json", + "CodexAppManager_aarch64.dmg", + "CodexAppManager_x86_64.dmg", + "CodexAppManager_aarch64.app.tar.gz", + "CodexAppManager_aarch64.app.tar.gz.sig", + "CodexAppManager_x86_64.app.tar.gz", + "CodexAppManager_x86_64.app.tar.gz.sig", + `CodexAppManager_${version}_x64-setup.exe`, + `CodexAppManager_${version}_x64-setup.exe.sig`, + `CodexAppManager_${version}_arm64-setup.exe`, + `CodexAppManager_${version}_arm64-setup.exe.sig`, + ]; +} + +export function inspectReleaseForReuse(release, releaseTag) { + if (!release || typeof release !== "object" || Array.isArray(release)) { + throw new Error("GitHub release response must be a JSON object"); + } + const required = requiredReleaseAssetNames(releaseTag); + const assets = Array.isArray(release.assets) ? release.assets : []; + const missing = []; + for (const name of required) { + const matches = assets.filter((asset) => asset?.name === name); + if (matches.length !== 1 || !Number.isFinite(matches[0]?.size) || matches[0].size <= 0) { + missing.push(name); + } + } + + // Drafts are the only repairable state: their assets may still be replaced + // before publication. Never route a published Release back through upload. + if (release.draft !== false) { + return { + digests: {}, + missing, + reason: "draft", + reusable: false, + }; + } + if (release.immutable !== true) { + throw new Error( + `existing release ${releaseTag} is mutable; refusing to treat its current assets as canonical`, + ); + } + if (missing.length > 0) { + throw new Error( + `existing immutable release ${releaseTag} is missing required assets and cannot be repaired: ${missing.join(", ")}`, + ); + } + + const selectedAssets = assets.filter( + (asset) => asset?.name === "latest.json" || asset?.name?.startsWith("CodexAppManager"), + ); + const selectedNames = new Set(); + const digests = {}; + for (const asset of selectedAssets) { + const name = asset.name; + if (selectedNames.has(name)) { + throw new Error(`immutable release has duplicate downloadable asset names: ${name}`); + } + selectedNames.add(name); + if (!Number.isFinite(asset.size) || asset.size <= 0) { + throw new Error(`immutable release downloadable asset is empty: ${name}`); + } + const digest = asset.digest; + if (typeof digest !== "string" || !/^sha256:[0-9a-f]{64}$/.test(digest)) { + throw new Error(`immutable release asset has no canonical SHA-256 digest: ${name}`); + } + digests[name] = digest; + } + return { digests, missing: [], reason: null, reusable: true }; +} + +const isCli = process.argv[1]?.endsWith("check-release-reuse.mjs"); +if (isCli) { + const [, , releaseTag, releaseJsonPath] = process.argv; + const requestedTargetTag = process.env.REQUESTED_TARGET_TAG || ""; + const outputPath = process.env.GITHUB_OUTPUT; + try { + if (!releaseTag || !releaseJsonPath || !outputPath) { + throw new Error( + "usage: check-release-reuse.mjs with GITHUB_OUTPUT set", + ); + } + const release = JSON.parse(readFileSync(releaseJsonPath, "utf8")); + const result = inspectReleaseForReuse(release, releaseTag); + if (!result.reusable && requestedTargetTag) { + throw new Error( + `target_tag ${requestedTargetTag} is draft or missing required release assets: ${result.missing.join(", ") || result.reason}`, + ); + } + appendFileSync(outputPath, `release_reusable=${result.reusable ? "true" : "false"}\n`); + if (result.reusable) { + appendFileSync(outputPath, `release_asset_digests=${JSON.stringify(result.digests)}\n`); + console.log(`Existing immutable release ${releaseTag} has canonical digests for all assets`); + } else { + console.log( + `Existing release ${releaseTag} is not reusable yet (${result.reason}; ${result.missing.join(", ")})`, + ); + } + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + console.error(`::error::${detail}`); + process.exitCode = 1; + } +} diff --git a/scripts/mirror-release.mjs b/scripts/mirror-release.mjs new file mode 100755 index 0000000..c4bda1f --- /dev/null +++ b/scripts/mirror-release.mjs @@ -0,0 +1,1800 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { + createHash, + createPublicKey, + randomUUID, + verify as cryptoVerify, +} from "node:crypto"; +import { createReadStream, createWriteStream } from "node:fs"; +import { + mkdir, + mkdtemp, + readFile, + readdir, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { fileURLToPath } from "node:url"; + +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(SCRIPT_DIR, ".."); +const DEFAULT_MIRROR_BASE = "https://codexapp.agentsmirror.com/manager"; +const LATEST_KEY = "latest.json"; +const SUMMARY_SCHEMA_VERSION = 1; +const ED25519_SPKI_PREFIX = Buffer.from("302a300506032b6570032100", "hex"); +export const REQUIRED_UPDATER_PLATFORMS = [ + "darwin-aarch64", + "darwin-x86_64", + "windows-x86_64", + "windows-aarch64", +]; + +let interruptedBy = null; +let rollbackInProgress = false; +const activeChildren = new Set(); + +export class ConditionalWriteError extends Error { + constructor(message) { + super(message); + this.name = "ConditionalWriteError"; + } +} + +export class DowngradeBlockedError extends Error { + constructor(message) { + super(message); + this.name = "DowngradeBlockedError"; + } +} + +class WriteOutcomeUncertainError extends Error { + constructor(message) { + super(message); + this.name = "WriteOutcomeUncertainError"; + } +} + +function errorText(error) { + if (error instanceof Error) return error.message; + return String(error); +} + +function nowIso() { + return new Date().toISOString(); +} + +function safeSummaryError(error) { + return errorText(error).replace(/\s+/g, " ").trim().slice(0, 2_000); +} + +function strictBase64(value, label) { + const normalized = String(value).trim(); + if ( + normalized.length === 0 || + normalized.length % 4 !== 0 || + !/^[A-Za-z0-9+/]+={0,2}$/.test(normalized) + ) { + throw new Error(`${label} is not valid base64`); + } + const decoded = Buffer.from(normalized, "base64"); + if (decoded.toString("base64") !== normalized) { + throw new Error(`${label} is not canonical base64`); + } + return decoded; +} + +export function parseSemver(value) { + const text = String(value).trim().replace(/^v/, ""); + const match = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/.exec( + text, + ); + if (!match) throw new Error(`invalid semantic version: ${value}`); + const prerelease = match[4] ? match[4].split(".") : []; + for (const identifier of prerelease) { + if (/^\d+$/.test(identifier) && identifier.length > 1 && identifier.startsWith("0")) { + throw new Error(`invalid semantic version prerelease: ${value}`); + } + } + return { + raw: text, + major: BigInt(match[1]), + minor: BigInt(match[2]), + patch: BigInt(match[3]), + prerelease, + }; +} + +export function compareSemver(left, right) { + const a = parseSemver(left); + const b = parseSemver(right); + for (const key of ["major", "minor", "patch"]) { + if (a[key] < b[key]) return -1; + if (a[key] > b[key]) return 1; + } + if (a.prerelease.length === 0 && b.prerelease.length === 0) return 0; + if (a.prerelease.length === 0) return 1; + if (b.prerelease.length === 0) return -1; + const count = Math.max(a.prerelease.length, b.prerelease.length); + for (let index = 0; index < count; index += 1) { + const av = a.prerelease[index]; + const bv = b.prerelease[index]; + if (av === undefined) return -1; + if (bv === undefined) return 1; + if (av === bv) continue; + const aNumeric = /^\d+$/.test(av); + const bNumeric = /^\d+$/.test(bv); + if (aNumeric && bNumeric) return BigInt(av) < BigInt(bv) ? -1 : 1; + if (aNumeric !== bNumeric) return aNumeric ? -1 : 1; + return av < bv ? -1 : 1; + } + return 0; +} + +function assertSafeSegment(value, label) { + if (!/^[A-Za-z0-9][A-Za-z0-9._+-]{0,127}$/.test(value) || value.includes("..")) { + throw new Error(`${label} contains unsafe characters: ${value}`); + } + return value; +} + +function normalizeMirrorBase(value) { + const parsed = new URL(String(value || DEFAULT_MIRROR_BASE).replace(/\/$/, "")); + if (parsed.protocol !== "https:" || parsed.username || parsed.password || parsed.search || parsed.hash) { + throw new Error("MIRROR_BASE_URL must be an HTTPS origin/path without credentials, query, or fragment"); + } + return parsed.toString().replace(/\/$/, ""); +} + +export function candidateIdFromEnv(env = process.env) { + const explicit = env.MIRROR_CANDIDATE_ID?.trim(); + const fallback = `${env.GITHUB_RUN_ID || "local"}-${env.GITHUB_RUN_ATTEMPT || "1"}`; + return assertSafeSegment(explicit || fallback, "mirror candidate id"); +} + +export function candidateKeyFor(version, candidateId) { + const parsed = parseSemver(version).raw; + assertSafeSegment(parsed, "candidate version"); + assertSafeSegment(candidateId, "mirror candidate id"); + return `candidates/${parsed}/${candidateId}.json`; +} + +function promotionTokenForCandidate(candidateKey) { + return createHash("sha256").update(candidateKey).digest("hex"); +} + +async function readJson(path, label = path) { + let value; + try { + value = JSON.parse(await readFile(path, "utf8")); + } catch (error) { + throw new Error(`cannot read ${label}: ${errorText(error)}`); + } + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must contain a JSON object`); + } + return value; +} + +async function writeJson(path, value) { + await mkdir(dirname(resolve(path)), { recursive: true }); + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); +} + +async function sha256File(path) { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest("hex"); +} + +async function blake2bFile(path) { + const hash = createHash("blake2b512"); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest(); +} + +function parseUpdaterPublicKey(publicKeyBase64) { + const text = strictBase64(publicKeyBase64, "Tauri updater public key").toString("utf8"); + const lines = text.trim().split(/\r?\n/); + if (lines.length < 2) throw new Error("Tauri updater public key has invalid minisign encoding"); + const raw = strictBase64(lines[1], "minisign public key"); + if (raw.length !== 42 || !["Ed", "ED"].includes(raw.subarray(0, 2).toString("ascii"))) { + throw new Error("Tauri updater public key has unsupported minisign encoding"); + } + return { keyId: raw.subarray(2, 10), key: raw.subarray(10, 42) }; +} + +function parseUpdaterSignature(signatureBase64) { + const text = strictBase64(signatureBase64, "Tauri updater signature").toString("utf8"); + const lines = text.trim().split(/\r?\n/); + if (lines.length !== 4 || !lines[2].startsWith("trusted comment: ")) { + throw new Error("Tauri updater signature has invalid minisign encoding"); + } + const primary = strictBase64(lines[1], "minisign primary signature"); + const global = strictBase64(lines[3], "minisign global signature"); + if (primary.length !== 74 || global.length !== 64) { + throw new Error("Tauri updater signature has invalid minisign lengths"); + } + const algorithm = primary.subarray(0, 2).toString("ascii"); + if (!['Ed', 'ED'].includes(algorithm)) { + throw new Error(`unsupported minisign signature algorithm: ${algorithm}`); + } + return { + algorithm, + keyId: primary.subarray(2, 10), + signature: primary.subarray(10, 74), + trustedComment: lines[2].slice("trusted comment: ".length), + globalSignature: global, + }; +} + +export async function verifyTauriUpdaterSignature( + artifactPath, + signatureBase64, + publicKeyBase64, +) { + const publicKey = parseUpdaterPublicKey(publicKeyBase64); + const signature = parseUpdaterSignature(signatureBase64); + if (!publicKey.keyId.equals(signature.keyId)) { + throw new Error("Tauri updater signature key id does not match configured public key"); + } + const spki = Buffer.concat([ED25519_SPKI_PREFIX, publicKey.key]); + const key = createPublicKey({ key: spki, format: "der", type: "spki" }); + const payload = + signature.algorithm === "ED" + ? await blake2bFile(artifactPath) + : await readFile(artifactPath); + if (!cryptoVerify(null, payload, key, signature.signature)) { + throw new Error("Tauri updater artifact signature is invalid"); + } + const globalPayload = Buffer.concat([ + signature.signature, + Buffer.from(signature.trustedComment, "utf8"), + ]); + if (!cryptoVerify(null, globalPayload, key, signature.globalSignature)) { + throw new Error("Tauri updater trusted-comment signature is invalid"); + } + return true; +} + +function updaterArtifactsFromManifest(manifest, label) { + assertManifestShape(manifest, label); + const seenNames = new Set(); + return Object.entries(manifest.platforms) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([platform, entry]) => { + const parsed = new URL(entry.url); + const rawName = parsed.pathname.split("/").filter(Boolean).at(-1); + const name = rawName ? decodeURIComponent(rawName) : ""; + assertSafeSegment(name, `${label} platform ${platform} artifact name`); + if (seenNames.has(name)) { + throw new Error(`multiple updater platforms resolve to the same artifact: ${name}`); + } + seenNames.add(name); + return { name, platform, signature: entry.signature, url: entry.url }; + }); +} + +export async function verifyLocalUpdaterArtifacts({ + manifest, + distDir, + publicKey, +}) { + const dist = resolve(distDir); + const reports = []; + for (const artifact of updaterArtifactsFromManifest(manifest, "release latest.json")) { + const artifactPath = join(dist, artifact.name); + const sidecarPath = `${artifactPath}.sig`; + let sidecar; + try { + sidecar = (await readFile(sidecarPath, "utf8")).trim(); + } catch (error) { + throw new Error( + `local updater signature sidecar is missing for ${artifact.platform}: ${errorText(error)}`, + ); + } + if (sidecar !== artifact.signature) { + throw new Error( + `local updater signature sidecar does not match manifest for ${artifact.platform}`, + ); + } + const metadata = await stat(artifactPath).catch((error) => { + throw new Error( + `local updater artifact is missing for ${artifact.platform}: ${errorText(error)}`, + ); + }); + if (!metadata.isFile() || metadata.size <= 0) { + throw new Error(`local updater artifact is empty for ${artifact.platform}`); + } + await verifyTauriUpdaterSignature(artifactPath, artifact.signature, publicKey); + reports.push({ + name: artifact.name, + platform: artifact.platform, + sha256: await sha256File(artifactPath), + size: metadata.size, + }); + } + return { artifactCount: reports.length, artifacts: reports, verified: true }; +} + +function manifestReleaseIdentity(manifest) { + const platforms = Object.fromEntries( + Object.entries(manifest.platforms || {}) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([platform, entry]) => [ + platform, + { signature: entry?.signature || "", url: entry?.url || "" }, + ]), + ); + return JSON.stringify({ version: manifest.version, platforms }); +} + +function assertManifestShape(manifest, label) { + const version = parseSemver(manifest.version).raw; + if (!manifest.platforms || typeof manifest.platforms !== "object" || Array.isArray(manifest.platforms)) { + throw new Error(`${label} has no platforms object`); + } + const entries = Object.entries(manifest.platforms); + if (entries.length === 0) throw new Error(`${label} has no updater platforms`); + for (const [platform, entry] of entries) { + if (!entry || typeof entry.url !== "string" || typeof entry.signature !== "string") { + throw new Error(`${label} platform ${platform} is missing url/signature`); + } + if (!entry.url.trim()) { + throw new Error(`${label} platform ${platform} has an empty updater URL`); + } + if (!entry.signature.trim()) { + throw new Error(`${label} platform ${platform} has an empty updater signature`); + } + } + return version; +} + +function assertRequiredUpdaterPlatforms(manifest, label) { + const missing = REQUIRED_UPDATER_PLATFORMS.filter( + (platform) => !Object.hasOwn(manifest.platforms, platform), + ); + if (missing.length > 0) { + throw new Error(`${label} is missing required updater platforms: ${missing.join(", ")}`); + } +} + +export function assertCandidateMatchesRelease(candidateManifest, artifactManifest, releaseTag) { + const tag = String(releaseTag); + if (!tag.startsWith("v")) { + throw new Error(`release tag must start with v: ${releaseTag}`); + } + const expectedVersion = tag.slice(1); + if (parseSemver(expectedVersion).raw !== expectedVersion) { + throw new Error(`release tag must contain a canonical semantic version: ${releaseTag}`); + } + + assertManifestShape(candidateManifest, "candidate latest.json"); + assertManifestShape(artifactManifest, "artifact-derived latest.json"); + assertRequiredUpdaterPlatforms(candidateManifest, "candidate latest.json"); + assertRequiredUpdaterPlatforms(artifactManifest, "artifact-derived latest.json"); + if (candidateManifest.version !== expectedVersion) { + throw new Error( + `candidate latest.json version ${candidateManifest.version} does not match release tag ${releaseTag}`, + ); + } + if (artifactManifest.version !== expectedVersion) { + throw new Error( + `artifact-derived latest.json version ${artifactManifest.version} does not match release tag ${releaseTag}`, + ); + } + if (manifestReleaseIdentity(candidateManifest) !== manifestReleaseIdentity(artifactManifest)) { + throw new Error( + "candidate latest.json platforms/signatures do not match the artifact-derived manifest", + ); + } + return { + platformCount: Object.keys(candidateManifest.platforms).length, + version: expectedVersion, + }; +} + +export async function createMirrorManifest(distDir, mirrorBaseValue = DEFAULT_MIRROR_BASE) { + const dist = resolve(distDir); + const sourcePath = join(dist, "latest.json"); + const manifest = await readJson(sourcePath, "release latest.json"); + const version = assertManifestShape(manifest, "release latest.json"); + const mirrorBase = normalizeMirrorBase(mirrorBaseValue); + const rewritten = structuredClone(manifest); + for (const [platform, entry] of Object.entries(rewritten.platforms)) { + const original = new URL(entry.url); + const rawName = original.pathname.split("/").filter(Boolean).at(-1); + if (!rawName) throw new Error(`platform ${platform} URL has no artifact name`); + const name = decodeURIComponent(rawName); + assertSafeSegment(name, `platform ${platform} artifact name`); + entry.url = `${mirrorBase}/${encodeURIComponent(version)}/${encodeURIComponent(name)}`; + } + const outputPath = join(dist, "latest.mirror.json"); + await writeJson(outputPath, rewritten); + return { manifest: rewritten, mirrorBase, outputPath, version }; +} + +async function expectedArtifactsFromManifest(manifest, distDir, mirrorBase) { + const version = assertManifestShape(manifest, "mirror candidate"); + const expectedBase = `${normalizeMirrorBase(mirrorBase)}/${encodeURIComponent(version)}/`; + const updaterByName = new Map(); + for (const [platform, entry] of Object.entries(manifest.platforms).sort(([a], [b]) => a.localeCompare(b))) { + if (!entry.url.startsWith(expectedBase)) { + throw new Error(`platform ${platform} URL is not under ${expectedBase}`); + } + const parsed = new URL(entry.url); + const rawName = parsed.pathname.split("/").filter(Boolean).at(-1); + const name = rawName ? decodeURIComponent(rawName) : ""; + assertSafeSegment(name, `platform ${platform} artifact name`); + if (updaterByName.has(name)) { + throw new Error(`multiple updater platforms resolve to the same artifact: ${name}`); + } + updaterByName.set(name, { platform, signature: entry.signature }); + } + + for (const [name, updater] of updaterByName) { + const sidecarPath = join(resolve(distDir), `${name}.sig`); + let sidecar; + try { + sidecar = (await readFile(sidecarPath, "utf8")).trim(); + } catch (error) { + throw new Error( + `local updater signature sidecar is missing for ${updater.platform}: ${errorText(error)}`, + ); + } + if (sidecar !== updater.signature) { + throw new Error( + `local updater signature sidecar does not match manifest for ${updater.platform}`, + ); + } + } + + const artifacts = []; + const names = (await readdir(distDir)) + .filter((name) => !["latest.json", "latest.mirror.json"].includes(name)) + .sort(); + for (const name of names) { + assertSafeSegment(name, "release artifact name"); + const localPath = join(resolve(distDir), name); + const metadata = await stat(localPath); + if (!metadata.isFile()) continue; + const updater = updaterByName.get(name); + artifacts.push({ + platform: updater?.platform || null, + name, + key: `${version}/${name}`, + localPath, + size: metadata.size, + sha256: await sha256File(localPath), + signature: updater ? updater.signature : null, + }); + updaterByName.delete(name); + } + if (updaterByName.size > 0) { + throw new Error( + `local release artifacts are missing: ${[...updaterByName.keys()].join(", ")}`, + ); + } + return artifacts; +} + +function publicProbeUrl(value, candidateKey, backend) { + const parsed = new URL(value); + if (parsed.protocol !== "https:") { + throw new Error(`public mirror probe requires HTTPS: ${value}`); + } + parsed.searchParams.set( + "cam_probe", + createHash("sha256").update(candidateKey).digest("hex").slice(0, 24), + ); + parsed.searchParams.set("cam_backend", backend); + return parsed.toString(); +} + +async function downloadPublicObject(fetchImpl, url, destination, label, backend) { + await mkdir(dirname(destination), { recursive: true }); + let lastError; + for (let attempt = 1; attempt <= 5; attempt += 1) { + await rm(destination, { force: true }); + try { + let response = await fetchImpl(url, { + headers: { + "Cache-Control": "no-cache", + "User-Agent": "Codex-App-Manager release verifier", + }, + redirect: backend === "ihep" ? "manual" : "follow", + signal: AbortSignal.timeout(300_000), + }); + const routedBackend = response.headers.get("X-Codex-Mirror-Backend"); + if (routedBackend !== backend) { + await response.body?.cancel().catch(() => {}); + throw new Error( + `Worker reported backend ${routedBackend || "missing"}, expected ${backend}`, + ); + } + if (backend === "ihep") { + if (response.status !== 302) { + await response.body?.cancel().catch(() => {}); + throw new Error(`Worker IHEP probe returned HTTP ${response.status}`); + } + const location = response.headers.get("Location"); + const redirected = location ? new URL(location, url) : null; + if (!redirected || redirected.protocol !== "https:") { + throw new Error("Worker IHEP probe did not return a secure redirect"); + } + await response.body?.cancel().catch(() => {}); + response = await fetchImpl(redirected, { + headers: { "User-Agent": "Codex-App-Manager release verifier" }, + redirect: "follow", + signal: AbortSignal.timeout(300_000), + }); + } + if (!response.ok || !response.body) { + await response.body?.cancel().catch(() => {}); + throw new Error(`HTTP ${response.status}`); + } + await pipeline(Readable.fromWeb(response.body), createWriteStream(destination)); + return { path: destination, size: (await stat(destination)).size, url }; + } catch (error) { + await rm(destination, { force: true }); + lastError = error; + if (attempt < 5) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, attempt * 1_000)); + } + } + } + throw new Error(`${label} public ${backend} download failed: ${errorText(lastError)}`); +} + +async function verifyPublicMirrorBackend({ + backend, + candidateKey, + candidateManifest, + candidatePath, + expectedByPlatform, + mirrorBase, + publicKey, + publicDir, + fetchImpl, +}) { + const encodedCandidateKey = candidateKey + .split("/") + .map((segment) => encodeURIComponent(segment)) + .join("/"); + const candidateUrl = publicProbeUrl( + `${normalizeMirrorBase(mirrorBase)}/${encodedCandidateKey}`, + candidateKey, + backend, + ); + const downloadedCandidate = join(publicDir, backend, "candidate.json"); + const candidateResponse = await downloadPublicObject( + fetchImpl, + candidateUrl, + downloadedCandidate, + "mirror candidate", + backend, + ); + const [localCandidateHash, publicCandidateHash] = await Promise.all([ + sha256File(candidatePath), + sha256File(downloadedCandidate), + ]); + if (localCandidateHash !== publicCandidateHash) { + throw new Error(`public mirror ${backend} candidate bytes do not match this release run`); + } + const publicManifest = await readJson( + downloadedCandidate, + `public mirror ${backend} candidate`, + ); + if ( + manifestReleaseIdentity(publicManifest) !== + manifestReleaseIdentity(candidateManifest) + ) { + throw new Error( + `public mirror ${backend} candidate release identity does not match this release run`, + ); + } + + const artifactReports = []; + for (const updater of updaterArtifactsFromManifest(candidateManifest, "mirror candidate")) { + const expected = expectedByPlatform.get(updater.platform); + if (!expected || expected.name !== updater.name) { + throw new Error( + `public mirror ${backend} probe has no bound artifact for ${updater.platform}`, + ); + } + const publicPath = join(publicDir, backend, updater.name); + const response = await downloadPublicObject( + fetchImpl, + publicProbeUrl(updater.url, candidateKey, backend), + publicPath, + `mirror artifact ${updater.name}`, + backend, + ); + if (response.size !== expected.size) { + throw new Error( + `public mirror ${backend} size mismatch for ${updater.name}: ${response.size} != ${expected.size}`, + ); + } + const publicHash = await sha256File(publicPath); + if (publicHash !== expected.sha256) { + throw new Error(`public mirror ${backend} sha256 mismatch for ${updater.name}`); + } + await verifyTauriUpdaterSignature(publicPath, updater.signature, publicKey); + artifactReports.push({ + name: updater.name, + platform: updater.platform, + sha256: publicHash, + size: response.size, + url: updater.url, + }); + await rm(publicPath, { force: true }); + } + return { + artifactCount: artifactReports.length, + artifacts: artifactReports, + backend, + candidateSha256: publicCandidateHash, + candidateSize: candidateResponse.size, + candidateUrl, + verified: true, + }; +} + +export async function verifyPublicMirrorRoute({ + candidateKey, + candidateManifest, + candidatePath, + expectedArtifacts, + mirrorBase, + publicKey, + workDir, + fetchImpl = fetch, +}) { + const expectedByPlatform = new Map( + expectedArtifacts + .filter((artifact) => artifact.platform) + .map((artifact) => [artifact.platform, artifact]), + ); + const backends = {}; + for (const backend of ["r2", "ihep"]) { + backends[backend] = await verifyPublicMirrorBackend({ + backend, + candidateKey, + candidateManifest, + candidatePath, + expectedByPlatform, + mirrorBase, + publicKey, + publicDir: join(workDir, "public-route"), + fetchImpl, + }); + } + return { + artifactCount: updaterArtifactsFromManifest(candidateManifest, "mirror candidate").length, + backendCount: Object.keys(backends).length, + backends, + verified: Object.values(backends).every((report) => report.verified), + }; +} + +function isNotFound(text) { + return /(?:\b404\b|NoSuchKey|Not Found|does not exist)/i.test(text); +} + +function isConditionalFailure(text) { + return /(?:\b409\b|\b412\b|PreconditionFailed|ConditionalRequestConflict|pre-?condition|conditional request conflict)/i.test( + text, + ); +} + +async function runProcess(command, args, options = {}) { + if (interruptedBy && !rollbackInProgress) { + throw new Error(`release interrupted by ${interruptedBy}`); + } + return await new Promise((resolvePromise, rejectPromise) => { + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env, + stdio: ["ignore", "pipe", "pipe"], + }); + activeChildren.add(child); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + if (stdout.length < 1_000_000) stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + if (stderr.length < 1_000_000) stderr += chunk.toString(); + }); + child.on("error", (error) => { + activeChildren.delete(child); + rejectPromise(error); + }); + child.on("close", (code, signal) => { + activeChildren.delete(child); + const result = { code: code ?? 1, signal, stderr, stdout }; + if (code === 0 || options.allowFailure) resolvePromise(result); + else rejectPromise(new Error(`${command} failed (${code ?? signal}): ${stderr || stdout}`)); + }); + }); +} + +export class AwsObjectStore { + constructor({ name, endpoint, bucket, region, prefix = "", accessKeyId, secretAccessKey, configPath }) { + this.name = name; + this.endpoint = endpoint; + this.bucket = bucket; + this.region = region || "auto"; + this.prefix = prefix.replace(/^\/+|\/+$/g, ""); + this.configPath = configPath; + this.processEnv = { + ...process.env, + AWS_ACCESS_KEY_ID: accessKeyId, + AWS_SECRET_ACCESS_KEY: secretAccessKey, + AWS_DEFAULT_REGION: this.region, + AWS_EC2_METADATA_DISABLED: "true", + AWS_CONFIG_FILE: configPath, + }; + } + + objectKey(key) { + return this.prefix ? `${this.prefix}/${key}` : key; + } + + s3Uri(key) { + return `s3://${this.bucket}/${this.objectKey(key)}`; + } + + async aws(args, options = {}) { + const { env: envOverride = {}, ...runOptions } = options; + return await runProcess( + "aws", + [...args, "--endpoint-url", this.endpoint], + { + ...runOptions, + env: { ...this.processEnv, ...envOverride }, + }, + ); + } + + async head(key) { + const result = await this.aws( + [ + "s3api", + "head-object", + "--bucket", + this.bucket, + "--key", + this.objectKey(key), + "--output", + "json", + ], + { allowFailure: true }, + ); + if (result.code !== 0) { + const detail = `${result.stderr}\n${result.stdout}`; + if (isNotFound(detail)) return null; + throw new Error(`${this.name}: cannot read object metadata for ${key}: ${detail.trim()}`); + } + const metadata = JSON.parse(result.stdout); + if (!metadata.ETag || !Number.isFinite(metadata.ContentLength)) { + throw new Error(`${this.name}: incomplete object metadata for ${key}`); + } + const objectMetadata = Object.fromEntries( + Object.entries(metadata.Metadata || {}).map(([name, value]) => [ + name.toLowerCase(), + String(value), + ]), + ); + return { etag: metadata.ETag, metadata: objectMetadata, size: metadata.ContentLength }; + } + + async download(key, destination) { + await mkdir(dirname(destination), { recursive: true }); + await this.aws([ + "s3", + "cp", + this.s3Uri(key), + destination, + "--only-show-errors", + ]); + } + + async snapshot(key, destination) { + for (let attempt = 1; attempt <= 3; attempt += 1) { + const before = await this.head(key); + if (!before) return { exists: false, etag: null, size: 0, path: null }; + await this.download(key, destination); + const after = await this.head(key); + const localSize = (await stat(destination)).size; + if (after && before.etag === after.etag && after.size === localSize) { + return { + exists: true, + etag: after.etag, + metadata: after.metadata, + size: localSize, + path: destination, + }; + } + await rm(destination, { force: true }); + if (attempt === 3) { + throw new Error(`${this.name}: ${key} changed repeatedly while being downloaded`); + } + } + throw new Error(`${this.name}: could not snapshot ${key}`); + } + + async putObject( + localPath, + key, + { contentType, ifMatch, ifNoneMatch, metadata, singleAttempt = false } = {}, + ) { + const args = [ + "s3api", + "put-object", + "--bucket", + this.bucket, + "--key", + this.objectKey(key), + "--body", + localPath, + "--content-type", + contentType || "application/octet-stream", + "--output", + "json", + ]; + if (ifMatch) args.push("--if-match", ifMatch); + if (ifNoneMatch) args.push("--if-none-match", ifNoneMatch); + if (metadata && Object.keys(metadata).length > 0) { + args.push( + "--metadata", + Object.entries(metadata) + .map(([name, value]) => `${name}=${value}`) + .join(","), + ); + } + const result = await this.aws(args, { + allowFailure: true, + ...(singleAttempt ? { env: { AWS_MAX_ATTEMPTS: "1" } } : {}), + }); + if (result.code !== 0) { + const detail = `${result.stderr}\n${result.stdout}`.trim(); + if (isConditionalFailure(detail)) { + throw new ConditionalWriteError(`${this.name}: conditional write rejected for ${key}`); + } + throw new Error(`${this.name}: upload failed for ${key}: ${detail}`); + } + let response; + try { + response = JSON.parse(result.stdout); + } catch (error) { + throw new WriteOutcomeUncertainError( + `${this.name}: ${key} upload succeeded but its response was not valid JSON: ${errorText(error)}`, + ); + } + if (typeof response.ETag !== "string" || !response.ETag) { + throw new WriteOutcomeUncertainError( + `${this.name}: ${key} upload response did not include the committed ETag`, + ); + } + return { etag: response.ETag, size: (await stat(localPath)).size }; + } + + async putImmutable(localPath, key, contentType, metadata = {}) { + const existing = await this.head(key); + if (existing) return { status: "existing", ...existing }; + try { + const uploaded = await this.putObject(localPath, key, { + contentType, + ifNoneMatch: "*", + metadata, + }); + return { status: "uploaded", ...uploaded }; + } catch (error) { + if (!(error instanceof ConditionalWriteError)) throw error; + const raced = await this.head(key); + if (!raced) throw error; + return { status: "existing-after-race", ...raced }; + } + } + + async putLatestConditional(localPath, expectedEtag, promotionToken = "") { + return await this.putObject(localPath, LATEST_KEY, { + contentType: "application/json", + ...(expectedEtag ? { ifMatch: expectedEtag } : { ifNoneMatch: "*" }), + ...(promotionToken + ? { metadata: { "cam-promotion-token": promotionToken } } + : {}), + singleAttempt: true, + }); + } +} + +function contentType(name) { + if (name.endsWith(".json")) return "application/json"; + if (name.endsWith(".tar.gz")) return "application/gzip"; + if (name.endsWith(".dmg")) return "application/x-apple-diskimage"; + return "application/octet-stream"; +} + +function requiredValue(env, name, backendName) { + const value = env[name]?.trim(); + if (!value) throw new Error(`${backendName} backend requires ${name}`); + return value; +} + +export function backendConfigsFromEnv(env, configPath) { + const required = (env.MIRROR_REQUIRED_BACKENDS || "r2,ihep") + .split(",") + .map((name) => name.trim().toLowerCase()) + .filter(Boolean); + const unknown = required.filter((name) => !["r2", "ihep"].includes(name)); + if (unknown.length > 0) throw new Error(`unknown MIRROR_REQUIRED_BACKENDS: ${unknown.join(", ")}`); + if (!required.includes("r2") || !required.includes("ihep")) { + throw new Error("stable mirror publication requires both r2 and ihep backends"); + } + return [ + new AwsObjectStore({ + name: "r2", + endpoint: requiredValue(env, "MANAGER_R2_S3_ENDPOINT", "r2"), + bucket: env.MANAGER_R2_BUCKET?.trim() || "codex-app-manager", + region: "auto", + accessKeyId: requiredValue(env, "MANAGER_R2_ACCESS_KEY_ID", "r2"), + secretAccessKey: requiredValue(env, "MANAGER_R2_SECRET_ACCESS_KEY", "r2"), + configPath, + }), + new AwsObjectStore({ + name: "ihep", + endpoint: requiredValue(env, "MANAGER_IHEP_S3_ENDPOINT", "ihep"), + bucket: requiredValue(env, "MANAGER_IHEP_S3_BUCKET", "ihep"), + region: env.MANAGER_IHEP_S3_REGION?.trim() || "auto", + prefix: env.MANAGER_IHEP_S3_PREFIX?.trim() || "", + accessKeyId: requiredValue(env, "MANAGER_IHEP_S3_ACCESS_KEY_ID", "ihep"), + secretAccessKey: requiredValue(env, "MANAGER_IHEP_S3_SECRET_ACCESS_KEY", "ihep"), + configPath, + }), + ]; +} + +export function downgradeOverrideFromEnv(env = process.env) { + const requested = env.MIRROR_ALLOW_DOWNGRADE === "1" || env.MIRROR_ALLOW_DOWNGRADE === "true"; + const reason = env.MIRROR_DOWNGRADE_REASON?.trim() || ""; + const eventName = env.GITHUB_EVENT_NAME || ""; + const originalActor = env.GITHUB_ACTOR || ""; + const triggeringActor = env.GITHUB_TRIGGERING_ACTOR || ""; + const actor = triggeringActor || originalActor; + const repository = env.GITHUB_REPOSITORY || ""; + const runId = env.GITHUB_RUN_ID || ""; + const workflowRefName = env.MIRROR_WORKFLOW_REF_NAME || ""; + const defaultBranch = env.MIRROR_DEFAULT_BRANCH || ""; + const runUrl = repository && runId ? `https://github.com/${repository}/actions/runs/${runId}` : ""; + if (requested) { + if (eventName !== "workflow_dispatch") { + throw new Error("mirror downgrade override is accepted only from workflow_dispatch"); + } + if (!workflowRefName || !defaultBranch || workflowRefName !== defaultBranch) { + throw new Error("mirror downgrade override must run from the repository default branch"); + } + if (reason.length < 10) { + throw new Error("mirror downgrade override requires an audit reason of at least 10 characters"); + } + if (!actor || !runUrl) { + throw new Error("mirror downgrade override requires GitHub actor and run audit metadata"); + } + } + return { + actor, + defaultBranch, + eventName, + originalActor, + reason, + requested, + runUrl, + used: false, + triggeringActor, + workflowRefName, + }; +} + +async function updaterPublicKeyFromRepo() { + const config = await readJson(join(REPO_ROOT, "src-tauri", "tauri.conf.json"), "tauri.conf.json"); + const pubkey = config.plugins?.updater?.pubkey; + if (typeof pubkey !== "string" || !pubkey.trim()) { + throw new Error("tauri.conf.json has no updater public key"); + } + return pubkey.trim(); +} + +export async function verifyBackendCandidate({ + backend, + candidateKey, + candidatePath, + candidateManifest, + distDir, + mirrorBase, + publicKey, + workDir, + expectedArtifacts: preparedArtifacts, + candidateSha256, + promotionToken, +}) { + const backendDir = join(workDir, backend.name); + await mkdir(backendDir, { recursive: true }); + const downloadedCandidate = join(backendDir, "candidate.json"); + const remoteCandidate = await backend.snapshot(candidateKey, downloadedCandidate); + if (!remoteCandidate.exists) throw new Error(`${backend.name}: candidate is missing at ${candidateKey}`); + if ( + promotionToken && + remoteCandidate.metadata?.["cam-promotion-token"] !== promotionToken + ) { + throw new Error( + `${backend.name}: candidate metadata does not preserve the promotion token`, + ); + } + const [localCandidateHash, remoteCandidateHash] = await Promise.all([ + candidateSha256 ? Promise.resolve(candidateSha256) : sha256File(candidatePath), + sha256File(downloadedCandidate), + ]); + if (localCandidateHash !== remoteCandidateHash) { + throw new Error(`${backend.name}: candidate bytes do not match this release run`); + } + const remoteManifest = await readJson(downloadedCandidate, `${backend.name} candidate`); + const expectedVersion = assertManifestShape(candidateManifest, "local mirror candidate"); + const remoteVersion = assertManifestShape(remoteManifest, `${backend.name} candidate`); + if (remoteVersion !== expectedVersion || manifestReleaseIdentity(remoteManifest) !== manifestReleaseIdentity(candidateManifest)) { + throw new Error(`${backend.name}: candidate release identity does not match version ${expectedVersion}`); + } + + const expectedArtifacts = + preparedArtifacts || + (await expectedArtifactsFromManifest(candidateManifest, distDir, mirrorBase)); + const artifactReports = []; + for (const artifact of expectedArtifacts) { + const remotePath = join(backendDir, artifact.name); + const remote = await backend.snapshot(artifact.key, remotePath); + if (!remote.exists) throw new Error(`${backend.name}: artifact is missing: ${artifact.key}`); + const remoteHash = await sha256File(remotePath); + if (remote.size !== artifact.size) { + throw new Error( + `${backend.name}: size mismatch for ${artifact.name}: ${remote.size} != ${artifact.size}`, + ); + } + if (remoteHash !== artifact.sha256) { + throw new Error(`${backend.name}: sha256 mismatch for ${artifact.name}`); + } + if (artifact.platform) { + if (!artifact.signature) { + throw new Error(`${backend.name}: updater artifact ${artifact.name} has no signature`); + } + await verifyTauriUpdaterSignature(remotePath, artifact.signature, publicKey); + } + artifactReports.push({ + name: artifact.name, + platform: artifact.platform, + sha256: remoteHash, + signatureVerified: artifact.platform ? true : null, + size: remote.size, + }); + await rm(remotePath, { force: true }); + } + return { + artifactCount: artifactReports.length, + artifacts: artifactReports, + candidateEtag: remoteCandidate.etag, + candidateSha256: remoteCandidateHash, + verified: true, + version: remoteVersion, + }; +} + +function initialSummary(phase, version, candidateKey, override) { + return { + schemaVersion: SUMMARY_SCHEMA_VERSION, + phase, + candidateVersion: version, + candidateKey, + startedAt: nowIso(), + finishedAt: null, + outcome: "running", + error: null, + override: { ...override }, + backends: [], + }; +} + +async function finishSummary(path, summary, outcome, error = null) { + summary.finishedAt = nowIso(); + summary.outcome = outcome; + summary.error = error ? safeSummaryError(error) : null; + await writeJson(path, summary); +} + +export async function stageMirrors({ + backends, + candidateKey, + candidatePath, + distDir, + summaryPath, + version, + override, +}) { + const summary = initialSummary("stage", version, candidateKey, override); + summary.backends = backends.map((backend) => ({ + name: backend.name, + candidate: null, + assets: [], + status: "not-started", + })); + const promotionToken = promotionTokenForCandidate(candidateKey); + let activeSummary = null; + try { + const names = (await readdir(distDir)).sort(); + const assets = names.filter((name) => !["latest.json", "latest.mirror.json"].includes(name)); + for (const backend of backends) { + const backendSummary = summary.backends.find((entry) => entry.name === backend.name); + activeSummary = backendSummary; + backendSummary.status = "staging"; + for (const name of assets) { + assertSafeSegment(name, "release artifact name"); + const localPath = join(distDir, name); + const metadata = await stat(localPath); + if (!metadata.isFile()) continue; + const uploaded = await backend.putImmutable( + localPath, + `${version}/${name}`, + contentType(name), + ); + backendSummary.assets.push({ name, status: uploaded.status }); + } + const candidate = await backend.putImmutable( + candidatePath, + candidateKey, + "application/json", + { "cam-promotion-token": promotionToken }, + ); + backendSummary.candidate = { key: candidateKey, status: candidate.status }; + backendSummary.status = "staged"; + } + await finishSummary(summaryPath, summary, "staged"); + return summary; + } catch (error) { + if (activeSummary?.status === "staging") { + activeSummary.status = "failed"; + activeSummary.error = safeSummaryError(error); + } + await finishSummary(summaryPath, summary, "failed", error); + throw error; + } +} + +async function readCurrentState(backend, workDir, candidateManifest) { + const path = join(workDir, `${backend.name}-previous-latest.json`); + const snapshot = await backend.snapshot(LATEST_KEY, path); + if (!snapshot.exists) { + throw new Error( + `${backend.name}: latest.json is absent; seed both backends with the same valid baseline before enabling transactional promotion`, + ); + } + const manifest = await readJson(path, `${backend.name} current latest.json`); + const currentVersion = assertManifestShape(manifest, `${backend.name} current latest.json`); + const candidateVersion = assertManifestShape(candidateManifest, "candidate latest.json"); + const comparison = compareSemver(currentVersion, candidateVersion); + let decision; + if (comparison < 0) decision = "promote-forward"; + else if (comparison > 0) decision = "blocked-downgrade"; + else if (manifestReleaseIdentity(manifest) === manifestReleaseIdentity(candidateManifest)) { + decision = "idempotent"; + } else { + decision = "blocked-same-version-mismatch"; + } + return { + backend, + currentIdentity: manifestReleaseIdentity(manifest), + currentManifest: manifest, + currentVersion, + decision, + previous: snapshot, + }; +} + +async function snapshotMatchesFile(snapshot, expectedPath) { + if (!snapshot.exists) return false; + const [actual, expected] = await Promise.all([ + sha256File(snapshot.path), + sha256File(expectedPath), + ]); + return actual === expected; +} + +async function observeAmbiguousWrite( + state, + candidatePath, + workDir, + promotionToken, +) { + try { + const observed = await state.backend.snapshot( + LATEST_KEY, + join(workDir, `${state.backend.name}-ambiguous-latest.json`), + ); + if ( + observed.metadata?.["cam-promotion-token"] === promotionToken && + (await snapshotMatchesFile(observed, candidatePath)) + ) { + return observed; + } + } catch { + // The original write error remains the primary failure. Rollback reporting + // will make any uncertainty visible in the audit summary. + } + return null; +} + +async function withRollbackIo(callback) { + const previous = rollbackInProgress; + rollbackInProgress = true; + try { + return await callback(); + } finally { + rollbackInProgress = previous; + } +} + +async function rollbackTouched(touched, summaryByName, workDir) { + const failures = []; + rollbackInProgress = true; + try { + for (const item of [...touched].reverse()) { + const backendSummary = summaryByName.get(item.state.backend.name); + try { + const owned = await item.state.backend.snapshot( + LATEST_KEY, + join(workDir, `${item.state.backend.name}-rollback-ownership.json`), + ); + if ( + !owned.exists || + owned.etag !== item.promotedEtag || + owned.metadata?.["cam-promotion-token"] !== item.promotionToken + ) { + throw new ConditionalWriteError( + `${item.state.backend.name}: latest.json is no longer owned by this promotion`, + ); + } + if (item.state.previous.exists) { + const restored = await item.state.backend.putLatestConditional( + item.state.previous.path, + item.promotedEtag, + `rollback-${randomUUID()}`, + ); + const check = await item.state.backend.snapshot( + LATEST_KEY, + join(workDir, `${item.state.backend.name}-rollback-check.json`), + ); + if (!(await snapshotMatchesFile(check, item.state.previous.path))) { + throw new Error("rollback verification did not match previous latest.json"); + } + item.promotedEtag = restored.etag; + } else { + throw new Error("transactional promotion cannot roll back an unseeded backend"); + } + backendSummary.rollback = "restored"; + } catch (error) { + backendSummary.rollback = "failed"; + backendSummary.rollbackError = safeSummaryError(error); + failures.push(`${item.state.backend.name}: ${errorText(error)}`); + } + } + } finally { + rollbackInProgress = false; + } + return failures; +} + +export async function promoteCandidateTransaction({ + backends, + candidateManifest, + candidatePath, + override, + summary, + workDir, + hooks = {}, + promotionToken = randomUUID(), +}) { + const summaryByName = new Map(summary.backends.map((entry) => [entry.name, entry])); + const states = []; + for (const backend of backends) { + try { + states.push(await readCurrentState(backend, workDir, candidateManifest)); + } catch (error) { + const backendSummary = summaryByName.get(backend.name); + if (backendSummary) backendSummary.error = safeSummaryError(error); + throw error; + } + } + await hooks.afterSnapshots?.(states); + + for (const state of states) { + const backendSummary = summaryByName.get(state.backend.name); + backendSummary.currentVersion = state.currentVersion; + backendSummary.decision = state.decision; + } + + const mismatch = states.find((state) => state.decision === "blocked-same-version-mismatch"); + if (mismatch) { + throw new Error( + `${mismatch.backend.name}: current latest has the candidate version but different artifact/signature identity`, + ); + } + const downgradeStates = states.filter((state) => state.decision === "blocked-downgrade"); + if (downgradeStates.length > 0 && !override.requested) { + throw new DowngradeBlockedError( + `candidate ${candidateManifest.version} is older than ${downgradeStates + .map((state) => `${state.backend.name}=${state.currentVersion}`) + .join(", ")}`, + ); + } + if (downgradeStates.length > 0) { + override.used = true; + summary.override.used = true; + for (const state of downgradeStates) { + state.decision = "promote-downgrade-override"; + summaryByName.get(state.backend.name).decision = state.decision; + } + } + + const toPromote = states.filter((state) => state.decision !== "idempotent"); + if (toPromote.length === 0) { + for (const state of states) { + const backendSummary = summaryByName.get(state.backend.name); + try { + const finalSnapshot = await state.backend.snapshot( + LATEST_KEY, + join(workDir, `${state.backend.name}-idempotent-final.json`), + ); + if (!finalSnapshot.exists || finalSnapshot.etag !== state.previous.etag) { + throw new ConditionalWriteError( + `${state.backend.name}: latest.json changed after the idempotent snapshot`, + ); + } + const finalManifest = await readJson( + finalSnapshot.path, + `${state.backend.name} idempotent final latest.json`, + ); + if ( + manifestReleaseIdentity(finalManifest) !== + manifestReleaseIdentity(candidateManifest) + ) { + throw new ConditionalWriteError( + `${state.backend.name}: latest.json no longer exposes the idempotent candidate`, + ); + } + backendSummary.finalVersion = finalManifest.version; + } catch (error) { + backendSummary.error = safeSummaryError(error); + throw error; + } + } + return { outcome: "idempotent", states }; + } + + const touched = []; + let activeState = null; + try { + for (const state of toPromote) { + activeState = state; + const backendSummary = summaryByName.get(state.backend.name); + await hooks.beforeWrite?.(state); + let promoted; + try { + promoted = await state.backend.putLatestConditional( + candidatePath, + state.previous.etag, + promotionToken, + ); + } catch (error) { + if (error instanceof ConditionalWriteError) { + throw error; + } + const ambiguous = await withRollbackIo(() => + observeAmbiguousWrite( + state, + candidatePath, + workDir, + promotionToken, + ), + ); + if (ambiguous) { + touched.push({ + state, + promotedEtag: ambiguous.etag, + promotionToken, + }); + backendSummary.promotion = "write-outcome-ambiguous"; + } else { + backendSummary.promotion = "write-outcome-unresolved"; + backendSummary.rollback = "uncertain"; + throw new WriteOutcomeUncertainError( + `${state.backend.name}: write failed and ownership could not be resolved after ${errorText(error)}`, + ); + } + throw error; + } + const touchedItem = { + state, + promotedEtag: promoted.etag, + promotionToken, + }; + touched.push(touchedItem); + backendSummary.promotion = "written"; + const check = await state.backend.snapshot( + LATEST_KEY, + join(workDir, `${state.backend.name}-promoted-check.json`), + ); + if ( + check.etag !== promoted.etag || + check.metadata?.["cam-promotion-token"] !== promotionToken + ) { + touched.pop(); + backendSummary.promotion = "ownership-lost"; + backendSummary.rollback = "skipped-concurrent-change"; + throw new ConditionalWriteError( + `${state.backend.name}: latest.json changed after the conditional write`, + ); + } + if (!(await snapshotMatchesFile(check, candidatePath))) { + throw new Error(`${state.backend.name}: promoted latest.json failed byte verification`); + } + backendSummary.promotion = "verified"; + await hooks.afterWrite?.(state); + } + + for (const state of states) { + activeState = state; + const finalSnapshot = await state.backend.snapshot( + LATEST_KEY, + join(workDir, `${state.backend.name}-final-latest.json`), + ); + if (!finalSnapshot.exists) throw new Error(`${state.backend.name}: latest.json disappeared`); + const finalManifest = await readJson( + finalSnapshot.path, + `${state.backend.name} final latest.json`, + ); + if (manifestReleaseIdentity(finalManifest) !== manifestReleaseIdentity(candidateManifest)) { + throw new Error(`${state.backend.name}: final latest.json does not expose the candidate release`); + } + summaryByName.get(state.backend.name).finalVersion = finalManifest.version; + } + return { outcome: override.used ? "downgrade-override-promoted" : "promoted", states }; + } catch (error) { + if (activeState) { + summaryByName.get(activeState.backend.name).error = safeSummaryError(error); + } + const rollbackFailures = await rollbackTouched(touched, summaryByName, workDir); + const ownershipUncertain = error instanceof WriteOutcomeUncertainError; + const failures = ownershipUncertain + ? [...rollbackFailures, `${activeState?.backend.name || "backend"}: write ownership unresolved`] + : rollbackFailures; + summary.rollback = { + attempted: touched.length > 0, + complete: failures.length === 0, + failures, + }; + if (failures.length > 0) { + throw new Error( + `${errorText(error)}; rollback incomplete: ${failures.join("; ")}`, + ); + } + if (touched.length > 0) summary.outcome = "rolled-back"; + throw error; + } +} + +function mirrorVerificationRows(backends, includeTransaction = false) { + return backends.map((backend) => ({ + name: backend.name, + candidateVerification: "not-started", + ...(includeTransaction + ? { + currentVersion: null, + decision: null, + promotion: "not-started", + rollback: "not-needed", + } + : {}), + })); +} + +async function verifyMirrorCandidates({ + backends, + candidateKey, + candidateManifest, + candidatePath, + distDir, + mirrorBase, + publicKey, + summary, + tempRoot, + fetchImpl = fetch, +}) { + const [expectedArtifacts, candidateSha256] = await Promise.all([ + expectedArtifactsFromManifest(candidateManifest, distDir, mirrorBase), + sha256File(candidatePath), + ]); + const promotionToken = promotionTokenForCandidate(candidateKey); + + // Both storage origins must finish a complete candidate + artifact readback + // before any mutable latest.json write. Verification is sequential to cap + // runner disk use. + for (const backend of backends) { + const backendSummary = summary.backends.find((entry) => entry.name === backend.name); + backendSummary.candidateVerification = "running"; + try { + backendSummary.candidate = await verifyBackendCandidate({ + backend, + candidateKey, + candidatePath, + candidateManifest, + distDir, + mirrorBase, + publicKey, + workDir: join(tempRoot, "backend"), + expectedArtifacts, + candidateSha256, + promotionToken, + }); + backendSummary.candidateVerification = "passed"; + } catch (error) { + backendSummary.candidateVerification = "failed"; + backendSummary.error = safeSummaryError(error); + throw error; + } + } + + // Direct S3 readback does not prove that the Worker route used by clients is + // healthy or bound to the expected bucket. Fetch the run-specific candidate + // and every updater payload through their real public HTTPS URLs as a separate + // fail-closed gate. + summary.publicRouteVerification = "running"; + try { + summary.publicRoute = await verifyPublicMirrorRoute({ + candidateKey, + candidateManifest, + candidatePath, + expectedArtifacts, + mirrorBase, + publicKey, + workDir: join(tempRoot, "public"), + fetchImpl, + }); + summary.publicRouteVerification = "passed"; + } catch (error) { + summary.publicRouteVerification = "failed"; + summary.publicRouteError = safeSummaryError(error); + throw error; + } + + return { candidateSha256, expectedArtifacts, promotionToken }; +} + +export async function verifyMirrors({ + backends, + candidateKey, + candidateManifest, + candidatePath, + distDir, + mirrorBase, + override, + publicKey, + summaryPath, + tempRoot, + fetchImpl, +}) { + const version = assertManifestShape(candidateManifest, "candidate latest.json"); + assertRequiredUpdaterPlatforms(candidateManifest, "candidate latest.json"); + const summary = initialSummary("verify", version, candidateKey, override); + summary.backends = mirrorVerificationRows(backends); + summary.publicRouteVerification = "not-started"; + try { + await verifyMirrorCandidates({ + backends, + candidateKey, + candidateManifest, + candidatePath, + distDir, + mirrorBase, + publicKey, + summary, + tempRoot, + fetchImpl, + }); + await finishSummary(summaryPath, summary, "verified"); + return summary; + } catch (error) { + await finishSummary(summaryPath, summary, "failed", error); + throw error; + } +} + +export async function promoteMirrors({ + backends, + candidateKey, + candidateManifest, + candidatePath, + distDir, + mirrorBase, + override, + publicKey, + summaryPath, + tempRoot, + hooks, + fetchImpl, +}) { + const version = assertManifestShape(candidateManifest, "candidate latest.json"); + assertRequiredUpdaterPlatforms(candidateManifest, "candidate latest.json"); + const summary = initialSummary("promote", version, candidateKey, override); + summary.backends = mirrorVerificationRows(backends, true); + summary.publicRouteVerification = "not-started"; + try { + const { promotionToken } = await verifyMirrorCandidates({ + backends, + candidateKey, + candidateManifest, + candidatePath, + distDir, + mirrorBase, + publicKey, + summary, + tempRoot: join(tempRoot, "verify"), + fetchImpl, + }); + + const transaction = await promoteCandidateTransaction({ + backends, + candidateManifest, + candidatePath, + override, + summary, + workDir: join(tempRoot, "transaction"), + hooks, + promotionToken, + }); + await finishSummary(summaryPath, summary, transaction.outcome); + return summary; + } catch (error) { + const outcome = + error instanceof DowngradeBlockedError + ? "blocked-downgrade" + : summary.outcome === "rolled-back" + ? "rolled-back" + : "failed"; + await finishSummary(summaryPath, summary, outcome, error); + throw error; + } +} + +async function createAwsConfig(tempRoot) { + const path = join(tempRoot, "aws-config"); + await writeFile(path, "[default]\nregion = auto\ns3 =\n addressing_style = path\n"); + return path; +} + +function installSignalHandlers() { + const handler = (signal) => { + if (interruptedBy) return; + interruptedBy = signal; + for (const child of activeChildren) child.kill("SIGTERM"); + }; + process.on("SIGINT", handler); + process.on("SIGTERM", handler); + return () => { + process.off("SIGINT", handler); + process.off("SIGTERM", handler); + }; +} + +async function main() { + const distArg = process.argv[2]; + if (!distArg) throw new Error("usage: sync-mirror.sh "); + const distDir = resolve(distArg); + const phase = process.env.MIRROR_PHASE || "all"; + if (!["all", "stage", "verify", "promote"].includes(phase)) { + throw new Error( + `unsupported MIRROR_PHASE=${phase} (expected all, stage, verify, or promote)`, + ); + } + const override = downgradeOverrideFromEnv(process.env); + if (override.requested) { + console.log( + `::warning::[mirror] emergency downgrade override requested by ${safeSummaryError( + override.actor, + )}: ${safeSummaryError(override.reason)} (${override.runUrl})`, + ); + } + const candidateId = candidateIdFromEnv(process.env); + const mirror = await createMirrorManifest( + distDir, + process.env.MIRROR_BASE_URL || DEFAULT_MIRROR_BASE, + ); + const candidateKey = candidateKeyFor(mirror.version, candidateId); + const tempRoot = await mkdtemp(join(tmpdir(), "cam-mirror-release-")); + const removeSignalHandlers = installSignalHandlers(); + try { + const configPath = await createAwsConfig(tempRoot); + const backends = backendConfigsFromEnv(process.env, configPath); + if (phase === "all" || phase === "stage") { + const stageSummaryPath = resolve( + process.env.MIRROR_STAGE_SUMMARY_PATH || "mirror-stage-summary.json", + ); + console.log(`::group::[mirror] stage ${mirror.version} candidate=${candidateKey}`); + await stageMirrors({ + backends, + candidateKey, + candidatePath: mirror.outputPath, + distDir, + summaryPath: stageSummaryPath, + version: mirror.version, + override, + }); + console.log("::endgroup::"); + } + if (phase === "verify") { + const verificationSummaryPath = resolve( + process.env.MIRROR_VERIFICATION_SUMMARY_PATH || + "mirror-verification-summary.json", + ); + const publicKey = process.env.MIRROR_UPDATER_PUBLIC_KEY || (await updaterPublicKeyFromRepo()); + console.log(`::group::[mirror] verify ${mirror.version}`); + await verifyMirrors({ + backends, + candidateKey, + candidateManifest: mirror.manifest, + candidatePath: mirror.outputPath, + distDir, + mirrorBase: mirror.mirrorBase, + override, + publicKey, + summaryPath: verificationSummaryPath, + tempRoot, + }); + console.log("::endgroup::"); + } + if (phase === "all" || phase === "promote") { + const promotionSummaryPath = resolve( + process.env.MIRROR_PROMOTION_SUMMARY_PATH || "mirror-promotion-summary.json", + ); + const publicKey = process.env.MIRROR_UPDATER_PUBLIC_KEY || (await updaterPublicKeyFromRepo()); + console.log(`::group::[mirror] verify and promote ${mirror.version}`); + await promoteMirrors({ + backends, + candidateKey, + candidateManifest: mirror.manifest, + candidatePath: mirror.outputPath, + distDir, + mirrorBase: mirror.mirrorBase, + override, + publicKey, + summaryPath: promotionSummaryPath, + tempRoot, + }); + console.log("::endgroup::"); + } + } finally { + removeSignalHandlers(); + await rm(tempRoot, { force: true, recursive: true }); + } + if (interruptedBy) throw new Error(`release interrupted by ${interruptedBy}`); +} + +const isCli = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (isCli) { + main().catch((error) => { + console.error(`::error::[mirror] ${safeSummaryError(error)}`); + process.exitCode = interruptedBy === "SIGINT" ? 130 : interruptedBy === "SIGTERM" ? 143 : 1; + }); +} diff --git a/scripts/mirror-release.test.mjs b/scripts/mirror-release.test.mjs new file mode 100644 index 0000000..3039574 --- /dev/null +++ b/scripts/mirror-release.test.mjs @@ -0,0 +1,1345 @@ +import { + createHash, + generateKeyPairSync, + sign as cryptoSign, +} from "node:crypto"; +import { execFile } from "node:child_process"; +import { + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + AwsObjectStore, + ConditionalWriteError, + DowngradeBlockedError, + assertCandidateMatchesRelease, + candidateKeyFor, + compareSemver, + createMirrorManifest, + downgradeOverrideFromEnv, + promoteCandidateTransaction, + promoteMirrors, + verifyBackendCandidate, + verifyLocalUpdaterArtifacts, + verifyMirrors, + verifyPublicMirrorRoute, + verifyTauriUpdaterSignature, +} from "./mirror-release.mjs"; +import { + inspectReleaseForReuse, + requiredReleaseAssetNames, +} from "./check-release-reuse.mjs"; + +const roots = []; +const execFileAsync = promisify(execFile); + +async function tempRoot(name) { + const root = await mkdtemp(join(tmpdir(), `cam-${name}-`)); + roots.push(root); + return root; +} + +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { force: true, recursive: true }))); +}); + +function hash(body) { + return createHash("sha256").update(body).digest("hex"); +} + +function manifest(version, signature = "test-signature") { + return { + version, + notes: `release ${version}`, + pub_date: "2026-01-01T00:00:00.000Z", + platforms: { + "windows-x86_64": { + signature, + url: `https://mirror.example/manager/${version}/manager-${version}.exe`, + }, + }, + }; +} + +function completeManifest(version, signature = "test-signature") { + return { + version, + notes: `release ${version}`, + pub_date: "2026-01-01T00:00:00.000Z", + platforms: Object.fromEntries( + [ + "darwin-aarch64", + "darwin-x86_64", + "windows-x86_64", + "windows-aarch64", + ].map((platform) => [ + platform, + { + signature, + url: `https://mirror.example/manager/${version}/manager-${platform}-${version}.bin`, + }, + ]), + ), + }; +} + +function completeRelease(releaseTag, overrides = {}) { + const digest = `sha256:${"a".repeat(64)}`; + return { + draft: false, + immutable: true, + assets: requiredReleaseAssetNames(releaseTag).map((name) => ({ + digest, + name, + size: 1, + })), + ...overrides, + }; +} + +async function writeManifest(root, name, value) { + const path = join(root, name); + await mkdir(root, { recursive: true }); + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); + return path; +} + +function summaryFor(backends, override = overrideOff()) { + return { + backends: backends.map((backend) => ({ + name: backend.name, + candidateVerification: "passed", + promotion: "not-started", + rollback: "not-needed", + })), + outcome: "running", + override: { ...override }, + }; +} + +function overrideOff() { + return { + actor: "", + eventName: "push", + reason: "", + requested: false, + runUrl: "", + used: false, + }; +} + +class MemoryBackend { + constructor(name, objects = {}) { + this.name = name; + this.objects = new Map(); + this.counter = 0; + this.latestPutAttempts = 0; + for (const [key, body] of Object.entries(objects)) this.set(key, body); + } + + set(key, body, metadata = {}) { + const bytes = Buffer.isBuffer(body) ? Buffer.from(body) : Buffer.from(body); + this.counter += 1; + this.objects.set(key, { + body: bytes, + etag: `"${hash(bytes)}-${this.counter}"`, + metadata: { ...metadata }, + }); + } + + body(key) { + return this.objects.get(key)?.body; + } + + async head(key) { + const object = this.objects.get(key); + return object + ? { + etag: object.etag, + metadata: { ...object.metadata }, + size: object.body.length, + } + : null; + } + + async snapshot(key, destination) { + const object = this.objects.get(key); + if (!object) return { exists: false, etag: null, path: null, size: 0 }; + await mkdir(join(destination, ".."), { recursive: true }); + await writeFile(destination, object.body); + return { + exists: true, + etag: object.etag, + metadata: { ...object.metadata }, + path: destination, + size: object.body.length, + }; + } + + async putImmutable(localPath, key, _contentType, metadata = {}) { + const existing = this.objects.get(key); + if (existing) return { status: "existing", etag: existing.etag, size: existing.body.length }; + const body = await readFile(localPath); + this.set(key, body, metadata); + const stored = this.objects.get(key); + return { status: "uploaded", etag: stored.etag, size: stored.body.length }; + } + + async putLatestConditional(localPath, expectedEtag, promotionToken = "") { + this.latestPutAttempts += 1; + const current = this.objects.get("latest.json"); + const matches = expectedEtag ? current?.etag === expectedEtag : !current; + if (!matches) throw new ConditionalWriteError(`${this.name}: stale latest ETag`); + if (this.failLatestPut) throw new Error(`${this.name}: simulated write failure`); + this.set( + "latest.json", + await readFile(localPath), + promotionToken ? { "cam-promotion-token": promotionToken } : {}, + ); + const stored = this.objects.get("latest.json"); + return { etag: stored.etag, size: stored.body.length }; + } + + async deleteLatestConditional(expectedEtag) { + const current = this.objects.get("latest.json"); + if (!current || current.etag !== expectedEtag) { + throw new ConditionalWriteError(`${this.name}: stale delete ETag`); + } + this.objects.delete("latest.json"); + } +} + +function encodedUpdaterSignature(privateKey, keyId, artifact) { + const digest = createHash("blake2b512").update(artifact).digest(); + const signature = cryptoSign(null, digest, privateKey); + const trustedComment = "timestamp:1700000000\tfile:test-artifact"; + const globalSignature = cryptoSign( + null, + Buffer.concat([signature, Buffer.from(trustedComment)]), + privateKey, + ); + const primary = Buffer.concat([Buffer.from("ED"), keyId, signature]); + const text = [ + "untrusted comment: signature from test key", + primary.toString("base64"), + `trusted comment: ${trustedComment}`, + globalSignature.toString("base64"), + ].join("\n"); + return Buffer.from(`${text}\n`).toString("base64"); +} + +function updaterFixture(artifact) { + const { privateKey, publicKey } = generateKeyPairSync("ed25519"); + const publicDer = publicKey.export({ format: "der", type: "spki" }); + const rawPublicKey = publicDer.subarray(publicDer.length - 32); + const keyId = Buffer.from("0102030405060708", "hex"); + const minisignPublic = Buffer.concat([Buffer.from("Ed"), keyId, rawPublicKey]); + const publicText = [ + "untrusted comment: minisign public key: test", + minisignPublic.toString("base64"), + ].join("\n"); + return { + publicKey: Buffer.from(`${publicText}\n`).toString("base64"), + signature: encodedUpdaterSignature(privateKey, keyId, artifact), + }; +} + +function publicRouteFetch(objects, requests = [], { ihepObjects = objects } = {}) { + return async (value) => { + const url = new URL(value); + requests.push(url); + if (url.hostname === "ihep.example") { + const body = ihepObjects.get(url.pathname); + return body === undefined + ? new Response("not found", { status: 404 }) + : new Response(body, { status: 200 }); + } + const backend = url.searchParams.get("cam_backend"); + if (backend === "ihep") { + return new Response(null, { + status: 302, + headers: { + Location: `https://ihep.example${url.pathname}`, + "X-Codex-Mirror-Backend": "ihep", + }, + }); + } + const body = objects.get(url.pathname); + return body === undefined + ? new Response("not found", { status: 404 }) + : new Response(body, { + status: 200, + headers: { "X-Codex-Mirror-Backend": "r2" }, + }); + }; +} + +describe("semantic release ordering", () => { + it("orders stable and prerelease versions without lexical mistakes", () => { + expect(compareSemver("1.10.0", "1.9.9")).toBe(1); + expect(compareSemver("1.0.0-rc.2", "1.0.0-rc.10")).toBe(-1); + expect(compareSemver("1.0.0", "1.0.0-rc.10")).toBe(1); + expect(compareSemver("v2.0.0+build.1", "2.0.0+build.2")).toBe(0); + }); +}); + +describe("release candidate binding", () => { + it("binds the candidate version and complete platform identity to the release tag", () => { + const derived = completeManifest("1.2.3", "artifact-derived-signature"); + const candidate = structuredClone(derived); + + expect(assertCandidateMatchesRelease(candidate, derived, "v1.2.3")).toEqual({ + platformCount: 4, + version: "1.2.3", + }); + + const wrongVersion = structuredClone(candidate); + wrongVersion.version = "9.9.9"; + expect(() => assertCandidateMatchesRelease(wrongVersion, derived, "v1.2.3")).toThrow( + "does not match release tag", + ); + + const missingPlatform = structuredClone(candidate); + delete missingPlatform.platforms["windows-aarch64"]; + expect(() => assertCandidateMatchesRelease(missingPlatform, derived, "v1.2.3")).toThrow( + "missing required updater platforms: windows-aarch64", + ); + + const changedSignature = structuredClone(candidate); + changedSignature.platforms["windows-x86_64"].signature = "different-signature"; + expect(() => assertCandidateMatchesRelease(changedSignature, derived, "v1.2.3")).toThrow( + "platforms/signatures do not match", + ); + }); +}); + +describe("existing GitHub Release reuse", () => { + it("requires GitHub immutability and canonical SHA-256 asset digests", () => { + const releaseTag = "v1.2.3"; + const valid = inspectReleaseForReuse(completeRelease(releaseTag), releaseTag); + expect(valid.reusable).toBe(true); + expect(Object.keys(valid.digests)).toHaveLength(11); + + expect(() => + inspectReleaseForReuse(completeRelease(releaseTag, { immutable: false }), releaseTag), + ).toThrow("is mutable"); + + const mutableAndIncomplete = completeRelease(releaseTag, { immutable: false }); + mutableAndIncomplete.assets.pop(); + expect(() => inspectReleaseForReuse(mutableAndIncomplete, releaseTag)).toThrow("is mutable"); + + const immutableAndIncomplete = completeRelease(releaseTag); + immutableAndIncomplete.assets.pop(); + expect(() => inspectReleaseForReuse(immutableAndIncomplete, releaseTag)).toThrow( + "is missing required assets and cannot be repaired", + ); + + const repairableDraft = completeRelease(releaseTag, { draft: true, immutable: false }); + repairableDraft.assets.pop(); + expect(inspectReleaseForReuse(repairableDraft, releaseTag)).toMatchObject({ + reason: "draft", + reusable: false, + }); + + const missingDigest = completeRelease(releaseTag); + delete missingDigest.assets[0].digest; + expect(() => inspectReleaseForReuse(missingDigest, releaseTag)).toThrow( + "has no canonical SHA-256 digest", + ); + + const unpinnedExtra = completeRelease(releaseTag); + unpinnedExtra.assets.push({ name: "CodexAppManager_extra.zip", size: 10 }); + expect(() => inspectReleaseForReuse(unpinnedExtra, releaseTag)).toThrow( + "has no canonical SHA-256 digest", + ); + }); +}); + +describe("Tauri updater verification", () => { + it("accepts the configured minisign format and rejects changed bytes", async () => { + const root = await tempRoot("updater-signature"); + const artifact = Buffer.from("signed updater bytes"); + const fixture = updaterFixture(artifact); + const path = join(root, "artifact.bin"); + await writeFile(path, artifact); + + await expect( + verifyTauriUpdaterSignature(path, fixture.signature, fixture.publicKey), + ).resolves.toBe(true); + + await writeFile(path, Buffer.from("tampered updater bytes")); + await expect( + verifyTauriUpdaterSignature(path, fixture.signature, fixture.publicKey), + ).rejects.toThrow("artifact signature is invalid"); + }); + + it("verifies local manifest payloads and sidecars before publication", async () => { + const root = await tempRoot("local-updater-signatures"); + const artifact = Buffer.from("locally signed updater bytes"); + const fixture = updaterFixture(artifact); + const candidate = manifest("1.2.3", fixture.signature); + const artifactName = "manager-1.2.3.exe"; + await writeFile(join(root, artifactName), artifact); + await writeFile(join(root, `${artifactName}.sig`), `${fixture.signature}\n`); + + await expect( + verifyLocalUpdaterArtifacts({ + manifest: candidate, + distDir: root, + publicKey: fixture.publicKey, + }), + ).resolves.toMatchObject({ artifactCount: 1, verified: true }); + + const wrongKey = updaterFixture(Buffer.from("different release")).publicKey; + await expect( + verifyLocalUpdaterArtifacts({ + manifest: candidate, + distDir: root, + publicKey: wrongKey, + }), + ).rejects.toThrow("artifact signature is invalid"); + }); +}); + +describe("public mirror route verification", () => { + it("downloads the candidate and every updater payload through public URLs", async () => { + const root = await tempRoot("public-route"); + const artifact = Buffer.from("public updater payload"); + const fixture = updaterFixture(artifact); + const candidate = completeManifest("1.2.3", fixture.signature); + const candidateKey = candidateKeyFor("1.2.3", "public-run"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const objects = new Map([ + [`/manager/${candidateKey}`, await readFile(candidatePath)], + ]); + const expectedArtifacts = Object.entries(candidate.platforms).map( + ([platform, entry]) => { + const name = new URL(entry.url).pathname.split("/").at(-1); + objects.set(`/manager/1.2.3/${name}`, artifact); + return { + key: `1.2.3/${name}`, + localPath: join(root, name), + name, + platform, + sha256: hash(artifact), + signature: fixture.signature, + size: artifact.length, + }; + }, + ); + const requests = []; + + const report = await verifyPublicMirrorRoute({ + candidateKey, + candidateManifest: candidate, + candidatePath, + expectedArtifacts, + mirrorBase: "https://mirror.example/manager", + publicKey: fixture.publicKey, + workDir: join(root, "verify"), + fetchImpl: publicRouteFetch(objects, requests), + }); + + expect(report).toMatchObject({ artifactCount: 4, backendCount: 2, verified: true }); + expect(report.backends.r2).toMatchObject({ artifactCount: 4, verified: true }); + expect(report.backends.ihep).toMatchObject({ artifactCount: 4, verified: true }); + expect(requests).toHaveLength(15); + const workerRequests = requests.filter((url) => url.hostname === "mirror.example"); + expect(workerRequests).toHaveLength(10); + expect(workerRequests.every((url) => url.searchParams.has("cam_probe"))).toBe(true); + expect(new Set(workerRequests.map((url) => url.searchParams.get("cam_backend")))).toEqual( + new Set(["r2", "ihep"]), + ); + + const corruptedName = expectedArtifacts[0].name; + const corruptedIhep = new Map(objects); + corruptedIhep.set(`/manager/1.2.3/${corruptedName}`, Buffer.from("corrupt")); + await expect( + verifyPublicMirrorRoute({ + candidateKey, + candidateManifest: candidate, + candidatePath, + expectedArtifacts, + mirrorBase: "https://mirror.example/manager", + publicKey: fixture.publicKey, + workDir: join(root, "corrupt"), + fetchImpl: publicRouteFetch(objects, [], { ihepObjects: corruptedIhep }), + }), + ).rejects.toThrow(`public mirror ihep size mismatch for ${corruptedName}`); + }); +}); + +describe("AWS conditional writes", () => { + function objectStore() { + return new AwsObjectStore({ + name: "r2", + endpoint: "https://r2.example.invalid", + bucket: "manager", + region: "auto", + accessKeyId: "test-access-key", + secretAccessKey: "test-secret-key", + configPath: "/tmp/test-aws-config", + }); + } + + it("uses one attempt and trusts the committing PutObject response ETag", async () => { + const root = await tempRoot("conditional-put"); + const candidatePath = await writeManifest(root, "candidate.json", manifest("1.2.3")); + const backend = objectStore(); + let call; + backend.aws = async (args, options) => { + call = { args, options }; + return { code: 0, stderr: "", stdout: '{"ETag":"\\"committed-etag\\""}' }; + }; + backend.head = async () => { + throw new Error("post-write HEAD must not determine write ownership"); + }; + + const result = await backend.putLatestConditional( + candidatePath, + '"previous-etag"', + "promotion-token", + ); + + expect(result.etag).toBe('"committed-etag"'); + expect(call.options.env).toEqual({ AWS_MAX_ATTEMPTS: "1" }); + expect(call.args).toEqual( + expect.arrayContaining([ + "--if-match", + '"previous-etag"', + "--metadata", + "cam-promotion-token=promotion-token", + ]), + ); + }); + + it("reports a final 412 as a conditional conflict without claiming ownership", async () => { + const root = await tempRoot("conditional-conflict"); + const candidatePath = await writeManifest(root, "candidate.json", manifest("1.2.3")); + const backend = objectStore(); + backend.aws = async () => ({ + code: 1, + stderr: "PreconditionFailed (412)", + stdout: "", + }); + + await expect( + backend.putLatestConditional(candidatePath, '"stale-etag"', "promotion-token"), + ).rejects.toBeInstanceOf(ConditionalWriteError); + }); +}); + +describe("backend candidate verification", () => { + it("downloads candidate and artifact and verifies version, size, hash, and signature", async () => { + const root = await tempRoot("candidate-verify"); + const dist = join(root, "dist"); + await mkdir(dist, { recursive: true }); + const artifact = Buffer.from("release artifact from object storage"); + const fixture = updaterFixture(artifact); + const candidate = manifest("1.2.3", fixture.signature); + const artifactName = "manager-1.2.3.exe"; + const dmgName = "CodexAppManager_aarch64.dmg"; + const signatureName = `${artifactName}.sig`; + const dmg = Buffer.from("macOS installer bytes"); + const signatureSidecar = Buffer.from(`${fixture.signature}\n`); + await writeFile(join(dist, artifactName), artifact); + await writeFile(join(dist, dmgName), dmg); + await writeFile(join(dist, signatureName), signatureSidecar); + const candidatePath = await writeManifest(dist, "latest.mirror.json", candidate); + const key = candidateKeyFor("1.2.3", "run-1"); + const backend = new MemoryBackend("r2", { + [key]: await readFile(candidatePath), + [`1.2.3/${artifactName}`]: artifact, + [`1.2.3/${dmgName}`]: dmg, + [`1.2.3/${signatureName}`]: signatureSidecar, + }); + + const report = await verifyBackendCandidate({ + backend, + candidateKey: key, + candidatePath, + candidateManifest: candidate, + distDir: dist, + mirrorBase: "https://mirror.example/manager", + publicKey: fixture.publicKey, + workDir: join(root, "verify"), + }); + + expect(report.verified).toBe(true); + expect(report.artifactCount).toBe(3); + expect(report.artifacts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: artifactName, + sha256: hash(artifact), + signatureVerified: true, + size: artifact.length, + }), + expect.objectContaining({ + name: dmgName, + sha256: hash(dmg), + signatureVerified: null, + size: dmg.length, + }), + expect.objectContaining({ + name: signatureName, + sha256: hash(signatureSidecar), + signatureVerified: null, + size: signatureSidecar.length, + }), + ]), + ); + }); + + it("rejects an empty platform signature instead of silently skipping verification", async () => { + const root = await tempRoot("candidate-empty-signature"); + const dist = join(root, "dist"); + await mkdir(dist, { recursive: true }); + const candidate = manifest("1.2.3", ""); + const candidatePath = await writeManifest(dist, "latest.mirror.json", candidate); + const key = candidateKeyFor("1.2.3", "run-empty-signature"); + const backend = new MemoryBackend("r2", { + [key]: await readFile(candidatePath), + }); + + await expect( + verifyBackendCandidate({ + backend, + candidateKey: key, + candidatePath, + candidateManifest: candidate, + distDir: dist, + mirrorBase: "https://mirror.example/manager", + publicKey: "deliberately-invalid-public-key", + workDir: join(root, "verify"), + }), + ).rejects.toThrow("empty updater signature"); + }); + + it("requires each manifest signature to match its local sidecar", async () => { + const root = await tempRoot("candidate-sidecar-mismatch"); + const dist = join(root, "dist"); + await mkdir(dist, { recursive: true }); + const artifact = Buffer.from("release artifact from object storage"); + const fixture = updaterFixture(artifact); + const candidate = manifest("1.2.3", fixture.signature); + const artifactName = "manager-1.2.3.exe"; + await writeFile(join(dist, artifactName), artifact); + await writeFile(join(dist, `${artifactName}.sig`), "different-signature\n"); + const candidatePath = await writeManifest(dist, "latest.mirror.json", candidate); + const key = candidateKeyFor("1.2.3", "run-sidecar-mismatch"); + const backend = new MemoryBackend("r2", { + [key]: await readFile(candidatePath), + }); + + await expect( + verifyBackendCandidate({ + backend, + candidateKey: key, + candidatePath, + candidateManifest: candidate, + distDir: dist, + mirrorBase: "https://mirror.example/manager", + publicKey: fixture.publicKey, + workDir: join(root, "verify"), + }), + ).rejects.toThrow("sidecar does not match manifest"); + }); +}); + +describe("pre-publication mirror verification", () => { + it("verifies both origins and the public route without writing latest.json", async () => { + const root = await tempRoot("prepublish-verify"); + const dist = join(root, "dist"); + await mkdir(dist, { recursive: true }); + const artifact = Buffer.from("stable updater payload"); + const fixture = updaterFixture(artifact); + const candidate = completeManifest("1.2.3", fixture.signature); + const candidatePath = await writeManifest(dist, "latest.mirror.json", candidate); + const candidateKey = candidateKeyFor("1.2.3", "verify-run"); + const candidateBytes = await readFile(candidatePath); + const stagedObjects = {}; + const publicObjects = new Map([[`/manager/${candidateKey}`, candidateBytes]]); + for (const entry of Object.values(candidate.platforms)) { + const name = new URL(entry.url).pathname.split("/").at(-1); + const sidecar = Buffer.from(`${fixture.signature}\n`); + await writeFile(join(dist, name), artifact); + await writeFile(join(dist, `${name}.sig`), sidecar); + stagedObjects[`1.2.3/${name}`] = artifact; + stagedObjects[`1.2.3/${name}.sig`] = sidecar; + publicObjects.set(`/manager/1.2.3/${name}`, artifact); + } + const current = Buffer.from(`${JSON.stringify(manifest("1.0.0"))}\n`); + const r2 = new MemoryBackend("r2", { + "latest.json": current, + [candidateKey]: candidateBytes, + ...stagedObjects, + }); + const ihep = new MemoryBackend("ihep", { + "latest.json": current, + [candidateKey]: candidateBytes, + ...stagedObjects, + }); + const promotionToken = hash(Buffer.from(candidateKey)); + r2.objects.get(candidateKey).metadata["cam-promotion-token"] = promotionToken; + ihep.objects.get(candidateKey).metadata["cam-promotion-token"] = promotionToken; + const summaryPath = join(root, "mirror-verification-summary.json"); + + const summary = await verifyMirrors({ + backends: [r2, ihep], + candidateKey, + candidateManifest: candidate, + candidatePath, + distDir: dist, + mirrorBase: "https://mirror.example/manager", + override: overrideOff(), + publicKey: fixture.publicKey, + summaryPath, + tempRoot: join(root, "verify"), + fetchImpl: publicRouteFetch(publicObjects), + }); + + expect(summary.outcome).toBe("verified"); + expect(summary.publicRouteVerification).toBe("passed"); + expect(Object.keys(summary.publicRoute.backends)).toEqual(["r2", "ihep"]); + expect(summary.backends.map((backend) => backend.candidateVerification)).toEqual([ + "passed", + "passed", + ]); + expect([r2.latestPutAttempts, ihep.latestPutAttempts]).toEqual([0, 0]); + expect(JSON.parse(await readFile(summaryPath, "utf8")).outcome).toBe("verified"); + + const corruptedName = new URL(Object.values(candidate.platforms)[0].url).pathname + .split("/") + .at(-1); + const corruptedIhep = new Map(publicObjects); + corruptedIhep.set(`/manager/1.2.3/${corruptedName}`, Buffer.from("corrupt")); + await expect( + promoteMirrors({ + backends: [r2, ihep], + candidateKey, + candidateManifest: candidate, + candidatePath, + distDir: dist, + mirrorBase: "https://mirror.example/manager", + override: overrideOff(), + publicKey: fixture.publicKey, + summaryPath: join(root, "failed-promotion-summary.json"), + tempRoot: join(root, "failed-promote"), + fetchImpl: publicRouteFetch(publicObjects, [], { ihepObjects: corruptedIhep }), + }), + ).rejects.toThrow(`public mirror ihep size mismatch for ${corruptedName}`); + expect([r2.latestPutAttempts, ihep.latestPutAttempts]).toEqual([0, 0]); + }); +}); + +describe("monotonic mirror promotion", () => { + it("blocks an old tag rerun without writing either backend", async () => { + const root = await tempRoot("old-rerun"); + const current = manifest("2.0.0"); + const candidate = manifest("1.9.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = `${JSON.stringify(current)}\n`; + const backends = [ + new MemoryBackend("r2", { "latest.json": initial }), + new MemoryBackend("ihep", { "latest.json": initial }), + ]; + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + summary: summaryFor(backends), + workDir: join(root, "transaction"), + }), + ).rejects.toBeInstanceOf(DowngradeBlockedError); + + expect(backends.map((backend) => backend.latestPutAttempts)).toEqual([0, 0]); + expect(backends.map((backend) => JSON.parse(backend.body("latest.json")).version)).toEqual([ + "2.0.0", + "2.0.0", + ]); + }); + + it("treats a same-version rerun as a no-write idempotent success", async () => { + const root = await tempRoot("same-version"); + const current = manifest("2.0.0"); + current.pub_date = "2026-01-01T00:00:00Z"; + const candidate = manifest("2.0.0"); + candidate.pub_date = "2026-02-01T00:00:00Z"; + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const original = Buffer.from(`${JSON.stringify(current)}\n`); + const backends = [ + new MemoryBackend("r2", { "latest.json": original }), + new MemoryBackend("ihep", { "latest.json": original }), + ]; + + const result = await promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + summary: summaryFor(backends), + workDir: join(root, "transaction"), + }); + + expect(result.outcome).toBe("idempotent"); + expect(backends.map((backend) => backend.latestPutAttempts)).toEqual([0, 0]); + expect(backends.every((backend) => backend.body("latest.json").equals(original))).toBe(true); + }); + + it("rejects a concurrent latest change before returning idempotent", async () => { + const root = await tempRoot("same-version-race"); + const candidate = manifest("2.0.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const original = Buffer.from(`${JSON.stringify(candidate)}\n`); + const r2 = new MemoryBackend("r2", { "latest.json": original }); + const ihep = new MemoryBackend("ihep", { "latest.json": original }); + const backends = [r2, ihep]; + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + summary: summaryFor(backends), + workDir: join(root, "transaction"), + hooks: { + afterSnapshots: () => { + ihep.set("latest.json", `${JSON.stringify(manifest("2.1.0"))}\n`); + }, + }, + }), + ).rejects.toBeInstanceOf(ConditionalWriteError); + + expect([r2.latestPutAttempts, ihep.latestPutAttempts]).toEqual([0, 0]); + expect(JSON.parse(r2.body("latest.json")).version).toBe("2.0.0"); + expect(JSON.parse(ihep.body("latest.json")).version).toBe("2.1.0"); + }); + + it("uses conditional writes so a concurrent newer release wins without mixed latest pointers", async () => { + const root = await tempRoot("concurrent"); + const initial = `${JSON.stringify(manifest("1.0.0"))}\n`; + const backends = [ + new MemoryBackend("r2", { "latest.json": initial }), + new MemoryBackend("ihep", { "latest.json": initial }), + ]; + const older = manifest("1.1.0"); + const newer = manifest("1.2.0"); + const olderPath = await writeManifest(root, "older.json", older); + const newerPath = await writeManifest(root, "newer.json", newer); + let releaseOlderSnapshots; + let olderSnapshotsReached; + const holdOlder = new Promise((resolvePromise) => { + releaseOlderSnapshots = resolvePromise; + }); + const olderReady = new Promise((resolvePromise) => { + olderSnapshotsReached = resolvePromise; + }); + + const olderRun = promoteCandidateTransaction({ + backends, + candidateManifest: older, + candidatePath: olderPath, + override: overrideOff(), + summary: summaryFor(backends), + workDir: join(root, "older-transaction"), + hooks: { + afterSnapshots: async () => { + olderSnapshotsReached(); + await holdOlder; + }, + }, + }); + await olderReady; + + await promoteCandidateTransaction({ + backends, + candidateManifest: newer, + candidatePath: newerPath, + override: overrideOff(), + summary: summaryFor(backends), + workDir: join(root, "newer-transaction"), + }); + releaseOlderSnapshots(); + + await expect(olderRun).rejects.toBeInstanceOf(ConditionalWriteError); + expect(backends.map((backend) => JSON.parse(backend.body("latest.json")).version)).toEqual([ + "1.2.0", + "1.2.0", + ]); + }); + + it("heals a hard-terminated partial promotion without rewriting the committed backend", async () => { + const root = await tempRoot("partial-terminated-transaction"); + const current = manifest("1.0.0"); + const candidate = manifest("1.1.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = `${JSON.stringify(current)}\n`; + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new MemoryBackend("ihep", { "latest.json": initial }); + // Model SIGKILL/OOM after the first conditional write committed but before + // the process could enter its rollback handler. + const r2Previous = await r2.head("latest.json"); + await r2.putLatestConditional(candidatePath, r2Previous.etag, "terminated-run"); + const r2AttemptsAfterTermination = r2.latestPutAttempts; + const backends = [r2, ihep]; + const failedResumeSummary = summaryFor(backends); + const realIhepPut = ihep.putLatestConditional.bind(ihep); + let failFirstResume = true; + ihep.putLatestConditional = async (...args) => { + if (failFirstResume) { + failFirstResume = false; + ihep.latestPutAttempts += 1; + throw new ConditionalWriteError("ihep: simulated conditional outage"); + } + return await realIhepPut(...args); + }; + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + promotionToken: "this-run", + summary: failedResumeSummary, + workDir: join(root, "failed-resume"), + }), + ).rejects.toBeInstanceOf(ConditionalWriteError); + + expect(r2.latestPutAttempts).toBe(r2AttemptsAfterTermination); + expect(backends.map((backend) => JSON.parse(backend.body("latest.json")).version)).toEqual([ + "1.1.0", + "1.0.0", + ]); + expect(failedResumeSummary.rollback).toEqual( + expect.objectContaining({ attempted: false, complete: true }), + ); + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + promotionToken: "fresh-run", + summary: summaryFor(backends), + workDir: join(root, "successful-resume"), + }), + ).resolves.toEqual(expect.objectContaining({ outcome: "promoted" })); + + expect(r2.latestPutAttempts).toBe(r2AttemptsAfterTermination); + expect(ihep.latestPutAttempts).toBe(2); + expect(backends.map((backend) => JSON.parse(backend.body("latest.json")).version)).toEqual([ + "1.1.0", + "1.1.0", + ]); + expect(r2.objects.get("latest.json").metadata["cam-promotion-token"]).toBe( + "terminated-run", + ); + expect(ihep.objects.get("latest.json").metadata["cam-promotion-token"]).toBe("fresh-run"); + }); + + it("fails closed before writing when either backend has no baseline latest.json", async () => { + const root = await tempRoot("unseeded-backend"); + const candidate = manifest("1.1.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const r2 = new MemoryBackend("r2", { + "latest.json": `${JSON.stringify(manifest("1.0.0"))}\n`, + }); + const ihep = new MemoryBackend("ihep"); + const backends = [r2, ihep]; + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + summary: summaryFor(backends), + workDir: join(root, "transaction"), + }), + ).rejects.toThrow("seed both backends"); + + expect(backends.map((backend) => backend.latestPutAttempts)).toEqual([0, 0]); + expect(ihep.body("latest.json")).toBeUndefined(); + }); + + it("never rolls back an ETag that a concurrent writer owns", async () => { + const root = await tempRoot("etag-ownership"); + const current = manifest("1.0.0"); + const candidate = manifest("1.1.0"); + const external = manifest("1.2.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const r2 = new MemoryBackend("r2", { + "latest.json": `${JSON.stringify(current)}\n`, + }); + const ihep = new MemoryBackend("ihep", { + "latest.json": `${JSON.stringify(current)}\n`, + }); + const realPut = r2.putLatestConditional.bind(r2); + r2.putLatestConditional = async (...args) => { + const committed = await realPut(...args); + r2.set( + "latest.json", + `${JSON.stringify(external)}\n`, + { "cam-promotion-token": "external-run" }, + ); + return committed; + }; + const backends = [r2, ihep]; + const summary = summaryFor(backends); + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + promotionToken: "this-run", + summary, + workDir: join(root, "transaction"), + }), + ).rejects.toBeInstanceOf(ConditionalWriteError); + + expect(JSON.parse(r2.body("latest.json")).version).toBe("1.2.0"); + expect(JSON.parse(ihep.body("latest.json")).version).toBe("1.0.0"); + expect(summary.backends.find((backend) => backend.name === "r2").rollback).toBe( + "skipped-concurrent-change", + ); + }); + + it("observes and rolls back a committed write whose process response was lost", async () => { + const root = await tempRoot("ambiguous-commit"); + const current = manifest("1.0.0"); + const candidate = manifest("1.1.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = Buffer.from(`${JSON.stringify(current)}\n`); + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new MemoryBackend("ihep", { "latest.json": initial }); + const realPut = r2.putLatestConditional.bind(r2); + let firstWrite = true; + r2.putLatestConditional = async (localPath, expectedEtag, promotionToken) => { + if (!firstWrite) return await realPut(localPath, expectedEtag, promotionToken); + firstWrite = false; + r2.latestPutAttempts += 1; + const currentObject = r2.objects.get("latest.json"); + if (currentObject?.etag !== expectedEtag) { + throw new ConditionalWriteError("r2: stale latest ETag"); + } + r2.set( + "latest.json", + await readFile(localPath), + { "cam-promotion-token": promotionToken }, + ); + throw new Error("simulated process termination after server commit"); + }; + const backends = [r2, ihep]; + const summary = summaryFor(backends); + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + promotionToken: "interrupted-run", + summary, + workDir: join(root, "transaction"), + }), + ).rejects.toThrow("simulated process termination"); + + expect(backends.every((backend) => backend.body("latest.json").equals(initial))).toBe(true); + expect(summary.rollback).toEqual( + expect.objectContaining({ attempted: true, complete: true }), + ); + }); + + it("rolls back the first backend when execution is interrupted between writes", async () => { + const root = await tempRoot("interrupted"); + const current = manifest("1.0.0"); + const candidate = manifest("1.1.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = Buffer.from(`${JSON.stringify(current)}\n`); + const backends = [ + new MemoryBackend("r2", { "latest.json": initial }), + new MemoryBackend("ihep", { "latest.json": initial }), + ]; + const summary = summaryFor(backends); + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + summary, + workDir: join(root, "transaction"), + hooks: { + afterWrite: (state) => { + if (state.backend.name === "r2") throw new Error("simulated SIGTERM"); + }, + }, + }), + ).rejects.toThrow("simulated SIGTERM"); + + expect(backends.every((backend) => backend.body("latest.json").equals(initial))).toBe(true); + expect(summary.backends.find((backend) => backend.name === "r2").rollback).toBe("restored"); + expect(summary.backends.find((backend) => backend.name === "ihep").promotion).toBe( + "not-started", + ); + }); + + it("does not write latest when one backend candidate verification fails", async () => { + const root = await tempRoot("one-backend-fails"); + const dist = join(root, "dist"); + await mkdir(dist, { recursive: true }); + const artifact = Buffer.from("valid release artifact"); + const fixture = updaterFixture(artifact); + const candidate = completeManifest("1.1.0", fixture.signature); + const artifactNames = Object.values(candidate.platforms).map((entry) => + new URL(entry.url).pathname.split("/").at(-1), + ); + for (const name of artifactNames) { + await writeFile(join(dist, name), artifact); + await writeFile(join(dist, `${name}.sig`), `${fixture.signature}\n`); + } + const candidatePath = await writeManifest(dist, "latest.mirror.json", candidate); + const candidateKey = candidateKeyFor("1.1.0", "run-2"); + const current = Buffer.from(`${JSON.stringify(manifest("1.0.0"))}\n`); + const stagedObjects = Object.fromEntries( + artifactNames.flatMap((name) => [ + [`1.1.0/${name}`, artifact], + [`1.1.0/${name}.sig`, Buffer.from(`${fixture.signature}\n`)], + ]), + ); + const r2 = new MemoryBackend("r2", { + "latest.json": current, + [candidateKey]: await readFile(candidatePath), + ...stagedObjects, + }); + const missingArtifact = artifactNames.at(-1); + const ihepObjects = { ...stagedObjects }; + delete ihepObjects[`1.1.0/${missingArtifact}`]; + const ihep = new MemoryBackend("ihep", { + "latest.json": current, + [candidateKey]: await readFile(candidatePath), + // Deliberately missing one required platform artifact. + ...ihepObjects, + }); + const promotionToken = hash(Buffer.from(candidateKey)); + r2.objects.get(candidateKey).metadata["cam-promotion-token"] = promotionToken; + ihep.objects.get(candidateKey).metadata["cam-promotion-token"] = promotionToken; + const summaryPath = join(root, "promotion-summary.json"); + + await expect( + promoteMirrors({ + backends: [r2, ihep], + candidateKey, + candidateManifest: candidate, + candidatePath, + distDir: dist, + mirrorBase: "https://mirror.example/manager", + override: overrideOff(), + publicKey: fixture.publicKey, + summaryPath, + tempRoot: join(root, "promote"), + }), + ).rejects.toThrow("ihep: artifact is missing"); + + expect([r2.latestPutAttempts, ihep.latestPutAttempts]).toEqual([0, 0]); + const audit = JSON.parse(await readFile(summaryPath, "utf8")); + expect(audit.outcome).toBe("failed"); + expect(audit.backends.map((backend) => backend.candidateVerification)).toEqual([ + "passed", + "failed", + ]); + }); + + it("requires a workflow-dispatch audit trail for emergency downgrade override", () => { + expect(() => + downgradeOverrideFromEnv({ + MIRROR_ALLOW_DOWNGRADE: "1", + MIRROR_DOWNGRADE_REASON: "urgent rollback after launch regression", + GITHUB_EVENT_NAME: "push", + GITHUB_ACTOR: "release-admin", + GITHUB_REPOSITORY: "owner/repo", + GITHUB_RUN_ID: "42", + }), + ).toThrow("only from workflow_dispatch"); + + expect(() => + downgradeOverrideFromEnv({ + MIRROR_ALLOW_DOWNGRADE: "1", + MIRROR_DEFAULT_BRANCH: "main", + MIRROR_DOWNGRADE_REASON: "urgent rollback after launch regression", + MIRROR_WORKFLOW_REF_NAME: "release-experiment", + GITHUB_EVENT_NAME: "workflow_dispatch", + GITHUB_ACTOR: "release-admin", + GITHUB_REPOSITORY: "owner/repo", + GITHUB_RUN_ID: "42", + }), + ).toThrow("default branch"); + + const override = downgradeOverrideFromEnv({ + MIRROR_ALLOW_DOWNGRADE: "1", + MIRROR_DEFAULT_BRANCH: "main", + MIRROR_DOWNGRADE_REASON: "urgent rollback after launch regression", + MIRROR_WORKFLOW_REF_NAME: "main", + GITHUB_EVENT_NAME: "workflow_dispatch", + GITHUB_ACTOR: "release-admin", + GITHUB_TRIGGERING_ACTOR: "incident-commander", + GITHUB_REPOSITORY: "owner/repo", + GITHUB_RUN_ID: "42", + }); + expect(override).toEqual( + expect.objectContaining({ + actor: "incident-commander", + originalActor: "release-admin", + reason: "urgent rollback after launch regression", + requested: true, + runUrl: "https://github.com/owner/repo/actions/runs/42", + }), + ); + }); + + it("uses an audited override to downgrade both backends consistently", async () => { + const root = await tempRoot("audited-downgrade"); + const current = manifest("3.0.0"); + const candidate = manifest("2.5.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = `${JSON.stringify(current)}\n`; + const backends = [ + new MemoryBackend("r2", { "latest.json": initial }), + new MemoryBackend("ihep", { "latest.json": initial }), + ]; + const override = { + actor: "release-admin", + eventName: "workflow_dispatch", + reason: "rollback production crash regression", + requested: true, + runUrl: "https://github.com/owner/repo/actions/runs/42", + used: false, + }; + const summary = summaryFor(backends, override); + + const result = await promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override, + summary, + workDir: join(root, "transaction"), + }); + + expect(result.outcome).toBe("downgrade-override-promoted"); + expect(summary.override.used).toBe(true); + expect(backends.map((backend) => JSON.parse(backend.body("latest.json")).version)).toEqual([ + "2.5.0", + "2.5.0", + ]); + }); +}); + +describe("mirror manifest rewriting", () => { + it("uses an immutable version path and a per-run candidate key", async () => { + const root = await tempRoot("manifest-rewrite"); + const dist = join(root, "dist"); + await mkdir(dist, { recursive: true }); + const source = manifest("3.4.5"); + source.platforms["windows-x86_64"].url = + "https://github.com/owner/repo/releases/download/v3.4.5/manager-3.4.5.exe"; + await writeManifest(dist, "latest.json", source); + + const result = await createMirrorManifest(dist, "https://mirror.example/manager"); + + expect(result.manifest.platforms["windows-x86_64"].url).toBe( + "https://mirror.example/manager/3.4.5/manager-3.4.5.exe", + ); + expect(candidateKeyFor("3.4.5", "987-2")).toBe("candidates/3.4.5/987-2.json"); + }); +}); + +describe("release audit summary", () => { + it("renders per-backend verification, monotonic decisions, rollback, and override audit", async () => { + const root = await tempRoot("release-summary"); + const summaryPath = join(root, "github-summary.md"); + await writeManifest(root, "latest.json", manifest("4.0.0")); + await writeManifest(root, "mirror-stage-summary.json", { + candidateKey: "candidates/4.0.0/42-1.json", + candidateVersion: "4.0.0", + outcome: "staged", + }); + await writeManifest(root, "mirror-verification-summary.json", { + candidateKey: "candidates/4.0.0/42-1.json", + candidateVersion: "4.0.0", + outcome: "verified", + publicRouteVerification: "passed", + }); + await writeManifest(root, "mirror-promotion-summary.json", { + candidateKey: "candidates/4.0.0/42-1.json", + candidateVersion: "4.0.0", + outcome: "downgrade-override-promoted", + override: { + actor: "incident-commander", + originalActor: "release-admin", + reason: "rollback broken production release", + requested: true, + runUrl: "https://github.com/owner/repo/actions/runs/42", + used: true, + }, + backends: [ + { + name: "r2", + candidateVerification: "passed", + currentVersion: "5.0.0", + decision: "promote-downgrade-override", + promotion: "verified", + rollback: "not-needed", + finalVersion: "4.0.0", + }, + { + name: "ihep", + candidateVerification: "passed", + currentVersion: "5.0.0", + decision: "promote-downgrade-override", + promotion: "verified", + rollback: "not-needed", + finalVersion: "4.0.0", + }, + ], + }); + + await execFileAsync(process.execPath, [join(process.cwd(), "scripts/write-release-summary.mjs")], { + cwd: root, + env: { + ...process.env, + GITHUB_REF_NAME: "v4.0.0", + GITHUB_REPOSITORY: "owner/repo", + GITHUB_STEP_SUMMARY: summaryPath, + }, + }); + + const rendered = await readFile(summaryPath, "utf8"); + expect(rendered).toContain("Promotion outcome: **downgrade-override-promoted**"); + expect(rendered).toContain("Pre-publish verification: **verified**"); + expect(rendered).toContain("Public route verification: **passed**"); + expect(rendered).toContain("| r2 | passed | 5.0.0 | promote-downgrade-override | verified"); + expect(rendered).toContain("rollback broken production release"); + expect(rendered).toContain("by `incident-commander` (original workflow actor: `release-admin`)"); + expect(rendered).toContain("https://github.com/owner/repo/actions/runs/42"); + }); +}); diff --git a/scripts/release-workflow.test.mjs b/scripts/release-workflow.test.mjs new file mode 100644 index 0000000..7f919f6 --- /dev/null +++ b/scripts/release-workflow.test.mjs @@ -0,0 +1,164 @@ +import { readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +import { + assertImmutableReleasesEnabled, + verifyImmutableReleases, +} from "./check-immutable-releases.mjs"; + +const repoRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); +const workflow = await readFile(join(repoRoot, ".github/workflows/release.yml"), "utf8"); +const mirrorRelease = await readFile(join(repoRoot, "scripts/mirror-release.mjs"), "utf8"); +const releaseJob = workflow.slice(workflow.indexOf(" release:\n")); + +describe("release workflow recovery invariants", () => { + it("queues every release run instead of replacing an older pending tag", () => { + expect(workflow).toMatch( + /concurrency:\n\s+group: release-latest-\$\{\{ github\.repository \}\}\n\s+cancel-in-progress: false\n\s+queue: max/, + ); + }); + + it("refreshes immutable Release state inside the release job on failed-job reruns", () => { + expect(releaseJob).toContain("id: live_release"); + expect(releaseJob).toContain( + "if: ${{ steps.live_release.outputs.release_reusable != 'true' }}", + ); + expect(releaseJob).toContain( + "RELEASE_ASSET_DIGESTS: ${{ steps.live_release.outputs.release_asset_digests }}", + ); + expect(releaseJob).toContain( + 'if [[ "${{ steps.live_release.outputs.release_reusable }}" == "true" ]]; then', + ); + expect(releaseJob).not.toContain( + "if: ${{ needs.prepare.outputs.release_reusable != 'true' }}", + ); + }); + + it("uses the target tag updater trust root without executing historical scripts", () => { + const refresh = releaseJob.indexOf("- name: Refresh immutable release state"); + const resolveTrust = releaseJob.indexOf( + "- name: Resolve updater trust root for release tag", + ); + const download = releaseJob.indexOf("- name: Download canonical build artifacts"); + const localVerify = releaseJob.indexOf( + "- name: Verify local updater signatures before immutable publication", + ); + const stage = releaseJob.indexOf("- name: Stage CDN mirror candidate"); + const trustStep = releaseJob.slice(resolveTrust, download); + const localVerifyStep = releaseJob.slice(localVerify, stage); + + expect(resolveTrust).toBeGreaterThan(refresh); + expect(download).toBeGreaterThan(resolveTrust); + expect(trustStep).toContain("gh api --method GET"); + expect(trustStep).toContain("application/vnd.github.raw+json"); + expect(trustStep).toContain('-f ref="$RELEASE_TAG"'); + expect(trustStep).toContain("RELEASE_TAURI_CONFIG="); + expect(trustStep).toContain("MIRROR_UPDATER_PUBLIC_KEY="); + expect(trustStep).toContain('>> "$GITHUB_ENV"'); + expect(localVerifyStep).toContain('"$RELEASE_TAURI_CONFIG"'); + expect(localVerifyStep).not.toContain("src-tauri/tauri.conf.json"); + expect(mirrorRelease).toContain("process.env.MIRROR_UPDATER_PUBLIC_KEY ||"); + }); + + it("fails closed when immutable settings are disabled or cannot be queried", () => { + expect(assertImmutableReleasesEnabled({ enabled: true })).toEqual({ enabled: true }); + expect(() => assertImmutableReleasesEnabled({ enabled: false })).toThrow( + "GitHub Immutable Releases are disabled", + ); + expect(() => + verifyImmutableReleases({ + repository: "owner/repo", + token: "read-only-token", + runner: () => ({ status: 1, stderr: "HTTP 403", stdout: "" }), + }), + ).toThrow("could not verify GitHub Immutable Releases"); + + const prepare = workflow.slice( + workflow.indexOf(" prepare:\n"), + workflow.indexOf(" build:\n"), + ); + expect(prepare).toContain("environment: release"); + expect(prepare).toContain("GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }}"); + expect(prepare).toContain("run: node scripts/check-immutable-releases.mjs"); + expect(releaseJob).toContain("run: node scripts/check-immutable-releases.mjs"); + }); + + it("uploads stable and prerelease assets to a draft before publishing", () => { + const localVerify = releaseJob.indexOf( + "- name: Verify local updater signatures before immutable publication", + ); + const stage = releaseJob.indexOf("- name: Stage CDN mirror candidate"); + const mirrorVerify = releaseJob.indexOf( + "- name: Verify staged CDN mirror before immutable publication", + ); + const upload = releaseJob.indexOf("- name: Upload GitHub Release draft"); + const publish = releaseJob.indexOf("- name: Publish GitHub Release"); + const publishedVerify = releaseJob.indexOf( + "- name: Verify published immutable Release and asset digests", + ); + const attest = releaseJob.indexOf("- name: Attest build provenance"); + const promote = releaseJob.indexOf("- name: Promote CDN mirror latest"); + const winget = releaseJob.indexOf("- name: Trigger winget submission"); + const summary = releaseJob.indexOf("- name: Write release summary"); + expect(localVerify).toBeGreaterThan(-1); + expect(stage).toBeGreaterThan(localVerify); + expect(mirrorVerify).toBeGreaterThan(stage); + expect(upload).toBeGreaterThan(mirrorVerify); + expect(upload).toBeGreaterThan(-1); + expect(publish).toBeGreaterThan(upload); + expect(publishedVerify).toBeGreaterThan(publish); + expect(attest).toBeGreaterThan(publishedVerify); + expect(promote).toBeGreaterThan(attest); + + const uploadStep = releaseJob.slice(upload, publish); + const publishStep = releaseJob.slice(publish, publishedVerify); + const verifyStep = releaseJob.slice(publishedVerify, attest); + const attestStep = releaseJob.slice(attest, promote); + const promoteStep = releaseJob.slice(promote, winget); + const wingetStep = releaseJob.slice(winget, summary); + const localVerifyStep = releaseJob.slice(localVerify, stage); + const mirrorVerifyStep = releaseJob.slice(mirrorVerify, upload); + expect(localVerifyStep).toContain("node scripts/verify-release-artifacts.mjs"); + expect(mirrorVerifyStep).toContain("MIRROR_PHASE: verify"); + expect(mirrorVerifyStep).toContain("bash scripts/sync-mirror.sh dist"); + expect(uploadStep).toContain("draft: true"); + expect(uploadStep).toContain("prerelease: ${{ contains(env.RELEASE_TAG, '-') }}"); + expect(uploadStep).toContain("files: |"); + expect(publishStep).not.toMatch(/^\s+draft:/m); + expect(publishStep).not.toContain("files: |"); + expect(verifyStep).toContain("node scripts/check-release-reuse.mjs"); + expect(verifyStep).toContain("did not become immutable with canonical asset digests"); + expect(attestStep).toContain( + "steps.publish_release.outcome == 'success' || steps.release_source.outputs.existing == 'true'", + ); + expect(attestStep).toContain("actions/attest-build-provenance@"); + expect(attestStep).not.toContain("continue-on-error: true"); + expect(promoteStep).toContain("steps.attest.outcome == 'success'"); + expect(wingetStep).toContain("steps.attest.outcome == 'success'"); + expect(releaseJob.slice(0, upload)).toContain( + "rm -f dist/latest.mirror.json dist/latest.json", + ); + }); + + it("attests only immutable Release bytes on a failed-job reuse", () => { + const source = releaseJob.indexOf("- name: Resolve immutable release artifact source"); + const validate = releaseJob.indexOf("- name: Validate final release artifacts"); + const attest = releaseJob.indexOf("- name: Attest build provenance"); + const promote = releaseJob.indexOf("- name: Promote CDN mirror latest"); + const sourceStep = releaseJob.slice(source, validate); + const attestStep = releaseJob.slice(attest, promote); + + expect(sourceStep).toContain("gh release download"); + expect(sourceStep).toContain("--pattern 'CodexAppManager*'"); + expect(sourceStep).toContain("--pattern 'latest.json'"); + expect(sourceStep).toContain('actual_digest="sha256:$(sha256sum "$file"'); + expect(sourceStep).toContain('if [[ "$actual_digest" != "$expected_digest" ]]'); + expect(attestStep).toContain("dist/*"); + expect(attestStep).toContain("latest.json"); + expect(attestStep).not.toContain("SHA256SUMS"); + expect(attestStep).not.toContain("release-assets/*"); + }); +}); diff --git a/scripts/sync-mirror.sh b/scripts/sync-mirror.sh index d055373..de60715 100755 --- a/scripts/sync-mirror.sh +++ b/scripts/sync-mirror.sh @@ -1,243 +1,8 @@ #!/usr/bin/env bash -# Publish the manager's release artifacts + a URL-rewritten latest.json to the -# CDN mirror, so in-app self-update works without depending on GitHub (which is -# slow/blocked for the mainland-China audience). -# -# • R2 (global) — uploaded when MANAGER_R2_* env vars are set. -# • IHEP S3 (CN) — uploaded when MANAGER_IHEP_S3_* env vars are set. -# -# The worker at codexapp.agentsmirror.com/manager/* serves R2 globally and the -# presigned IHEP object for CN. The artifact bytes are identical to the GitHub -# release, so the signatures already embedded in latest.json stay valid — we -# only rewrite the download URLs. -# -# Usage: scripts/sync-mirror.sh -# MIRROR_PHASE=all upload versioned assets + latest.json (default, -# the original one-step behavior) -# MIRROR_PHASE=stage upload versioned assets + latest.candidate.json -# MIRROR_PHASE=promote health-check candidate, then publish latest.json -# -# holds the release assets + the GitHub latest.json produced -# by gen-updater-manifest.mjs. +# Compatibility entry point for the release workflow. The Node implementation +# owns policy, direct backend verification, conditional promotion, and rollback; +# keeping this wrapper preserves the existing local/CI command surface. set -euo pipefail -dist="${1:?usage: sync-mirror.sh }" -phase="${MIRROR_PHASE:-all}" -mirror_base="${MIRROR_BASE_URL:-https://codexapp.agentsmirror.com/manager}" - -case "$phase" in - all|stage|promote) ;; - *) - echo "::error::unsupported MIRROR_PHASE=$phase (expected all, stage, or promote)" >&2 - exit 2 - ;; -esac - -if [ ! -f "$dist/latest.json" ]; then - echo "::error::no latest.json in $dist" >&2 - exit 1 -fi - -# Installers live under a per-version path on the mirror (see below), so read the -# version once and reuse it for both the rewritten URLs and the upload keys. -version="$(node -e 'process.stdout.write(String(JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")).version||""))' "$dist/latest.json")" -if [ -z "$version" ]; then - echo "::error::latest.json has no version" >&2 - exit 1 -fi - -# 1) Mirror variant of latest.json: same signatures, URLs pointed at the mirror. -# Each installer URL gets a // segment so every release is an -# immutable, uniquely-addressed object. The macOS updater tarballs are renamed -# to versionless arch-only names upstream (CodexAppManager_.app.tar.gz), -# so without this each release would reuse one URL — and the worker's long -# installer cache could then serve a previous version's bytes against the new -# latest.json signature, breaking self-update. latest.json itself stays at the -# fixed root path the updater polls. -node -e ' - const fs = require("fs"); - const [dir, base, ver] = [process.argv[1], process.argv[2], process.argv[3]]; - const j = JSON.parse(fs.readFileSync(dir + "/latest.json", "utf8")); - for (const k of Object.keys(j.platforms || {})) { - const name = j.platforms[k].url.split("/").pop(); - j.platforms[k].url = `${base}/${ver}/${name}`; - } - fs.writeFileSync(dir + "/latest.mirror.json", JSON.stringify(j, null, 2) + "\n"); -' "$dist" "$mirror_base" "$version" - -content_type() { - case "$1" in - *.json) echo "application/json" ;; - *.tar.gz) echo "application/gzip" ;; - *.exe) echo "application/octet-stream" ;; - *.dmg) echo "application/x-apple-diskimage" ;; - *) echo "application/octet-stream" ;; - esac -} - -candidate_key="latest.candidate.json" -latest_key="latest.json" - -# Upload every asset (+ the mirror latest.json) to one S3-compatible endpoint. -# Path-style addressing works for both R2 and IHEP. -upload_assets() { # endpoint bucket region access_key secret_key phase [prefix] - local endpoint="$1" bucket="$2" region="$3" ak="$4" sk="$5" mode="$6" prefix="${7:-}" - local f name key - for f in "$dist"/*; do - name="$(basename "$f")" - [ "$name" = "latest.json" ] && continue # GitHub variant — skip - if [ "$name" = "latest.mirror.json" ]; then - if [ "$mode" = "stage" ]; then - key="$candidate_key" # candidate only; updater never polls it - else - key="$latest_key" # original one-step behavior - fi - else - key="$version/$name" # immutable, per-version object - fi - [ -n "$prefix" ] && key="${prefix%/}/$key" - AWS_ACCESS_KEY_ID="$ak" \ - AWS_SECRET_ACCESS_KEY="$sk" \ - AWS_DEFAULT_REGION="$region" \ - aws s3 cp "$f" "s3://$bucket/$key" \ - --endpoint-url "$endpoint" \ - --content-type "$(content_type "$name")" \ - --only-show-errors - echo " ↑ $key" - done -} - -upload_latest() { # endpoint bucket region access_key secret_key [prefix] - local endpoint="$1" bucket="$2" region="$3" ak="$4" sk="$5" prefix="${6:-}" - local key="$latest_key" - [ -n "$prefix" ] && key="${prefix%/}/$key" - AWS_ACCESS_KEY_ID="$ak" \ - AWS_SECRET_ACCESS_KEY="$sk" \ - AWS_DEFAULT_REGION="$region" \ - aws s3 cp "$dist/latest.mirror.json" "s3://$bucket/$key" \ - --endpoint-url "$endpoint" \ - --content-type "application/json" \ - --only-show-errors - echo " ↑ $key" -} - -check_candidate_health() { - local candidate_url="$mirror_base/$candidate_key" - local candidate urls - candidate="$(mktemp)" - urls="$(mktemp)" - CANDIDATE_TMP="$candidate" - CANDIDATE_URLS_TMP="$urls" - trap 'rm -f "$AWS_CONFIG_FILE" "${CANDIDATE_TMP:-}" "${CANDIDATE_URLS_TMP:-}"' EXIT - - echo "→ health-check $candidate_url" - curl -fsSL --retry 3 --max-time 30 "$candidate_url" -o "$candidate" - node - "$candidate" "$version" > "$urls" <<'NODE' -const fs = require("fs"); -const [path, expectedVersion] = process.argv.slice(2); -const manifest = JSON.parse(fs.readFileSync(path, "utf8")); -if (manifest.version !== expectedVersion) { - console.error(`candidate version ${manifest.version || "(missing)"} does not match ${expectedVersion}`); - process.exit(1); -} -const urls = Object.values(manifest.platforms || {}).map((platform) => platform && platform.url).filter(Boolean); -if (urls.length === 0) { - console.error("candidate has no platform URLs"); - process.exit(1); -} -for (const url of urls) console.log(url); -NODE - - while IFS= read -r url; do - [ -n "$url" ] || continue - echo " HEAD $url" - curl -fsSI --retry 3 --max-time 30 "$url" >/dev/null - done < "$urls" -} - -r2_configured() { - [ -n "${MANAGER_R2_S3_ENDPOINT:-}" ] && [ -n "${MANAGER_R2_ACCESS_KEY_ID:-}" ] && [ -n "${MANAGER_R2_SECRET_ACCESS_KEY:-}" ] -} - -ihep_configured() { - [ -n "${MANAGER_IHEP_S3_ENDPOINT:-}" ] && [ -n "${MANAGER_IHEP_S3_BUCKET:-}" ] && [ -n "${MANAGER_IHEP_S3_ACCESS_KEY_ID:-}" ] && [ -n "${MANAGER_IHEP_S3_SECRET_ACCESS_KEY:-}" ] -} - -sync_r2_assets() { - if r2_configured; then - echo "→ R2 (${MANAGER_R2_BUCKET:-codex-app-manager})" - upload_assets "$MANAGER_R2_S3_ENDPOINT" "${MANAGER_R2_BUCKET:-codex-app-manager}" "auto" \ - "$MANAGER_R2_ACCESS_KEY_ID" "$MANAGER_R2_SECRET_ACCESS_KEY" "$phase" - else - if [ "${ALLOW_STALE_MIRROR:-}" = "1" ]; then - echo "::warning::MANAGER_R2_* not set — skipped R2 mirror sync because ALLOW_STALE_MIRROR=1" - else - echo "::error::MANAGER_R2_* not set — refusing stable release with stale self-update mirror (set ALLOW_STALE_MIRROR=1 only for an intentional manual/pre-release bypass)" >&2 - exit 1 - fi - fi -} - -sync_ihep_assets() { - if ihep_configured; then - echo "→ IHEP S3 (${MANAGER_IHEP_S3_BUCKET})" - upload_assets "$MANAGER_IHEP_S3_ENDPOINT" "$MANAGER_IHEP_S3_BUCKET" "${MANAGER_IHEP_S3_REGION:-auto}" \ - "$MANAGER_IHEP_S3_ACCESS_KEY_ID" "$MANAGER_IHEP_S3_SECRET_ACCESS_KEY" "$phase" "${MANAGER_IHEP_S3_PREFIX:-}" - else - echo "IHEP S3 not configured — CN falls back to R2 via the worker" - fi -} - -promote_r2_latest() { - if r2_configured; then - echo "→ R2 promote (${MANAGER_R2_BUCKET:-codex-app-manager})" - upload_latest "$MANAGER_R2_S3_ENDPOINT" "${MANAGER_R2_BUCKET:-codex-app-manager}" "auto" \ - "$MANAGER_R2_ACCESS_KEY_ID" "$MANAGER_R2_SECRET_ACCESS_KEY" - else - if [ "${ALLOW_STALE_MIRROR:-}" = "1" ]; then - echo "::warning::MANAGER_R2_* not set — skipped R2 latest promote because ALLOW_STALE_MIRROR=1" - else - echo "::error::MANAGER_R2_* not set — cannot promote latest.json" >&2 - exit 1 - fi - fi -} - -promote_ihep_latest() { - if ihep_configured; then - echo "→ IHEP S3 promote (${MANAGER_IHEP_S3_BUCKET})" - upload_latest "$MANAGER_IHEP_S3_ENDPOINT" "$MANAGER_IHEP_S3_BUCKET" "${MANAGER_IHEP_S3_REGION:-auto}" \ - "$MANAGER_IHEP_S3_ACCESS_KEY_ID" "$MANAGER_IHEP_S3_SECRET_ACCESS_KEY" "${MANAGER_IHEP_S3_PREFIX:-}" - else - echo "IHEP S3 not configured — CN falls back to R2 via the worker" - fi -} - -# Force path-style addressing via a temp config — the AWS CLI ignores an -# AWS_S3_ADDRESSING_STYLE env var, and both R2 and IHEP need path-style here. -export AWS_EC2_METADATA_DISABLED=true -AWS_CONFIG_FILE="$(mktemp)" -export AWS_CONFIG_FILE -printf '[default]\nregion = auto\ns3 =\n addressing_style = path\n' > "$AWS_CONFIG_FILE" -trap 'rm -f "$AWS_CONFIG_FILE"' EXIT - -case "$phase" in - all|stage) - sync_r2_assets - sync_ihep_assets - echo "✓ mirror $phase done" - ;; - promote) - if r2_configured; then - check_candidate_health - elif [ "${ALLOW_STALE_MIRROR:-}" = "1" ]; then - echo "::warning::skipped mirror health check because MANAGER_R2_* is not set and ALLOW_STALE_MIRROR=1" - else - echo "::error::MANAGER_R2_* not set — cannot health-check or promote latest.json" >&2 - exit 1 - fi - promote_ihep_latest - promote_r2_latest - echo "✓ mirror promote done" - ;; -esac +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +exec node "$ROOT/scripts/mirror-release.mjs" "$@" diff --git a/scripts/validate-release-manifest.mjs b/scripts/validate-release-manifest.mjs new file mode 100644 index 0000000..ba25f4e --- /dev/null +++ b/scripts/validate-release-manifest.mjs @@ -0,0 +1,37 @@ +#!/usr/bin/env node + +import { readFile } from "node:fs/promises"; + +import { assertCandidateMatchesRelease } from "./mirror-release.mjs"; + +const [, , releaseTag, candidatePath, artifactManifestPath] = process.argv; +if (!releaseTag || !candidatePath || !artifactManifestPath) { + console.error( + "usage: validate-release-manifest.mjs ", + ); + process.exit(2); +} + +const readManifest = async (path, label) => { + try { + return JSON.parse(await readFile(path, "utf8")); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`cannot read ${label}: ${detail}`); + } +}; + +try { + const [candidate, artifactManifest] = await Promise.all([ + readManifest(candidatePath, "candidate latest.json"), + readManifest(artifactManifestPath, "artifact-derived latest.json"), + ]); + const result = assertCandidateMatchesRelease(candidate, artifactManifest, releaseTag); + console.log( + `[manifest] candidate is bound to ${releaseTag} (${result.platformCount} updater platforms)`, + ); +} catch (error) { + const detail = error instanceof Error ? error.message : String(error); + console.error(`::error::[manifest] ${detail}`); + process.exitCode = 1; +} diff --git a/scripts/verify-release-artifacts.mjs b/scripts/verify-release-artifacts.mjs new file mode 100644 index 0000000..7a9f25e --- /dev/null +++ b/scripts/verify-release-artifacts.mjs @@ -0,0 +1,63 @@ +#!/usr/bin/env node + +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { verifyLocalUpdaterArtifacts } from "./mirror-release.mjs"; + +function usage() { + return "usage: verify-release-artifacts.mjs "; +} + +async function readJson(path, label) { + try { + return JSON.parse(await readFile(path, "utf8")); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`cannot read ${label}: ${detail}`); + } +} + +async function main() { + const [manifestArg, distArg, configArg] = process.argv.slice(2); + if (!manifestArg || !distArg || !configArg) throw new Error(usage()); + + const manifest = await readJson(resolve(manifestArg), "updater manifest"); + const config = await readJson(resolve(configArg), "Tauri configuration"); + const publicKey = config.plugins?.updater?.pubkey; + if (typeof publicKey !== "string" || !publicKey.trim()) { + throw new Error("Tauri configuration has no updater public key"); + } + + const report = await verifyLocalUpdaterArtifacts({ + manifest, + distDir: resolve(distArg), + publicKey: publicKey.trim(), + }); + const summaryPath = resolve( + process.env.UPDATER_SIGNATURE_SUMMARY_PATH || + "updater-signature-verification.json", + ); + await writeFile( + summaryPath, + `${JSON.stringify( + { + ...report, + manifestVersion: manifest.version, + verifiedAt: new Date().toISOString(), + }, + null, + 2, + )}\n`, + ); + console.log( + `[release] cryptographically verified ${report.artifactCount} updater artifacts`, + ); +} + +main().catch((error) => { + console.error( + `::error::[release] ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; +}); diff --git a/scripts/write-release-summary.mjs b/scripts/write-release-summary.mjs index 299999d..22f11bf 100644 --- a/scripts/write-release-summary.mjs +++ b/scripts/write-release-summary.mjs @@ -2,7 +2,7 @@ import { appendFileSync, existsSync, readFileSync } from "node:fs"; const env = process.env; -const tag = env.GITHUB_REF_NAME || env.RELEASE_TAG || ""; +const tag = env.RELEASE_TAG || env.GITHUB_REF_NAME || ""; const repo = env.GITHUB_REPOSITORY || "Wangnov/Codex-App-Manager"; const mirrorBase = (env.MIRROR_BASE_URL || "https://codexapp.agentsmirror.com/manager").replace(/\/$/, ""); const summaryPath = env.GITHUB_STEP_SUMMARY; @@ -12,9 +12,27 @@ if (!summaryPath) { process.exit(0); } -const readJson = (path) => (existsSync(path) ? JSON.parse(readFileSync(path, "utf8")) : null); +const readJson = (path) => { + if (!existsSync(path)) return null; + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + return { readError: error instanceof Error ? error.message : String(error) }; + } +}; +const cell = (value) => + String(value ?? "") + .replace(/\s+/g, " ") + .replaceAll("|", "\\|") + .replaceAll("`", "'") + .replaceAll("<", "<") + .replaceAll(">", ">") + .slice(0, 500); const manifest = readJson("latest.json"); const manifestSummary = readJson("manifest-summary.json"); +const mirrorStage = readJson("mirror-stage-summary.json"); +const mirrorVerification = readJson("mirror-verification-summary.json"); +const mirrorPromotion = readJson("mirror-promotion-summary.json"); const version = manifest?.version || (tag ? tag.replace(/^v/, "") : ""); const releaseUrl = `https://github.com/${repo}/releases/tag/${tag}`; const latestUrl = `${mirrorBase}/latest.json`; @@ -69,6 +87,78 @@ if (checksums.length > 0) { rows.push("_SHA256SUMS was not generated._"); } +rows.push(""); +rows.push("## Mirror publication audit"); +rows.push(""); +if (mirrorStage || mirrorVerification || mirrorPromotion) { + const audit = mirrorPromotion || mirrorVerification || mirrorStage; + rows.push(`Candidate: \`${cell(audit?.candidateVersion || version)}\``); + rows.push(`Candidate object: \`${cell(audit?.candidateKey || "unknown")}\``); + rows.push(`Stage outcome: **${cell(mirrorStage?.outcome || "missing")}**`); + rows.push( + `Pre-publish verification: **${cell(mirrorVerification?.outcome || "not-run")}**`, + ); + rows.push(`Promotion outcome: **${cell(mirrorPromotion?.outcome || "not-run")}**`); + rows.push( + `Public route verification: **${cell( + mirrorPromotion?.publicRouteVerification || + mirrorVerification?.publicRouteVerification || + "not-run", + )}**`, + ); + if (mirrorPromotion?.error || mirrorVerification?.error || mirrorStage?.error) { + rows.push( + `Mirror error: \`${cell( + mirrorPromotion?.error || mirrorVerification?.error || mirrorStage?.error, + )}\``, + ); + } + rows.push(""); + rows.push("| Backend | Candidate verification | Previous | Decision | Promotion | Rollback | Final | Error |"); + rows.push("| --- | --- | --- | --- | --- | --- | --- | --- |"); + const backendRows = + mirrorPromotion?.backends || mirrorVerification?.backends || mirrorStage?.backends || []; + for (const backend of backendRows) { + rows.push( + `| ${cell(backend.name)} | ${cell(backend.candidateVerification || backend.status)} | ${cell( + backend.currentVersion || "absent", + )} | ${cell(backend.decision)} | ${cell(backend.promotion)} | ${cell( + backend.rollback, + )} | ${cell(backend.finalVersion)} | ${cell( + backend.error || backend.rollbackError, + )} |`, + ); + } + const override = audit?.override; + rows.push(""); + if (override?.requested) { + const originalActor = + override.originalActor && override.originalActor !== override.actor + ? ` (original workflow actor: \`${cell(override.originalActor)}\`)` + : ""; + rows.push( + `Emergency downgrade override: **requested=${cell(override.requested)}, used=${cell( + override.used, + )}** by \`${cell(override.actor)}\`${originalActor}; reason: ${cell(override.reason)}; [workflow audit](${cell( + override.runUrl, + )}).`, + ); + } else { + rows.push("Emergency downgrade override: not requested."); + } + if (mirrorPromotion?.rollback?.complete === false) { + rows.push( + "**Manual intervention required:** promotion rollback was incomplete; inspect the per-backend errors before another release.", + ); + } else if (["failed", "blocked-downgrade", "rolled-back"].includes(mirrorPromotion?.outcome)) { + rows.push( + "This run left no owned mirror-pointer advance in place; any completed rollback is shown per backend above.", + ); + } +} else { + rows.push("_No mirror audit record was produced (pre-release, skipped job, or failure before staging)._ "); +} + rows.push(""); rows.push("## Job status"); rows.push(""); @@ -76,6 +166,7 @@ rows.push("| Item | Status |"); rows.push("| --- | --- |"); rows.push(`| GitHub Release | ${env.PUBLISH_OUTCOME || "unknown"} |`); rows.push(`| Mirror stage | ${env.MIRROR_STAGE_OUTCOME || "skipped"} |`); +rows.push(`| Mirror pre-publish verify | ${env.MIRROR_VERIFY_OUTCOME || "skipped"} |`); rows.push(`| Mirror promote | ${env.MIRROR_PROMOTE_OUTCOME || "skipped"} |`); rows.push(`| winget dispatch | ${env.WINGET_OUTCOME || "skipped"} |`); rows.push(`| SBOM | ${env.SBOM_OUTCOME || "skipped"} |`); From b8f2e4769388e99fab403d9c916c5bb04ec7b6cf Mon Sep 17 00:00:00 2001 From: wangnov <48670012+Wangnov@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:59:19 +0800 Subject: [PATCH 3/8] fix(release): serialize IHEP mirror promotion --- .github/workflows/release.yml | 44 +- cloudflare/manager-download-router/README.md | 13 +- docs/release.md | 55 +- scripts/mirror-release.mjs | 605 ++++++++++++++--- scripts/mirror-release.test.mjs | 644 ++++++++++++++++++- scripts/release-workflow.test.mjs | 22 + scripts/write-release-summary.mjs | 12 +- 7 files changed, 1232 insertions(+), 163 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4ba4e6..295815f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,13 +20,16 @@ name: Release # MANAGER_R2_S3_ENDPOINT / MANAGER_R2_PROMOTION_ACCESS_KEY_ID / # MANAGER_R2_PROMOTION_SECRET_ACCESS_KEY # R2 S3-compatible read/write access -# MANAGER_IHEP_S3_ENDPOINT / MANAGER_IHEP_S3_BUCKET / -# MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID / +# vars.MANAGER_IHEP_S3_ENDPOINT / vars.MANAGER_IHEP_S3_BUCKET / +# vars.MANAGER_IHEP_S3_REGION / vars.MANAGER_IHEP_S3_PREFIX +# MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID / # MANAGER_IHEP_S3_PROMOTION_SECRET_ACCESS_KEY # IHEP S3-compatible read/write access -# Both mirror backends are required for stable publication. They must already -# contain a valid latest.json baseline and support conditional PutObject -# (If-Match / If-None-Match) plus HeadObject user-metadata round trips. +# Both mirror backends are required for stable publication and must already +# contain a valid latest.json baseline. R2 is the sole CAS authority and must +# enforce conditional PutObject plus HeadObject metadata round trips. IHEP is an +# unconditional single-writer follower and only needs PutObject + metadata +# round trips; promotion never relies on its ignored conditional headers. # GitHub Immutable Releases must be enabled before any release starts; the # workflow verifies the repository setting with IMMUTABLE_RELEASES_READ_TOKEN. # Every reused asset is also pinned to the API-provided sha256 digest. @@ -66,9 +69,10 @@ permissions: env: RELEASE_TAG: ${{ inputs.target_tag || github.ref_name }} -# Serialize every tag in the repository through one release lane. The mirror -# promotion also uses per-object ETag conditions, so an external/manual writer -# cannot bypass monotonicity merely by racing this workflow. +# Serialize every credentialed mirror writer through one release lane. This is a +# correctness boundary for the unconditional IHEP follower, not an optimization. +# R2 CAS plus pre/post ownership fencing handles a losing or accidentally +# overlapping writer, but there is no cross-provider atomic transaction. concurrency: group: release-latest-${{ github.repository }} cancel-in-progress: false @@ -718,10 +722,10 @@ jobs: MANAGER_R2_BUCKET: ${{ vars.MANAGER_R2_BUCKET }} MANAGER_R2_ACCESS_KEY_ID: ${{ secrets.MANAGER_R2_PROMOTION_ACCESS_KEY_ID }} MANAGER_R2_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_R2_PROMOTION_SECRET_ACCESS_KEY }} - MANAGER_IHEP_S3_ENDPOINT: ${{ secrets.MANAGER_IHEP_S3_ENDPOINT }} - MANAGER_IHEP_S3_BUCKET: ${{ secrets.MANAGER_IHEP_S3_BUCKET }} - MANAGER_IHEP_S3_REGION: ${{ secrets.MANAGER_IHEP_S3_REGION }} - MANAGER_IHEP_S3_PREFIX: ${{ secrets.MANAGER_IHEP_S3_PREFIX }} + MANAGER_IHEP_S3_ENDPOINT: ${{ vars.MANAGER_IHEP_S3_ENDPOINT }} + MANAGER_IHEP_S3_BUCKET: ${{ vars.MANAGER_IHEP_S3_BUCKET }} + MANAGER_IHEP_S3_REGION: ${{ vars.MANAGER_IHEP_S3_REGION }} + MANAGER_IHEP_S3_PREFIX: ${{ vars.MANAGER_IHEP_S3_PREFIX }} MANAGER_IHEP_S3_ACCESS_KEY_ID: ${{ secrets.MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID }} MANAGER_IHEP_S3_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_IHEP_S3_PROMOTION_SECRET_ACCESS_KEY }} run: | @@ -751,10 +755,10 @@ jobs: MANAGER_R2_BUCKET: ${{ vars.MANAGER_R2_BUCKET }} MANAGER_R2_ACCESS_KEY_ID: ${{ secrets.MANAGER_R2_PROMOTION_ACCESS_KEY_ID }} MANAGER_R2_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_R2_PROMOTION_SECRET_ACCESS_KEY }} - MANAGER_IHEP_S3_ENDPOINT: ${{ secrets.MANAGER_IHEP_S3_ENDPOINT }} - MANAGER_IHEP_S3_BUCKET: ${{ secrets.MANAGER_IHEP_S3_BUCKET }} - MANAGER_IHEP_S3_REGION: ${{ secrets.MANAGER_IHEP_S3_REGION }} - MANAGER_IHEP_S3_PREFIX: ${{ secrets.MANAGER_IHEP_S3_PREFIX }} + MANAGER_IHEP_S3_ENDPOINT: ${{ vars.MANAGER_IHEP_S3_ENDPOINT }} + MANAGER_IHEP_S3_BUCKET: ${{ vars.MANAGER_IHEP_S3_BUCKET }} + MANAGER_IHEP_S3_REGION: ${{ vars.MANAGER_IHEP_S3_REGION }} + MANAGER_IHEP_S3_PREFIX: ${{ vars.MANAGER_IHEP_S3_PREFIX }} MANAGER_IHEP_S3_ACCESS_KEY_ID: ${{ secrets.MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID }} MANAGER_IHEP_S3_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_IHEP_S3_PROMOTION_SECRET_ACCESS_KEY }} run: | @@ -913,10 +917,10 @@ jobs: MANAGER_R2_BUCKET: ${{ vars.MANAGER_R2_BUCKET }} MANAGER_R2_ACCESS_KEY_ID: ${{ secrets.MANAGER_R2_PROMOTION_ACCESS_KEY_ID }} MANAGER_R2_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_R2_PROMOTION_SECRET_ACCESS_KEY }} - MANAGER_IHEP_S3_ENDPOINT: ${{ secrets.MANAGER_IHEP_S3_ENDPOINT }} - MANAGER_IHEP_S3_BUCKET: ${{ secrets.MANAGER_IHEP_S3_BUCKET }} - MANAGER_IHEP_S3_REGION: ${{ secrets.MANAGER_IHEP_S3_REGION }} - MANAGER_IHEP_S3_PREFIX: ${{ secrets.MANAGER_IHEP_S3_PREFIX }} + MANAGER_IHEP_S3_ENDPOINT: ${{ vars.MANAGER_IHEP_S3_ENDPOINT }} + MANAGER_IHEP_S3_BUCKET: ${{ vars.MANAGER_IHEP_S3_BUCKET }} + MANAGER_IHEP_S3_REGION: ${{ vars.MANAGER_IHEP_S3_REGION }} + MANAGER_IHEP_S3_PREFIX: ${{ vars.MANAGER_IHEP_S3_PREFIX }} MANAGER_IHEP_S3_ACCESS_KEY_ID: ${{ secrets.MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID }} MANAGER_IHEP_S3_SECRET_ACCESS_KEY: ${{ secrets.MANAGER_IHEP_S3_PROMOTION_SECRET_ACCESS_KEY }} run: | diff --git a/cloudflare/manager-download-router/README.md b/cloudflare/manager-download-router/README.md index bb08eea..bdfd10f 100644 --- a/cloudflare/manager-download-router/README.md +++ b/cloudflare/manager-download-router/README.md @@ -64,14 +64,15 @@ So each release uploads to the mirror (`release.yml` → `scripts/sync-mirror.sh | --- | --- | | `MANAGER_R2_S3_ENDPOINT` | `https://d39dc6c92d1c4cfde580bf13e946b616.r2.cloudflarestorage.com` | | `MANAGER_R2_PROMOTION_ACCESS_KEY_ID` / `MANAGER_R2_PROMOTION_SECRET_ACCESS_KEY` | R2 S3 API token for the current protected release workflow | -| `MANAGER_IHEP_S3_ENDPOINT` / `_BUCKET` | IHEP endpoint and bucket (`_REGION`/`_PREFIX` optional) | +| `MANAGER_IHEP_S3_ENDPOINT` / `_BUCKET` environment variables | IHEP endpoint and bucket (`_REGION`/`_PREFIX` variables optional) | | `MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID` / `MANAGER_IHEP_S3_PROMOTION_SECRET_ACCESS_KEY` | IHEP token for the current protected release workflow | -Both backends must already contain the same valid `latest.json` baseline and must -support conditional `PutObject` (`If-Match` / `If-None-Match`) plus custom -user-metadata round trips through `HeadObject`. Delete the legacy access-key secret -names after migration so historical workflow revisions cannot perform their old -unconditional write. +Both backends must already contain the same valid `latest.json` baseline and +preserve custom user metadata through `HeadObject`. R2 is the sole CAS authority +and must enforce conditional `PutObject` (`If-Match` / `If-None-Match`). IHEP is +the serialized unconditional follower; its ignored conditional headers are not a +promotion prerequisite. Delete the legacy access-key secret names after migration +so historical workflow revisions cannot perform their old unconditional write. Deploy this Worker revision before enabling the protected release workflow. The release gate intentionally rejects an older Worker that cannot force and identify diff --git a/docs/release.md b/docs/release.md index ff9369a..3e3489e 100644 --- a/docs/release.md +++ b/docs/release.md @@ -94,21 +94,38 @@ Stable releases use a stage/verify/promote, dual-backend protocol implemented by immutable with canonical asset digests. 5. Repeat the complete direct-backend and public-route readback immediately before promotion, then read each backend's current `latest.json` and compare semantic - versions. Newer candidates advance, same-version reruns are - no-write/idempotent, and older tags fail closed. -6. Publish with ETag (`If-Match` / `If-None-Match`) conditions. If the second - backend or final verification fails, restore every backend already changed, - also using ETag conditions so rollback cannot overwrite a concurrent writer. + versions. Newer candidates advance, a fully converged same-version rerun is a + no-write success, and older tags fail closed. If a hard-killed run left R2 on + the candidate while IHEP still lags, the rerun first reclaims R2 with CAS. +6. Treat R2 as the only linearization authority. The workflow conditionally + writes R2, verifies the committed ETag and promotion token, then writes IHEP + unconditionally as a follower. A CAS loser never writes IHEP. The winner + checks R2 ownership immediately before and after the follower write. If it is + superseded during that window, it either preserves the newer follower or + repairs only its own IHEP value from the newer stable R2 snapshot. If another + repair already moved IHEP to the exact R2 candidate after the CAS, that + follower is accepted. A higher version or any identity that cannot be proven + canonical is preserved but fails closed; it cannot force the owned R2 CAS to + roll back and it is never overwritten by the older run. +7. If IHEP fails, roll R2 back only while its committed ETag and token are still + owned by this transaction. IHEP is restored only when it still contains this + transaction's token and bytes; an unchanged baseline or a concurrent value is + preserved rather than overwritten. The release workflow has one repository-wide `release-latest-*` concurrency lane with `queue: max`, so every pending tag remains queued instead of a third tag -replacing the second. The object-store conditions remain the independent backstop -for manual/external writes. `mirror-stage-summary.json`, +replacing the second. This single-writer lane and the promotion-only credential +names are a correctness boundary because IHEP does not enforce conditional +writes. Do not run another credentialed promotion workflow outside this lane. +R2 CAS and the before/after ownership checks are defense in depth for accidental +overlap; they are not a claim of an atomic transaction across providers. +`mirror-stage-summary.json`, `mirror-verification-summary.json`, and `mirror-promotion-summary.json` are shown in the job summary and retained as workflow artifacts for 90 days. If a runner is -hard-killed after only one -`latest.json` write, a fresh run recognizes the exact candidate identity, leaves -that backend untouched, and conditionally advances only the lagging backend. +hard-killed after the R2 CAS but before IHEP follows, rerun that release or a newer +one: the new run reclaims R2 with CAS and converges IHEP. A hard kill during an +out-of-policy concurrent follower race can still leave IHEP temporarily stale; +rerunning the release currently authoritative in R2 is the recovery procedure. Before `prepare` permits any build, and again at the start of every release-job attempt, the workflow queries the repository Immutable Releases setting with a @@ -171,10 +188,6 @@ used, and the run URL are written to the promotion audit and job summary. | `MANAGER_R2_S3_ENDPOINT` | R2 S3-compatible endpoint | | `MANAGER_R2_PROMOTION_ACCESS_KEY_ID` | R2 write/read credential used only by the protected workflow | | `MANAGER_R2_PROMOTION_SECRET_ACCESS_KEY` | R2 write/read secret used only by the protected workflow | -| `MANAGER_IHEP_S3_ENDPOINT` | IHEP S3-compatible endpoint | -| `MANAGER_IHEP_S3_BUCKET` | IHEP bucket | -| `MANAGER_IHEP_S3_REGION` | IHEP region; defaults to `auto` when empty | -| `MANAGER_IHEP_S3_PREFIX` | optional object-key prefix for IHEP | | `MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID` | IHEP write/read credential used only by the protected workflow | | `MANAGER_IHEP_S3_PROMOTION_SECRET_ACCESS_KEY` | IHEP write/read secret used only by the protected workflow | @@ -193,6 +206,10 @@ those exact names; leaving any of them available defeats old-run isolation. | `AUTHENTICODE_REQUIRED` (repo **variable**) | `true` → fail release when PE is not `Valid` | | `WINDOWS_TIMESTAMP_URL` (repo **variable**) | optional RFC3161 timestamp URL | | `MANAGER_R2_BUCKET` (repo **variable**) | R2 bucket; defaults to `codex-app-manager` | +| `MANAGER_IHEP_S3_ENDPOINT` (`release` environment **variable**) | IHEP S3-compatible endpoint | +| `MANAGER_IHEP_S3_BUCKET` (`release` environment **variable**) | IHEP bucket | +| `MANAGER_IHEP_S3_REGION` (`release` environment **variable**) | IHEP region; defaults to `auto` when empty | +| `MANAGER_IHEP_S3_PREFIX` (`release` environment **variable**) | optional object-key prefix for IHEP | Export your local .p12 / .p8 / .pfx to base64 with `base64 -i file -o -`. @@ -204,7 +221,9 @@ Export your local .p12 / .p8 / .pfx to base64 with `base64 -i file -o -`. > conceptually separate — see [`docs/windows-signing.md`](./windows-signing.md). > > Before enabling promotion, seed both S3-compatible endpoints with a valid -> `latest.json` baseline. Both endpoints must support conditional `PutObject` -> requests (`If-Match` / `If-None-Match`) and must preserve custom user metadata -> through `HeadObject`. Promotion fails closed if either baseline is absent or an -> endpoint cannot enforce those requirements. +> `latest.json` baseline. R2 must enforce conditional `PutObject` requests +> (`If-Match` / `If-None-Match`) and preserve custom user metadata through +> `HeadObject`. IHEP must preserve metadata and support ordinary read/write, but +> it is explicitly allowed to ignore conditional headers because the workflow +> uses it only as the serialized unconditional follower. Promotion fails closed +> if either baseline is absent. diff --git a/scripts/mirror-release.mjs b/scripts/mirror-release.mjs index c4bda1f..b7c5b9a 100755 --- a/scripts/mirror-release.mjs +++ b/scripts/mirror-release.mjs @@ -914,6 +914,16 @@ export class AwsObjectStore { singleAttempt: true, }); } + + async putLatestUnconditional(localPath, promotionToken = "") { + return await this.putObject(localPath, LATEST_KEY, { + contentType: "application/json", + ...(promotionToken + ? { metadata: { "cam-promotion-token": promotionToken } } + : {}), + singleAttempt: true, + }); + } } function contentType(name) { @@ -1178,7 +1188,7 @@ async function readCurrentState(backend, workDir, candidateManifest) { const snapshot = await backend.snapshot(LATEST_KEY, path); if (!snapshot.exists) { throw new Error( - `${backend.name}: latest.json is absent; seed both backends with the same valid baseline before enabling transactional promotion`, + `${backend.name}: latest.json is absent; seed both backends with the same valid baseline before enabling single-writer promotion`, ); } const manifest = await readJson(path, `${backend.name} current latest.json`); @@ -1246,54 +1256,305 @@ async function withRollbackIo(callback) { } } -async function rollbackTouched(touched, summaryByName, workDir) { +function promotionBackends(backends) { + const r2 = backends.filter((backend) => backend.name === "r2"); + const ihep = backends.filter((backend) => backend.name === "ihep"); + if (backends.length !== 2 || r2.length !== 1 || ihep.length !== 1) { + throw new Error("mirror promotion requires exactly one r2 authority and one ihep follower"); + } + return { ihep: ihep[0], r2: r2[0] }; +} + +async function assertSnapshotUnchanged(state) { + const current = await state.backend.head(LATEST_KEY); + if (!current || current.etag !== state.previous.etag) { + throw new ConditionalWriteError( + `${state.backend.name}: latest.json changed after the promotion snapshot`, + ); + } +} + +async function classifyFollowerCoverage(snapshot, candidateManifest, label) { + if (!snapshot.exists) throw new Error(`${label}: latest.json disappeared`); + const manifest = await readJson(snapshot.path, `${label} latest.json`); + const followerVersion = assertManifestShape(manifest, `${label} latest.json`); + const candidateVersion = assertManifestShape(candidateManifest, "candidate latest.json"); + const comparison = compareSemver(followerVersion, candidateVersion); + if (comparison > 0) { + return { coverage: "higher-version", manifest, version: followerVersion }; + } + if ( + comparison === 0 && + manifestReleaseIdentity(manifest) === manifestReleaseIdentity(candidateManifest) + ) { + return { coverage: "candidate", manifest, version: followerVersion }; + } + if (comparison === 0) { + throw new Error( + `${label}: latest.json has the candidate version but different artifact/signature identity`, + ); + } + return { coverage: "behind", manifest, version: followerVersion }; +} + +async function refreshFollowerAfterAuthorityCas(state, candidateManifest, workDir) { + const snapshot = await state.backend.snapshot( + LATEST_KEY, + join(workDir, "ihep-post-authority-cas.json"), + ); + if (!snapshot.exists) throw new Error("ihep: latest.json disappeared after the R2 CAS"); + if (snapshot.etag === state.previous.etag) { + return { coverage: "snapshot-unchanged", needsWrite: true, snapshot }; + } + const classified = await classifyFollowerCoverage( + snapshot, + candidateManifest, + "IHEP after R2 CAS", + ); + if (classified.coverage !== "candidate") { + throw new ConditionalWriteError( + `ihep: latest.json changed after the R2 CAS to ${classified.version}; exact candidate identity is required`, + ); + } + return { ...classified, needsWrite: false, snapshot }; +} + +async function ownedCandidateSnapshot({ + state, + candidatePath, + expectedEtag, + promotionToken, + workDir, + label, +}) { + const snapshot = await state.backend.snapshot( + LATEST_KEY, + join(workDir, `${state.backend.name}-${label}.json`), + ); + if ( + !snapshot.exists || + snapshot.etag !== expectedEtag || + snapshot.metadata?.["cam-promotion-token"] !== promotionToken || + !(await snapshotMatchesFile(snapshot, candidatePath)) + ) { + throw new ConditionalWriteError( + `${state.backend.name}: latest.json is no longer owned by this promotion`, + ); + } + return snapshot; +} + +async function rollbackAuthoritativePromotion({ + r2State, + ihepState, + r2Commit, + followerWriteAttempted, + candidatePath, + promotionToken, + summaryByName, + workDir, +}) { const failures = []; + const r2Summary = summaryByName.get("r2"); + const ihepSummary = summaryByName.get("ihep"); rollbackInProgress = true; try { - for (const item of [...touched].reverse()) { - const backendSummary = summaryByName.get(item.state.backend.name); + try { + await ownedCandidateSnapshot({ + state: r2State, + candidatePath, + expectedEtag: r2Commit.etag, + promotionToken, + workDir, + label: "rollback-ownership", + }); + } catch (error) { + r2Summary.rollback = "skipped-concurrent-change"; + r2Summary.rollbackError = safeSummaryError(error); + if (followerWriteAttempted) ihepSummary.rollback = "preserved-authority-lost"; + failures.push(`r2: ${errorText(error)}`); + return { failures, rolledBack: false }; + } + + try { + if (!r2State.previous.exists) { + throw new Error("authoritative promotion cannot roll back an unseeded R2 baseline"); + } + const restored = await r2State.backend.putLatestConditional( + r2State.previous.path, + r2Commit.etag, + `rollback-${randomUUID()}`, + ); + const check = await r2State.backend.snapshot( + LATEST_KEY, + join(workDir, "r2-rollback-check.json"), + ); + if ( + check.etag !== restored.etag || + !(await snapshotMatchesFile(check, r2State.previous.path)) + ) { + throw new Error("R2 rollback verification did not match the previous latest.json"); + } + r2Summary.rollback = "restored"; + } catch (error) { + r2Summary.rollback = "failed"; + r2Summary.rollbackError = safeSummaryError(error); + failures.push(`r2: ${errorText(error)}`); + if (followerWriteAttempted) ihepSummary.rollback = "preserved-authority-rollback-failed"; + return { failures, rolledBack: false }; + } + + if (followerWriteAttempted) { try { - const owned = await item.state.backend.snapshot( + const current = await ihepState.backend.snapshot( LATEST_KEY, - join(workDir, `${item.state.backend.name}-rollback-ownership.json`), + join(workDir, "ihep-rollback-current.json"), ); - if ( - !owned.exists || - owned.etag !== item.promotedEtag || - owned.metadata?.["cam-promotion-token"] !== item.promotionToken + if (await snapshotMatchesFile(current, ihepState.previous.path)) { + ihepSummary.rollback = "preserved-previous"; + } else if ( + current.exists && + current.metadata?.["cam-promotion-token"] === promotionToken && + (await snapshotMatchesFile(current, candidatePath)) ) { - throw new ConditionalWriteError( - `${item.state.backend.name}: latest.json is no longer owned by this promotion`, - ); - } - if (item.state.previous.exists) { - const restored = await item.state.backend.putLatestConditional( - item.state.previous.path, - item.promotedEtag, + const restored = await ihepState.backend.putLatestUnconditional( + ihepState.previous.path, `rollback-${randomUUID()}`, ); - const check = await item.state.backend.snapshot( + const check = await ihepState.backend.snapshot( LATEST_KEY, - join(workDir, `${item.state.backend.name}-rollback-check.json`), + join(workDir, "ihep-rollback-check.json"), ); - if (!(await snapshotMatchesFile(check, item.state.previous.path))) { - throw new Error("rollback verification did not match previous latest.json"); + if ( + check.etag !== restored.etag || + !(await snapshotMatchesFile(check, ihepState.previous.path)) + ) { + throw new Error("IHEP rollback verification did not match the previous latest.json"); } - item.promotedEtag = restored.etag; + ihepSummary.rollback = "restored-unconditionally"; } else { - throw new Error("transactional promotion cannot roll back an unseeded backend"); + // IHEP cannot enforce conditional writes. If its object is neither the + // snapshot nor this transaction's token+bytes, never overwrite it. + ihepSummary.rollback = "preserved-concurrent-change"; } - backendSummary.rollback = "restored"; } catch (error) { - backendSummary.rollback = "failed"; - backendSummary.rollbackError = safeSummaryError(error); - failures.push(`${item.state.backend.name}: ${errorText(error)}`); + ihepSummary.rollback = "failed"; + ihepSummary.rollbackError = safeSummaryError(error); + failures.push(`ihep: ${errorText(error)}`); } } } finally { rollbackInProgress = false; } - return failures; + return { failures, rolledBack: true }; +} + +async function reconcileFollowerToCurrentAuthority({ + r2State, + ihepState, + candidateManifest, + candidatePath, + promotionToken, + summaryByName, + workDir, +}) { + const authority = await r2State.backend.snapshot( + LATEST_KEY, + join(workDir, "r2-superseding-authority.json"), + ); + if (!authority.exists) throw new Error("r2: superseding latest.json disappeared"); + const authorityManifest = await readJson( + authority.path, + "R2 superseding authoritative latest.json", + ); + const authorityVersion = assertManifestShape( + authorityManifest, + "R2 superseding authoritative latest.json", + ); + const candidateVersion = assertManifestShape(candidateManifest, "candidate latest.json"); + const comparison = compareSemver(authorityVersion, candidateVersion); + if ( + comparison < 0 || + (comparison === 0 && + manifestReleaseIdentity(authorityManifest) !== + manifestReleaseIdentity(candidateManifest)) + ) { + throw new Error( + `r2: superseding authority ${authorityVersion} is not a safe monotonic successor of ${candidateVersion}`, + ); + } + + const currentFollower = await ihepState.backend.snapshot( + LATEST_KEY, + join(workDir, "ihep-supersession-current.json"), + ); + if (!currentFollower.exists) { + throw new Error("ihep: latest.json disappeared during supersession repair"); + } + if (await snapshotMatchesFile(currentFollower, authority.path)) { + summaryByName.get("ihep").supersession = "already-follows-r2"; + return authorityManifest; + } + + const followerManifest = await readJson( + currentFollower.path, + "IHEP latest.json during supersession repair", + ); + const followerVersion = assertManifestShape( + followerManifest, + "IHEP latest.json during supersession repair", + ); + if (compareSemver(followerVersion, authorityVersion) > 0) { + // Never let an older workflow overwrite a follower that has already moved + // beyond the R2 snapshot it observed. + summaryByName.get("ihep").supersession = "preserved-newer-follower"; + return followerManifest; + } + + const ownsFollower = + (currentFollower.metadata?.["cam-promotion-token"] === promotionToken && + (await snapshotMatchesFile(currentFollower, candidatePath))) || + (currentFollower.etag === ihepState.previous.etag && + (await snapshotMatchesFile(currentFollower, ihepState.previous.path))); + if (!ownsFollower) { + throw new ConditionalWriteError( + "ihep: cannot safely repair a follower object owned by another writer", + ); + } + + const repaired = await ihepState.backend.putLatestUnconditional( + authority.path, + `superseded-${randomUUID()}`, + ); + const check = await ihepState.backend.snapshot( + LATEST_KEY, + join(workDir, "ihep-supersession-check.json"), + ); + if ( + check.etag === repaired.etag && + (await snapshotMatchesFile(check, authority.path)) + ) { + summaryByName.get("ihep").supersession = "repaired-to-r2"; + return authorityManifest; + } + + if (check.exists) { + const racedManifest = await readJson( + check.path, + "IHEP latest.json after supersession repair race", + ); + const racedVersion = assertManifestShape( + racedManifest, + "IHEP latest.json after supersession repair race", + ); + if (compareSemver(racedVersion, authorityVersion) >= 0) { + summaryByName.get("ihep").supersession = "preserved-racing-successor"; + return racedManifest; + } + } + throw new ConditionalWriteError( + "ihep: follower changed to an older value during supersession repair", + ); } export async function promoteCandidateTransaction({ @@ -1306,9 +1567,10 @@ export async function promoteCandidateTransaction({ hooks = {}, promotionToken = randomUUID(), }) { + const { r2, ihep } = promotionBackends(backends); const summaryByName = new Map(summary.backends.map((entry) => [entry.name, entry])); const states = []; - for (const backend of backends) { + for (const backend of [r2, ihep]) { try { states.push(await readCurrentState(backend, workDir, candidateManifest)); } catch (error) { @@ -1348,8 +1610,11 @@ export async function promoteCandidateTransaction({ } } - const toPromote = states.filter((state) => state.decision !== "idempotent"); - if (toPromote.length === 0) { + const r2State = states.find((state) => state.backend === r2); + const ihepState = states.find((state) => state.backend === ihep); + let followerNeedsWrite = ihepState.decision !== "idempotent"; + const authorityNeedsWrite = r2State.decision !== "idempotent"; + if (!authorityNeedsWrite && !followerNeedsWrite) { for (const state of states) { const backendSummary = summaryByName.get(state.backend.name); try { @@ -1383,105 +1648,237 @@ export async function promoteCandidateTransaction({ return { outcome: "idempotent", states }; } - const touched = []; - let activeState = null; + if (!authorityNeedsWrite && followerNeedsWrite) { + r2State.decision = override.used + ? "claim-follower-downgrade-override" + : "claim-follower-reconciliation"; + summaryByName.get("r2").decision = r2State.decision; + } + + let activeState = r2State; + let r2Commit = null; + let followerWriteAttempted = false; + let followerCommit = null; + let supersededByAuthority = false; + let supersessionFailures = []; + let followerConflictAfterAuthority = false; try { - for (const state of toPromote) { - activeState = state; - const backendSummary = summaryByName.get(state.backend.name); - await hooks.beforeWrite?.(state); - let promoted; - try { - promoted = await state.backend.putLatestConditional( + // R2 is the only linearization authority. The follower snapshot must still + // be current before this workflow attempts the authoritative CAS. + await assertSnapshotUnchanged(ihepState); + await hooks.beforeWrite?.(r2State); + try { + r2Commit = await r2.putLatestConditional( + candidatePath, + r2State.previous.etag, + promotionToken, + ); + summaryByName.get("r2").promotion = "written-authoritative-cas"; + } catch (error) { + if (error instanceof ConditionalWriteError) throw error; + const ambiguous = await withRollbackIo(() => + observeAmbiguousWrite( + r2State, candidatePath, - state.previous.etag, + workDir, promotionToken, + ), + ); + if (!ambiguous) { + summaryByName.get("r2").promotion = "write-outcome-unresolved"; + summaryByName.get("r2").rollback = "uncertain"; + throw new WriteOutcomeUncertainError( + `r2: write failed and ownership could not be resolved after ${errorText(error)}`, + ); + } + r2Commit = ambiguous; + summaryByName.get("r2").promotion = "write-outcome-ambiguous"; + throw error; + } + + await ownedCandidateSnapshot({ + state: r2State, + candidatePath, + expectedEtag: r2Commit.etag, + promotionToken, + workDir, + label: "authoritative-check", + }); + summaryByName.get("r2").promotion = "verified-authoritative-cas"; + await hooks.afterWrite?.(r2State); + + if (followerNeedsWrite) { + await ownedCandidateSnapshot({ + state: r2State, + candidatePath, + expectedEtag: r2Commit.etag, + promotionToken, + workDir, + label: "pre-follower-ownership", + }); + activeState = ihepState; + let refreshedFollower; + try { + refreshedFollower = await refreshFollowerAfterAuthorityCas( + ihepState, + candidateManifest, + workDir, ); } catch (error) { - if (error instanceof ConditionalWriteError) { - throw error; - } - const ambiguous = await withRollbackIo(() => + followerConflictAfterAuthority = true; + summaryByName.get("r2").rollback = "preserved-authoritative-cas"; + summaryByName.get("ihep").promotion = "conflict-after-authority-cas"; + summaryByName.get("ihep").rollback = "preserved-unowned-follower"; + if (error instanceof ConditionalWriteError) throw error; + throw new ConditionalWriteError( + `ihep: cannot prove exact candidate identity after the R2 CAS: ${errorText(error)}`, + ); + } + if (!refreshedFollower.needsWrite) { + followerNeedsWrite = false; + summaryByName.get("ihep").promotion = "already-current-after-authority-cas"; + } + } + + if (followerNeedsWrite) { + activeState = ihepState; + await hooks.beforeWrite?.(ihepState); + followerWriteAttempted = true; + try { + followerCommit = await ihep.putLatestUnconditional(candidatePath, promotionToken); + summaryByName.get("ihep").promotion = "written-unconditional-follower"; + } catch (error) { + const observed = await withRollbackIo(() => observeAmbiguousWrite( - state, + ihepState, candidatePath, workDir, promotionToken, ), ); - if (ambiguous) { - touched.push({ - state, - promotedEtag: ambiguous.etag, - promotionToken, - }); - backendSummary.promotion = "write-outcome-ambiguous"; - } else { - backendSummary.promotion = "write-outcome-unresolved"; - backendSummary.rollback = "uncertain"; - throw new WriteOutcomeUncertainError( - `${state.backend.name}: write failed and ownership could not be resolved after ${errorText(error)}`, - ); + if (!observed) { + summaryByName.get("ihep").promotion = "write-outcome-unresolved"; + throw error; } - throw error; + followerCommit = observed; + summaryByName.get("ihep").promotion = "write-outcome-confirmed"; } - const touchedItem = { - state, - promotedEtag: promoted.etag, + await ownedCandidateSnapshot({ + state: ihepState, + candidatePath, + expectedEtag: followerCommit.etag, promotionToken, - }; - touched.push(touchedItem); - backendSummary.promotion = "written"; - const check = await state.backend.snapshot( - LATEST_KEY, - join(workDir, `${state.backend.name}-promoted-check.json`), - ); - if ( - check.etag !== promoted.etag || - check.metadata?.["cam-promotion-token"] !== promotionToken - ) { - touched.pop(); - backendSummary.promotion = "ownership-lost"; - backendSummary.rollback = "skipped-concurrent-change"; - throw new ConditionalWriteError( - `${state.backend.name}: latest.json changed after the conditional write`, + workDir, + label: "follower-check", + }); + summaryByName.get("ihep").promotion = "verified-unconditional-follower"; + await hooks.afterWrite?.(ihepState); + } else if (summaryByName.get("ihep").promotion === "not-started") { + summaryByName.get("ihep").promotion = "already-current"; + } + + activeState = r2State; + try { + await ownedCandidateSnapshot({ + state: r2State, + candidatePath, + expectedEtag: r2Commit.etag, + promotionToken, + workDir, + label: "final-authority", + }); + } catch (ownershipError) { + // The protected release workflow is serialized, but keep a fencing repair + // for accidental/manual overlap. An older writer that was superseded + // between its pre-IHEP check and its unconditional IHEP write repairs only + // an object it still owns, using the newer stable R2 snapshot. + supersededByAuthority = true; + summaryByName.get("r2").rollback = "skipped-superseded"; + try { + await withRollbackIo(() => + reconcileFollowerToCurrentAuthority({ + r2State, + ihepState, + candidateManifest, + candidatePath, + promotionToken, + summaryByName, + workDir, + }), ); + } catch (repairError) { + supersessionFailures = [`ihep: ${errorText(repairError)}`]; } - if (!(await snapshotMatchesFile(check, candidatePath))) { - throw new Error(`${state.backend.name}: promoted latest.json failed byte verification`); - } - backendSummary.promotion = "verified"; - await hooks.afterWrite?.(state); + throw ownershipError; } + summaryByName.get("r2").finalVersion = candidateManifest.version; - for (const state of states) { - activeState = state; - const finalSnapshot = await state.backend.snapshot( + activeState = ihepState; + let finalFollower; + let finalFollowerCoverage; + try { + finalFollower = await ihep.snapshot( LATEST_KEY, - join(workDir, `${state.backend.name}-final-latest.json`), + join(workDir, "ihep-final-latest.json"), + ); + finalFollowerCoverage = await classifyFollowerCoverage( + finalFollower, + candidateManifest, + "IHEP final", ); - if (!finalSnapshot.exists) throw new Error(`${state.backend.name}: latest.json disappeared`); - const finalManifest = await readJson( - finalSnapshot.path, - `${state.backend.name} final latest.json`, + } catch (error) { + followerConflictAfterAuthority = true; + summaryByName.get("r2").rollback = "preserved-authoritative-cas"; + summaryByName.get("ihep").rollback = "preserved-unowned-follower"; + throw new ConditionalWriteError( + `ihep: cannot prove exact final candidate identity: ${errorText(error)}`, ); - if (manifestReleaseIdentity(finalManifest) !== manifestReleaseIdentity(candidateManifest)) { - throw new Error(`${state.backend.name}: final latest.json does not expose the candidate release`); - } - summaryByName.get(state.backend.name).finalVersion = finalManifest.version; } + if (finalFollowerCoverage.coverage !== "candidate") { + followerConflictAfterAuthority = true; + summaryByName.get("r2").rollback = "preserved-authoritative-cas"; + summaryByName.get("ihep").rollback = "preserved-unowned-follower"; + throw new ConditionalWriteError( + `ihep: final latest.json is ${finalFollowerCoverage.version}; exact candidate identity is required`, + ); + } + if ( + followerCommit && + (finalFollower.etag !== followerCommit.etag || + finalFollower.metadata?.["cam-promotion-token"] !== promotionToken) + ) { + summaryByName.get("ihep").supersession = + "already-follows-r2"; + } + summaryByName.get("ihep").finalVersion = finalFollowerCoverage.manifest.version; return { outcome: override.used ? "downgrade-override-promoted" : "promoted", states }; } catch (error) { if (activeState) { summaryByName.get(activeState.backend.name).error = safeSummaryError(error); } - const rollbackFailures = await rollbackTouched(touched, summaryByName, workDir); - const ownershipUncertain = error instanceof WriteOutcomeUncertainError; - const failures = ownershipUncertain - ? [...rollbackFailures, `${activeState?.backend.name || "backend"}: write ownership unresolved`] - : rollbackFailures; + let failures = []; + let rolledBack = false; + if (followerConflictAfterAuthority) { + summary.authorityPreserved = true; + } else if (supersededByAuthority) { + failures = supersessionFailures; + } else if (r2Commit) { + ({ failures, rolledBack } = await rollbackAuthoritativePromotion({ + r2State, + ihepState, + r2Commit, + followerWriteAttempted, + candidatePath, + promotionToken, + summaryByName, + workDir, + })); + } else if (error instanceof WriteOutcomeUncertainError) { + failures = ["r2: write ownership unresolved"]; + } summary.rollback = { - attempted: touched.length > 0, + attempted: + Boolean(r2Commit) && !supersededByAuthority && !followerConflictAfterAuthority, complete: failures.length === 0, failures, }; @@ -1490,7 +1887,7 @@ export async function promoteCandidateTransaction({ `${errorText(error)}; rollback incomplete: ${failures.join("; ")}`, ); } - if (touched.length > 0) summary.outcome = "rolled-back"; + if (rolledBack) summary.outcome = "rolled-back"; throw error; } } diff --git a/scripts/mirror-release.test.mjs b/scripts/mirror-release.test.mjs index 3039574..a600896 100644 --- a/scripts/mirror-release.test.mjs +++ b/scripts/mirror-release.test.mjs @@ -143,6 +143,8 @@ class MemoryBackend { this.objects = new Map(); this.counter = 0; this.latestPutAttempts = 0; + this.conditionalLatestPutAttempts = 0; + this.unconditionalLatestPutAttempts = 0; for (const [key, body] of Object.entries(objects)) this.set(key, body); } @@ -196,6 +198,7 @@ class MemoryBackend { async putLatestConditional(localPath, expectedEtag, promotionToken = "") { this.latestPutAttempts += 1; + this.conditionalLatestPutAttempts += 1; const current = this.objects.get("latest.json"); const matches = expectedEtag ? current?.etag === expectedEtag : !current; if (!matches) throw new ConditionalWriteError(`${this.name}: stale latest ETag`); @@ -209,6 +212,19 @@ class MemoryBackend { return { etag: stored.etag, size: stored.body.length }; } + async putLatestUnconditional(localPath, promotionToken = "") { + this.latestPutAttempts += 1; + this.unconditionalLatestPutAttempts += 1; + if (this.failLatestPut) throw new Error(`${this.name}: simulated write failure`); + this.set( + "latest.json", + await readFile(localPath), + promotionToken ? { "cam-promotion-token": promotionToken } : {}, + ); + const stored = this.objects.get("latest.json"); + return { etag: stored.etag, size: stored.body.length }; + } + async deleteLatestConditional(expectedEtag) { const current = this.objects.get("latest.json"); if (!current || current.etag !== expectedEtag) { @@ -218,6 +234,22 @@ class MemoryBackend { } } +class ConditionIgnoringMemoryBackend extends MemoryBackend { + async putLatestConditional(localPath, _expectedEtag, promotionToken = "") { + // Model the real IHEP behavior: the request accepts If-Match but overwrites + // regardless. Production promotion must therefore never call this method. + this.latestPutAttempts += 1; + this.conditionalLatestPutAttempts += 1; + this.set( + "latest.json", + await readFile(localPath), + promotionToken ? { "cam-promotion-token": promotionToken } : {}, + ); + const stored = this.objects.get("latest.json"); + return { etag: stored.etag, size: stored.body.length }; + } +} + function encodedUpdaterSignature(privateKey, keyId, artifact) { const digest = createHash("blake2b512").update(artifact).digest(); const signature = cryptoSign(null, digest, privateKey); @@ -478,7 +510,7 @@ describe("public mirror route verification", () => { }); }); -describe("AWS conditional writes", () => { +describe("AWS latest writes", () => { function objectStore() { return new AwsObjectStore({ name: "r2", @@ -536,6 +568,30 @@ describe("AWS conditional writes", () => { backend.putLatestConditional(candidatePath, '"stale-etag"', "promotion-token"), ).rejects.toBeInstanceOf(ConditionalWriteError); }); + + it("omits conditional headers for the serialized IHEP follower write", async () => { + const root = await tempRoot("unconditional-follower-put"); + const candidatePath = await writeManifest(root, "candidate.json", manifest("1.2.3")); + const backend = objectStore(); + let call; + backend.aws = async (args, options) => { + call = { args, options }; + return { code: 0, stderr: "", stdout: '{"ETag":"\\"follower-etag\\""}' }; + }; + + const result = await backend.putLatestUnconditional(candidatePath, "promotion-token"); + + expect(result.etag).toBe('"follower-etag"'); + expect(call.options.env).toEqual({ AWS_MAX_ATTEMPTS: "1" }); + expect(call.args).toEqual( + expect.arrayContaining([ + "--metadata", + "cam-promotion-token=promotion-token", + ]), + ); + expect(call.args).not.toContain("--if-match"); + expect(call.args).not.toContain("--if-none-match"); + }); }); describe("backend candidate verification", () => { @@ -772,6 +828,101 @@ describe("monotonic mirror promotion", () => { ]); }); + it("never lets an old rerun fill a lagging IHEP follower behind newer R2", async () => { + const root = await tempRoot("old-rerun-lagging-follower"); + const candidate = manifest("1.9.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const r2 = new MemoryBackend("r2", { + "latest.json": `${JSON.stringify(manifest("2.0.0"))}\n`, + }); + const ihep = new ConditionIgnoringMemoryBackend("ihep", { + "latest.json": `${JSON.stringify(manifest("1.0.0"))}\n`, + }); + const backends = [r2, ihep]; + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + summary: summaryFor(backends), + workDir: join(root, "transaction"), + }), + ).rejects.toBeInstanceOf(DowngradeBlockedError); + + expect(r2.latestPutAttempts).toBe(0); + expect(ihep.latestPutAttempts).toBe(0); + expect(JSON.parse(r2.body("latest.json")).version).toBe("2.0.0"); + expect(JSON.parse(ihep.body("latest.json")).version).toBe("1.0.0"); + }); + + it("uses IHEP only as an unconditional follower even when it ignores conditions", async () => { + const root = await tempRoot("ihep-ignores-conditions"); + const candidate = manifest("1.1.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = `${JSON.stringify(manifest("1.0.0"))}\n`; + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new ConditionIgnoringMemoryBackend("ihep", { "latest.json": initial }); + const backends = [r2, ihep]; + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + promotionToken: "single-writer", + summary: summaryFor(backends), + workDir: join(root, "transaction"), + }), + ).resolves.toEqual(expect.objectContaining({ outcome: "promoted" })); + + expect(r2.conditionalLatestPutAttempts).toBe(1); + expect(ihep.conditionalLatestPutAttempts).toBe(0); + expect(ihep.unconditionalLatestPutAttempts).toBe(1); + expect(JSON.parse(ihep.body("latest.json")).version).toBe("1.1.0"); + }); + + it("does not let an R2 CAS loser touch condition-ignoring IHEP", async () => { + const root = await tempRoot("r2-cas-loser"); + const candidate = manifest("1.1.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = `${JSON.stringify(manifest("1.0.0"))}\n`; + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new ConditionIgnoringMemoryBackend("ihep", { "latest.json": initial }); + const realPut = r2.putLatestConditional.bind(r2); + let raced = false; + r2.putLatestConditional = async (...args) => { + if (!raced) { + raced = true; + r2.set( + "latest.json", + `${JSON.stringify(manifest("1.2.0"))}\n`, + { "cam-promotion-token": "newer-winner" }, + ); + } + return await realPut(...args); + }; + const backends = [r2, ihep]; + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + summary: summaryFor(backends), + workDir: join(root, "transaction"), + }), + ).rejects.toBeInstanceOf(ConditionalWriteError); + + expect(ihep.conditionalLatestPutAttempts).toBe(0); + expect(ihep.unconditionalLatestPutAttempts).toBe(0); + expect(JSON.parse(r2.body("latest.json")).version).toBe("1.2.0"); + expect(JSON.parse(ihep.body("latest.json")).version).toBe("1.0.0"); + }); + it("treats a same-version rerun as a no-write idempotent success", async () => { const root = await tempRoot("same-version"); const current = manifest("2.0.0"); @@ -829,7 +980,7 @@ describe("monotonic mirror promotion", () => { expect(JSON.parse(ihep.body("latest.json")).version).toBe("2.1.0"); }); - it("uses conditional writes so a concurrent newer release wins without mixed latest pointers", async () => { + it("uses R2 CAS so a concurrent newer release wins without mixed latest pointers", async () => { const root = await tempRoot("concurrent"); const initial = `${JSON.stringify(manifest("1.0.0"))}\n`; const backends = [ @@ -882,7 +1033,288 @@ describe("monotonic mirror promotion", () => { ]); }); - it("heals a hard-terminated partial promotion without rewriting the committed backend", async () => { + it("repairs IHEP when an older writer is superseded after its pre-write ownership check", async () => { + const root = await tempRoot("cross-version-follower-race"); + const initial = `${JSON.stringify(manifest("1.0.0"))}\n`; + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new ConditionIgnoringMemoryBackend("ihep", { "latest.json": initial }); + const backends = [r2, ihep]; + const older = manifest("2.0.0"); + const newer = manifest("3.0.0"); + const olderPath = await writeManifest(root, "older.json", older); + const newerPath = await writeManifest(root, "newer.json", newer); + const olderSummary = summaryFor(backends); + let releaseOlderFollower; + let olderAtFollower; + const holdOlderFollower = new Promise((resolvePromise) => { + releaseOlderFollower = resolvePromise; + }); + const olderFollowerReady = new Promise((resolvePromise) => { + olderAtFollower = resolvePromise; + }); + + const olderRun = promoteCandidateTransaction({ + backends, + candidateManifest: older, + candidatePath: olderPath, + override: overrideOff(), + promotionToken: "older-writer", + summary: olderSummary, + workDir: join(root, "older-transaction"), + hooks: { + beforeWrite: async (state) => { + if (state.backend.name === "ihep") { + olderAtFollower(); + await holdOlderFollower; + } + }, + }, + }); + await olderFollowerReady; + + await promoteCandidateTransaction({ + backends, + candidateManifest: newer, + candidatePath: newerPath, + override: overrideOff(), + promotionToken: "newer-writer", + summary: summaryFor(backends), + workDir: join(root, "newer-transaction"), + }); + releaseOlderFollower(); + + await expect(olderRun).rejects.toBeInstanceOf(ConditionalWriteError); + expect(JSON.parse(r2.body("latest.json")).version).toBe("3.0.0"); + expect(JSON.parse(ihep.body("latest.json")).version).toBe("3.0.0"); + expect(ihep.conditionalLatestPutAttempts).toBe(0); + expect(ihep.unconditionalLatestPutAttempts).toBe(3); + expect(olderSummary.backends.find((backend) => backend.name === "ihep").supersession).toBe( + "repaired-to-r2", + ); + }); + + it("keeps B's R2 v3 CAS when A repairs B's changed follower before B resumes", async () => { + const root = await tempRoot("post-cas-follower-repaired-by-older-writer"); + const initial = `${JSON.stringify(manifest("1.0.0"))}\n`; + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new ConditionIgnoringMemoryBackend("ihep", { "latest.json": initial }); + const backends = [r2, ihep]; + const aManifest = manifest("2.0.0"); + const bManifest = manifest("3.0.0"); + const aPath = await writeManifest(root, "a-v2.json", aManifest); + const bPath = await writeManifest(root, "b-v3.json", bManifest); + const bSummary = summaryFor(backends); + + let releaseA; + let signalAReady; + const holdA = new Promise((resolvePromise) => { + releaseA = resolvePromise; + }); + const aReady = new Promise((resolvePromise) => { + signalAReady = resolvePromise; + }); + const aRun = promoteCandidateTransaction({ + backends, + candidateManifest: aManifest, + candidatePath: aPath, + override: overrideOff(), + promotionToken: "writer-a-v2", + summary: summaryFor(backends), + workDir: join(root, "a-transaction"), + hooks: { + beforeWrite: async (state) => { + if (state.backend.name === "ihep") { + signalAReady(); + await holdA; + } + }, + }, + }); + await aReady; + + let releaseB; + let signalBCasCommitted; + const holdB = new Promise((resolvePromise) => { + releaseB = resolvePromise; + }); + const bCasCommitted = new Promise((resolvePromise) => { + signalBCasCommitted = resolvePromise; + }); + const bRun = promoteCandidateTransaction({ + backends, + candidateManifest: bManifest, + candidatePath: bPath, + override: overrideOff(), + promotionToken: "writer-b-v3", + summary: bSummary, + workDir: join(root, "b-transaction"), + hooks: { + afterWrite: async (state) => { + if (state.backend.name === "r2") { + signalBCasCommitted(); + await holdB; + } + }, + }, + }); + await bCasCommitted; + + releaseA(); + await expect(aRun).rejects.toBeInstanceOf(ConditionalWriteError); + expect(JSON.parse(ihep.body("latest.json")).version).toBe("3.0.0"); + + releaseB(); + await expect(bRun).resolves.toEqual(expect.objectContaining({ outcome: "promoted" })); + + expect(JSON.parse(r2.body("latest.json")).version).toBe("3.0.0"); + expect(JSON.parse(ihep.body("latest.json")).version).toBe("3.0.0"); + expect(r2.conditionalLatestPutAttempts).toBe(2); + expect(ihep.conditionalLatestPutAttempts).toBe(0); + expect(ihep.unconditionalLatestPutAttempts).toBe(2); + expect(bSummary.backends.find((backend) => backend.name === "r2").rollback).toBe( + "not-needed", + ); + expect(bSummary.backends.find((backend) => backend.name === "ihep").promotion).toBe( + "already-current-after-authority-cas", + ); + }); + + it("fails closed on a higher post-CAS follower without rolling R2 back", async () => { + const root = await tempRoot("post-cas-safe-successor"); + const candidate = manifest("2.0.0"); + const successor = manifest("3.0.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = `${JSON.stringify(manifest("1.0.0"))}\n`; + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new ConditionIgnoringMemoryBackend("ihep", { "latest.json": initial }); + const backends = [r2, ihep]; + const summary = summaryFor(backends); + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + promotionToken: "candidate-v2", + summary, + workDir: join(root, "transaction"), + hooks: { + afterWrite: (state) => { + if (state.backend.name === "r2") { + ihep.set( + "latest.json", + `${JSON.stringify(successor)}\n`, + { "cam-promotion-token": "successor-v3" }, + ); + } + }, + }, + }), + ).rejects.toBeInstanceOf(ConditionalWriteError); + + expect(JSON.parse(r2.body("latest.json")).version).toBe("2.0.0"); + expect(JSON.parse(ihep.body("latest.json")).version).toBe("3.0.0"); + expect(r2.conditionalLatestPutAttempts).toBe(1); + expect(ihep.unconditionalLatestPutAttempts).toBe(0); + expect(summary.authorityPreserved).toBe(true); + expect(summary.backends.find((backend) => backend.name === "r2").rollback).toBe( + "preserved-authoritative-cas", + ); + expect(summary.backends.find((backend) => backend.name === "ihep").promotion).toBe( + "conflict-after-authority-cas", + ); + }); + + it("fails closed on an unknown post-CAS identity without rolling R2 back", async () => { + const root = await tempRoot("post-cas-unknown-identity"); + const candidate = manifest("2.0.0"); + const unknown = structuredClone(candidate); + unknown.platforms["windows-x86_64"].signature = "unknown-signature"; + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = `${JSON.stringify(manifest("1.0.0"))}\n`; + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new ConditionIgnoringMemoryBackend("ihep", { "latest.json": initial }); + const backends = [r2, ihep]; + const summary = summaryFor(backends); + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + promotionToken: "candidate-v2", + summary, + workDir: join(root, "transaction"), + hooks: { + afterWrite: (state) => { + if (state.backend.name === "r2") { + ihep.set( + "latest.json", + `${JSON.stringify(unknown)}\n`, + { "cam-promotion-token": "unknown-v2" }, + ); + } + }, + }, + }), + ).rejects.toBeInstanceOf(ConditionalWriteError); + + expect(JSON.parse(r2.body("latest.json")).version).toBe("2.0.0"); + expect(JSON.parse(ihep.body("latest.json")).platforms["windows-x86_64"].signature).toBe( + "unknown-signature", + ); + expect(r2.conditionalLatestPutAttempts).toBe(1); + expect(ihep.unconditionalLatestPutAttempts).toBe(0); + expect(summary.authorityPreserved).toBe(true); + }); + + it("fails closed when the final follower becomes higher after this run writes it", async () => { + const root = await tempRoot("final-follower-safe-successor"); + const candidate = manifest("2.0.0"); + const successor = manifest("3.0.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = `${JSON.stringify(manifest("1.0.0"))}\n`; + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new ConditionIgnoringMemoryBackend("ihep", { "latest.json": initial }); + const backends = [r2, ihep]; + const summary = summaryFor(backends); + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + promotionToken: "candidate-v2", + summary, + workDir: join(root, "transaction"), + hooks: { + afterWrite: (state) => { + if (state.backend.name === "ihep") { + ihep.set( + "latest.json", + `${JSON.stringify(successor)}\n`, + { "cam-promotion-token": "successor-v3" }, + ); + } + }, + }, + }), + ).rejects.toBeInstanceOf(ConditionalWriteError); + + expect(JSON.parse(r2.body("latest.json")).version).toBe("2.0.0"); + expect(JSON.parse(ihep.body("latest.json")).version).toBe("3.0.0"); + expect(r2.conditionalLatestPutAttempts).toBe(1); + expect(ihep.unconditionalLatestPutAttempts).toBe(1); + expect(summary.authorityPreserved).toBe(true); + expect(summary.backends.find((backend) => backend.name === "r2").rollback).toBe( + "preserved-authoritative-cas", + ); + }); + + it("reclaims R2 CAS and heals a hard-terminated R2-only promotion", async () => { const root = await tempRoot("partial-terminated-transaction"); const current = manifest("1.0.0"); const candidate = manifest("1.1.0"); @@ -897,13 +1329,14 @@ describe("monotonic mirror promotion", () => { const r2AttemptsAfterTermination = r2.latestPutAttempts; const backends = [r2, ihep]; const failedResumeSummary = summaryFor(backends); - const realIhepPut = ihep.putLatestConditional.bind(ihep); + const realIhepPut = ihep.putLatestUnconditional.bind(ihep); let failFirstResume = true; - ihep.putLatestConditional = async (...args) => { + ihep.putLatestUnconditional = async (...args) => { if (failFirstResume) { failFirstResume = false; ihep.latestPutAttempts += 1; - throw new ConditionalWriteError("ihep: simulated conditional outage"); + ihep.unconditionalLatestPutAttempts += 1; + throw new Error("ihep: simulated follower outage"); } return await realIhepPut(...args); }; @@ -918,15 +1351,15 @@ describe("monotonic mirror promotion", () => { summary: failedResumeSummary, workDir: join(root, "failed-resume"), }), - ).rejects.toBeInstanceOf(ConditionalWriteError); + ).rejects.toThrow("simulated follower outage"); - expect(r2.latestPutAttempts).toBe(r2AttemptsAfterTermination); + expect(r2.latestPutAttempts).toBe(r2AttemptsAfterTermination + 2); expect(backends.map((backend) => JSON.parse(backend.body("latest.json")).version)).toEqual([ "1.1.0", "1.0.0", ]); expect(failedResumeSummary.rollback).toEqual( - expect.objectContaining({ attempted: false, complete: true }), + expect.objectContaining({ attempted: true, complete: true }), ); await expect( @@ -941,18 +1374,50 @@ describe("monotonic mirror promotion", () => { }), ).resolves.toEqual(expect.objectContaining({ outcome: "promoted" })); - expect(r2.latestPutAttempts).toBe(r2AttemptsAfterTermination); + expect(r2.latestPutAttempts).toBe(r2AttemptsAfterTermination + 3); expect(ihep.latestPutAttempts).toBe(2); expect(backends.map((backend) => JSON.parse(backend.body("latest.json")).version)).toEqual([ "1.1.0", "1.1.0", ]); expect(r2.objects.get("latest.json").metadata["cam-promotion-token"]).toBe( - "terminated-run", + "fresh-run", ); expect(ihep.objects.get("latest.json").metadata["cam-promotion-token"]).toBe("fresh-run"); }); + it("lets a newer rerun converge after a hard-terminated R2-only promotion", async () => { + const root = await tempRoot("newer-after-hard-termination"); + const previous = manifest("1.0.0"); + const interrupted = manifest("1.1.0"); + const newer = manifest("1.2.0"); + const interruptedPath = await writeManifest(root, "interrupted.json", interrupted); + const newerPath = await writeManifest(root, "newer.json", newer); + const initial = `${JSON.stringify(previous)}\n`; + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new ConditionIgnoringMemoryBackend("ihep", { "latest.json": initial }); + const before = await r2.head("latest.json"); + await r2.putLatestConditional(interruptedPath, before.etag, "terminated-run"); + const backends = [r2, ihep]; + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: newer, + candidatePath: newerPath, + override: overrideOff(), + promotionToken: "newer-rerun", + summary: summaryFor(backends), + workDir: join(root, "transaction"), + }), + ).resolves.toEqual(expect.objectContaining({ outcome: "promoted" })); + + expect(JSON.parse(r2.body("latest.json")).version).toBe("1.2.0"); + expect(JSON.parse(ihep.body("latest.json")).version).toBe("1.2.0"); + expect(ihep.conditionalLatestPutAttempts).toBe(0); + expect(ihep.unconditionalLatestPutAttempts).toBe(1); + }); + it("fails closed before writing when either backend has no baseline latest.json", async () => { const root = await tempRoot("unseeded-backend"); const candidate = manifest("1.1.0"); @@ -978,6 +1443,161 @@ describe("monotonic mirror promotion", () => { expect(ihep.body("latest.json")).toBeUndefined(); }); + it("CAS-rolls R2 back and preserves IHEP when the follower fails before writing", async () => { + const root = await tempRoot("follower-failure-before-write"); + const candidate = manifest("1.1.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = Buffer.from(`${JSON.stringify(manifest("1.0.0"))}\n`); + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new ConditionIgnoringMemoryBackend("ihep", { "latest.json": initial }); + ihep.failLatestPut = true; + const backends = [r2, ihep]; + const summary = summaryFor(backends); + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + promotionToken: "failed-follower", + summary, + workDir: join(root, "transaction"), + }), + ).rejects.toThrow("simulated write failure"); + + expect(r2.body("latest.json").equals(initial)).toBe(true); + expect(ihep.body("latest.json").equals(initial)).toBe(true); + expect(r2.conditionalLatestPutAttempts).toBe(2); + expect(ihep.conditionalLatestPutAttempts).toBe(0); + expect(ihep.unconditionalLatestPutAttempts).toBe(1); + expect(summary.backends.find((backend) => backend.name === "r2").rollback).toBe("restored"); + expect(summary.backends.find((backend) => backend.name === "ihep").rollback).toBe( + "preserved-previous", + ); + }); + + it("restores only its own IHEP follower write after a later local failure", async () => { + const root = await tempRoot("follower-failure-after-write"); + const candidate = manifest("1.1.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = Buffer.from(`${JSON.stringify(manifest("1.0.0"))}\n`); + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new ConditionIgnoringMemoryBackend("ihep", { "latest.json": initial }); + const backends = [r2, ihep]; + const summary = summaryFor(backends); + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + promotionToken: "restore-follower", + summary, + workDir: join(root, "transaction"), + hooks: { + afterWrite: (state) => { + if (state.backend.name === "ihep") throw new Error("simulated post-write failure"); + }, + }, + }), + ).rejects.toThrow("simulated post-write failure"); + + expect(r2.body("latest.json").equals(initial)).toBe(true); + expect(ihep.body("latest.json").equals(initial)).toBe(true); + expect(ihep.conditionalLatestPutAttempts).toBe(0); + expect(ihep.unconditionalLatestPutAttempts).toBe(2); + expect(summary.backends.find((backend) => backend.name === "ihep").rollback).toBe( + "restored-unconditionally", + ); + }); + + it("does not roll R2 back when IHEP fails after a newer authority takes ownership", async () => { + const root = await tempRoot("follower-failure-authority-lost"); + const candidate = manifest("1.1.0"); + const successor = manifest("1.2.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = `${JSON.stringify(manifest("1.0.0"))}\n`; + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new ConditionIgnoringMemoryBackend("ihep", { "latest.json": initial }); + ihep.putLatestUnconditional = async () => { + ihep.latestPutAttempts += 1; + ihep.unconditionalLatestPutAttempts += 1; + r2.set( + "latest.json", + `${JSON.stringify(successor)}\n`, + { "cam-promotion-token": "newer-authority" }, + ); + throw new Error("simulated IHEP outage after R2 supersession"); + }; + const backends = [r2, ihep]; + const summary = summaryFor(backends); + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + promotionToken: "older-authority", + summary, + workDir: join(root, "transaction"), + }), + ).rejects.toThrow("rollback incomplete"); + + expect(JSON.parse(r2.body("latest.json")).version).toBe("1.2.0"); + expect(JSON.parse(ihep.body("latest.json")).version).toBe("1.0.0"); + expect(r2.conditionalLatestPutAttempts).toBe(1); + expect(ihep.conditionalLatestPutAttempts).toBe(0); + expect(summary.backends.find((backend) => backend.name === "r2").rollback).toBe( + "skipped-concurrent-change", + ); + }); + + it("preserves a concurrent newer IHEP value while rolling back owned R2", async () => { + const root = await tempRoot("preserve-concurrent-follower"); + const candidate = manifest("1.1.0"); + const followerSuccessor = manifest("1.2.0"); + const candidatePath = await writeManifest(root, "candidate.json", candidate); + const initial = `${JSON.stringify(manifest("1.0.0"))}\n`; + const r2 = new MemoryBackend("r2", { "latest.json": initial }); + const ihep = new ConditionIgnoringMemoryBackend("ihep", { "latest.json": initial }); + const backends = [r2, ihep]; + const summary = summaryFor(backends); + + await expect( + promoteCandidateTransaction({ + backends, + candidateManifest: candidate, + candidatePath, + override: overrideOff(), + promotionToken: "owned-follower", + summary, + workDir: join(root, "transaction"), + hooks: { + afterWrite: (state) => { + if (state.backend.name === "ihep") { + ihep.set( + "latest.json", + `${JSON.stringify(followerSuccessor)}\n`, + { "cam-promotion-token": "newer-follower" }, + ); + throw new Error("simulated failure after concurrent IHEP advance"); + } + }, + }, + }), + ).rejects.toThrow("simulated failure after concurrent IHEP advance"); + + expect(JSON.parse(r2.body("latest.json")).version).toBe("1.0.0"); + expect(JSON.parse(ihep.body("latest.json")).version).toBe("1.2.0"); + expect(summary.backends.find((backend) => backend.name === "r2").rollback).toBe("restored"); + expect(summary.backends.find((backend) => backend.name === "ihep").rollback).toBe( + "preserved-concurrent-change", + ); + }); + it("never rolls back an ETag that a concurrent writer owns", async () => { const root = await tempRoot("etag-ownership"); const current = manifest("1.0.0"); @@ -1013,7 +1633,7 @@ describe("monotonic mirror promotion", () => { summary, workDir: join(root, "transaction"), }), - ).rejects.toBeInstanceOf(ConditionalWriteError); + ).rejects.toThrow("rollback incomplete"); expect(JSON.parse(r2.body("latest.json")).version).toBe("1.2.0"); expect(JSON.parse(ihep.body("latest.json")).version).toBe("1.0.0"); diff --git a/scripts/release-workflow.test.mjs b/scripts/release-workflow.test.mjs index 7f919f6..d223a6f 100644 --- a/scripts/release-workflow.test.mjs +++ b/scripts/release-workflow.test.mjs @@ -21,6 +21,28 @@ describe("release workflow recovery invariants", () => { ); }); + it("enforces the single-writer boundary for the unconditional IHEP follower", () => { + expect(workflow).toContain( + "correctness boundary for the unconditional IHEP follower", + ); + expect(releaseJob).toContain("Enforce legacy mirror credential revocation"); + expect(releaseJob).toContain("MANAGER_R2_PROMOTION_ACCESS_KEY_ID"); + expect(releaseJob).toContain("MANAGER_IHEP_S3_PROMOTION_ACCESS_KEY_ID"); + expect(releaseJob).toContain( + "MANAGER_IHEP_S3_ENDPOINT: ${{ vars.MANAGER_IHEP_S3_ENDPOINT }}", + ); + expect(releaseJob).toContain( + "MANAGER_IHEP_S3_BUCKET: ${{ vars.MANAGER_IHEP_S3_BUCKET }}", + ); + expect(releaseJob).not.toContain( + "MANAGER_IHEP_S3_ENDPOINT: ${{ secrets.MANAGER_IHEP_S3_ENDPOINT }}", + ); + expect(mirrorRelease).toContain( + "followerCommit = await ihep.putLatestUnconditional(candidatePath, promotionToken)", + ); + expect(mirrorRelease).not.toContain("await ihep.putLatestConditional("); + }); + it("refreshes immutable Release state inside the release job on failed-job reruns", () => { expect(releaseJob).toContain("id: live_release"); expect(releaseJob).toContain( diff --git a/scripts/write-release-summary.mjs b/scripts/write-release-summary.mjs index 22f11bf..0c14802 100644 --- a/scripts/write-release-summary.mjs +++ b/scripts/write-release-summary.mjs @@ -114,8 +114,8 @@ if (mirrorStage || mirrorVerification || mirrorPromotion) { ); } rows.push(""); - rows.push("| Backend | Candidate verification | Previous | Decision | Promotion | Rollback | Final | Error |"); - rows.push("| --- | --- | --- | --- | --- | --- | --- | --- |"); + rows.push("| Backend | Candidate verification | Previous | Decision | Promotion | Supersession | Rollback | Final | Error |"); + rows.push("| --- | --- | --- | --- | --- | --- | --- | --- | --- |"); const backendRows = mirrorPromotion?.backends || mirrorVerification?.backends || mirrorStage?.backends || []; for (const backend of backendRows) { @@ -123,6 +123,8 @@ if (mirrorStage || mirrorVerification || mirrorPromotion) { `| ${cell(backend.name)} | ${cell(backend.candidateVerification || backend.status)} | ${cell( backend.currentVersion || "absent", )} | ${cell(backend.decision)} | ${cell(backend.promotion)} | ${cell( + backend.supersession, + )} | ${cell( backend.rollback, )} | ${cell(backend.finalVersion)} | ${cell( backend.error || backend.rollbackError, @@ -146,7 +148,11 @@ if (mirrorStage || mirrorVerification || mirrorPromotion) { } else { rows.push("Emergency downgrade override: not requested."); } - if (mirrorPromotion?.rollback?.complete === false) { + if (mirrorPromotion?.authorityPreserved) { + rows.push( + "**Authority preserved:** R2 kept its verified CAS commit, but IHEP did not expose the exact canonical candidate identity. The follower was not overwritten and this run failed closed.", + ); + } else if (mirrorPromotion?.rollback?.complete === false) { rows.push( "**Manual intervention required:** promotion rollback was incomplete; inspect the per-backend errors before another release.", ); From 47ee1e7bac801b80a3edde0db542952408cef88e Mon Sep 17 00:00:00 2001 From: wangnov <48670012+Wangnov@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:36:46 +0800 Subject: [PATCH 4/8] fix(release): validate immutable publication provenance --- .github/workflows/release.yml | 213 ++++++++++++++++++++++++------ scripts/check-release-version.mjs | 177 +++++++++++++++++++++++++ scripts/release-workflow.test.mjs | 155 +++++++++++++++++++--- scripts/write-release-summary.mjs | 3 +- 4 files changed, 488 insertions(+), 60 deletions(-) create mode 100644 scripts/check-release-version.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 295815f..1b46d7e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,8 +49,7 @@ on: inputs: target_tag: description: "Existing vX.Y.Z release to republish through the current protected workflow" - required: false - default: "" + required: true type: string allow_mirror_downgrade: description: "Emergency only: allow this tag to replace a newer mirror latest.json" @@ -79,8 +78,55 @@ concurrency: queue: max jobs: + # A manual dispatch may execute only from the protected default branch and + # must name an existing target_tag. This also rejects an empty-input dispatch + # from an attacker-controlled branch named like a release tag. + # Keep the rejecting path free of checkout, environments, permissions, and + # secrets so an untrusted ref cannot reach any credential-bearing job. Compare + # the complete ref, not ref_name: a tag named like the default branch must fail. + reject_untrusted_target_dispatch: + if: ${{ github.event_name == 'workflow_dispatch' && github.ref != format('refs/heads/{0}', github.event.repository.default_branch) }} + runs-on: ubuntu-latest + permissions: {} + steps: + - name: Reject manual dispatch outside the default branch + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + DISPATCH_REF: ${{ github.ref }} + run: | + echo "::error::target_tag must be dispatched from refs/heads/$DEFAULT_BRANCH, not $DISPATCH_REF" + exit 1 + + preflight: + if: ${{ github.event_name != 'workflow_dispatch' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + runs-on: ubuntu-latest + permissions: {} + steps: + - name: Validate release invocation without secrets + shell: bash + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + DISPATCH_REF: ${{ github.ref }} + EVENT_NAME: ${{ github.event_name }} + REQUESTED_TARGET_TAG: ${{ inputs.target_tag || '' }} + run: | + set -euo pipefail + if [[ "$EVENT_NAME" == "workflow_dispatch" && "$DISPATCH_REF" != "refs/heads/$DEFAULT_BRANCH" ]]; then + echo "::error::workflow_dispatch must run from refs/heads/$DEFAULT_BRANCH, not $DISPATCH_REF" + exit 1 + fi + if [[ "$EVENT_NAME" == "workflow_dispatch" && -z "$REQUESTED_TARGET_TAG" ]]; then + echo "::error::workflow_dispatch requires a non-empty target_tag" + exit 1 + fi + if [[ ! "$RELEASE_TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then + echo "::error::release tag must be a semantic vX.Y.Z tag: $RELEASE_TAG" + exit 1 + fi + prepare: - if: ${{ startsWith(inputs.target_tag || github.ref_name, 'v') }} + needs: preflight + if: ${{ needs.preflight.result == 'success' }} runs-on: ubuntu-latest environment: release permissions: @@ -97,6 +143,26 @@ jobs: with: node-version: 20 + # Verify the release source rather than the workflow source. For a + # target_tag recovery the workflow itself comes from the protected default + # branch, while the six application version declarations come from the tag. + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ env.RELEASE_TAG }} + path: release-source + persist-credentials: false + + - name: Match release tag to every application version declaration + shell: bash + run: | + set -euo pipefail + node scripts/check-release-version.mjs source "$RELEASE_TAG" release-source + release_source_sha="$(git -C release-source rev-parse HEAD)" + if [[ "$GITHUB_EVENT_NAME" == "push" && "$release_source_sha" != "$GITHUB_SHA" ]]; then + echo "::error::release tag $RELEASE_TAG moved away from triggering commit $GITHUB_SHA" + exit 1 + fi + - name: Require GitHub Immutable Releases env: GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }} @@ -106,16 +172,10 @@ jobs: id: lookup shell: bash env: - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - DISPATCH_REF_NAME: ${{ github.ref_name }} GH_TOKEN: ${{ github.token }} REQUESTED_TARGET_TAG: ${{ inputs.target_tag || '' }} run: | set -euo pipefail - if [[ -n "$REQUESTED_TARGET_TAG" && "$DISPATCH_REF_NAME" != "$DEFAULT_BRANCH" ]]; then - echo "::error::target_tag must be dispatched from the default branch ($DEFAULT_BRANCH), not $DISPATCH_REF_NAME" - exit 1 - fi if [[ ! "$RELEASE_TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then echo "::error::release tag must be a semantic vX.Y.Z tag: $RELEASE_TAG" exit 1 @@ -492,6 +552,27 @@ jobs: with: node-version: 20 + # Re-run this gate inside every release-job attempt. GitHub's "Re-run + # failed jobs" does not re-run prepare, and no immutable publication may + # rely only on a previous attempt's version decision. + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ env.RELEASE_TAG }} + path: release-source + persist-credentials: false + + - name: Re-check release tag against application versions + shell: bash + run: | + set -euo pipefail + node scripts/check-release-version.mjs source "$RELEASE_TAG" release-source + release_source_sha="$(git -C release-source rev-parse HEAD)" + if [[ "$GITHUB_EVENT_NAME" == "push" && "$release_source_sha" != "$GITHUB_SHA" ]]; then + echo "::error::release tag $RELEASE_TAG moved away from triggering commit $GITHUB_SHA" + exit 1 + fi + echo "RELEASE_SOURCE_SHA=$release_source_sha" >> "$GITHUB_ENV" + # Re-check on every release-job attempt. `prepare` is not rerun by # "Re-run failed jobs", and publishing a resumed draft while the setting is # disabled would create a mutable canonical source. @@ -640,6 +721,7 @@ jobs: run: | echo "::group::[build] validate final publishable artifacts" set -euo pipefail + node scripts/check-release-version.mjs artifacts "$RELEASE_TAG" dist ls -la dist missing=0 require() { @@ -686,12 +768,10 @@ jobs: if [[ "${{ steps.release_source.outputs.existing }}" == "true" ]]; then cp "$RUNNER_TEMP/published-latest.json" latest.json fi - if [[ "$RELEASE_TAG" != *-* ]]; then - node scripts/validate-release-manifest.mjs \ - "$RELEASE_TAG" \ - latest.json \ - "$RUNNER_TEMP/artifact-derived-latest.json" - fi + node scripts/validate-release-manifest.mjs \ + "$RELEASE_TAG" \ + latest.json \ + "$RUNNER_TEMP/artifact-derived-latest.json" # A wrong private/public updater-key pairing cannot be repaired after an # immutable GitHub Release is published. Verify the exact local payloads @@ -704,12 +784,68 @@ jobs: dist \ "$RELEASE_TAURI_CONFIG" + # Create provenance only for bytes produced by this release run, before a + # draft can become immutable. A later failed-job rerun must never mint a + # new statement that falsely attributes already-published bytes to the + # rerun's OIDC identity. + - name: Attest fresh build provenance + id: attest_fresh + if: ${{ steps.release_source.outputs.existing != 'true' }} + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + with: + subject-path: | + dist/* + latest.json + + # Existing immutable releases are accepted only when their downloaded, + # API-digest-pinned bytes already carry provenance from this repository's + # release workflow and exact tag ref. This is verification, not a new OIDC + # statement; missing historical provenance fails closed. + - name: Verify existing immutable release provenance + id: verify_existing_provenance + if: ${{ steps.release_source.outputs.existing == 'true' }} + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + signer_workflow="$GITHUB_REPOSITORY/.github/workflows/release.yml" + for file in dist/* latest.json; do + gh attestation verify "$file" \ + --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$signer_workflow" \ + --source-ref "refs/tags/$RELEASE_TAG" \ + --source-digest "$RELEASE_SOURCE_SHA" \ + --deny-self-hosted-runners >/dev/null + echo "Verified existing provenance: $(basename "$file")" + done + + - name: Resolve provenance gate + id: provenance + shell: bash + env: + FRESH_ATTESTATION_OUTCOME: ${{ steps.attest_fresh.outcome }} + EXISTING_PROVENANCE_OUTCOME: ${{ steps.verify_existing_provenance.outcome }} + RELEASE_REUSED: ${{ steps.release_source.outputs.existing }} + run: | + set -euo pipefail + if [[ "$RELEASE_REUSED" == "true" ]]; then + [[ "$FRESH_ATTESTATION_OUTCOME" == "skipped" ]] + [[ "$EXISTING_PROVENANCE_OUTCOME" == "success" ]] + echo "mode=verified-existing-immutable-release" >> "$GITHUB_OUTPUT" + else + [[ "$FRESH_ATTESTATION_OUTCOME" == "success" ]] + [[ "$EXISTING_PROVENANCE_OUTCOME" == "skipped" ]] + echo "mode=fresh-build-attested-before-publication" >> "$GITHUB_OUTPUT" + fi + echo "ready=true" >> "$GITHUB_OUTPUT" + # Stable releases stage immutable versioned assets before GitHub Release # publication, verify them separately, then promote latest.json only after # the GitHub Release succeeds. Pre-releases still skip the mirror entirely. - name: Stage CDN mirror candidate (R2 + IHEP S3) id: mirror_stage - if: ${{ !contains(env.RELEASE_TAG, '-') }} + if: ${{ steps.provenance.outputs.ready == 'true' && !contains(env.RELEASE_TAG, '-') }} env: MIRROR_PHASE: stage MIRROR_REQUIRED_BACKENDS: r2,ihep @@ -742,7 +878,7 @@ jobs: # branches must pass before GitHub publication becomes immutable. - name: Verify staged CDN mirror before immutable publication id: mirror_verify - if: ${{ !contains(env.RELEASE_TAG, '-') }} + if: ${{ steps.provenance.outputs.ready == 'true' && !contains(env.RELEASE_TAG, '-') }} env: MIRROR_PHASE: verify MIRROR_REQUIRED_BACKENDS: r2,ihep @@ -829,7 +965,7 @@ jobs: # releases are enabled: a published release can no longer accept assets. - name: Upload GitHub Release draft id: upload_release_draft - if: ${{ steps.release_source.outputs.existing != 'true' }} + if: ${{ steps.provenance.outputs.ready == 'true' && steps.release_source.outputs.existing != 'true' }} uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3 with: tag_name: ${{ env.RELEASE_TAG }} @@ -851,7 +987,7 @@ jobs: # after all uploads succeeded while retaining its stable/prerelease metadata. - name: Publish GitHub Release id: publish_release - if: ${{ steps.release_source.outputs.existing != 'true' }} + if: ${{ steps.provenance.outputs.ready == 'true' && steps.release_source.outputs.existing != 'true' }} uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3 with: tag_name: ${{ env.RELEASE_TAG }} @@ -884,27 +1020,25 @@ jobs: exit 1 fi cat "$verification_log" - - # Attest immediately after GitHub confirms the canonical immutable bytes, - # before mirror promotion can fail the job. Keep the subjects limited to - # updater payloads and latest.json: on a failed-job rerun those are fetched - # from the immutable Release and checked against GitHub's canonical SHA-256 - # digests. SHA256SUMS and SBOMs are intentionally excluded because they are - # regenerated locally and are not necessarily the bytes already published. - # A failed attestation must fail the job so the release stays retryable and - # neither mirror promotion nor winget dispatch can proceed without proof. - - name: Attest build provenance - id: attest - if: ${{ steps.publish_release.outcome == 'success' || steps.release_source.outputs.existing == 'true' }} - uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 - with: - subject-path: | - dist/* - latest.json + release_asset_digests="$(sed -n 's/^release_asset_digests=//p' "$verification_output")" + jq -e 'type == "object" and length > 0' <<<"$release_asset_digests" >/dev/null || { + echo "::error::published immutable digest evidence is missing" + exit 1 + } + for file in dist/* latest.json; do + name="$(basename "$file")" + expected_digest="$(jq -er --arg name "$name" '.[$name]' <<<"$release_asset_digests")" + actual_digest="sha256:$(sha256sum "$file" | cut -d' ' -f1)" + if [[ "$actual_digest" != "$expected_digest" ]]; then + echo "::error::published immutable digest does not match attested local bytes: $name" + exit 1 + fi + echo "Verified published attested digest: $name" + done - name: Promote CDN mirror latest (R2 + IHEP S3) id: mirror_promote - if: ${{ success() && steps.attest.outcome == 'success' && !contains(env.RELEASE_TAG, '-') }} + if: ${{ success() && steps.provenance.outputs.ready == 'true' && !contains(env.RELEASE_TAG, '-') }} env: MIRROR_PHASE: promote MIRROR_REQUIRED_BACKENDS: r2,ihep @@ -934,7 +1068,7 @@ jobs: # so a re-run is harmless. - name: Trigger winget submission id: winget - if: ${{ success() && steps.attest.outcome == 'success' && !contains(env.RELEASE_TAG, '-') }} + if: ${{ success() && steps.provenance.outputs.ready == 'true' && !contains(env.RELEASE_TAG, '-') }} env: GH_TOKEN: ${{ github.token }} run: | @@ -954,7 +1088,8 @@ jobs: MIRROR_PROMOTE_OUTCOME: ${{ steps.mirror_promote.outcome }} WINGET_OUTCOME: ${{ steps.winget.outputs.status || steps.winget.outcome }} SBOM_OUTCOME: ${{ steps.sbom.outcome }} - ATTESTATION_OUTCOME: ${{ steps.attest.outcome }} + ATTESTATION_OUTCOME: ${{ steps.attest_fresh.outcome }} + PROVENANCE_MODE: ${{ steps.provenance.outputs.mode }} run: node scripts/write-release-summary.mjs - name: Upload release audit records diff --git a/scripts/check-release-version.mjs b/scripts/check-release-version.mjs new file mode 100644 index 0000000..30a8441 --- /dev/null +++ b/scripts/check-release-version.mjs @@ -0,0 +1,177 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { requiredReleaseAssetNames } from "./check-release-reuse.mjs"; + +const RELEASE_TAG_PATTERN = + /^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/; + +export function releaseVersionFromTag(releaseTag) { + const tag = String(releaseTag); + if (!RELEASE_TAG_PATTERN.test(tag)) { + throw new Error(`release tag must be a semantic vX.Y.Z tag: ${releaseTag}`); + } + return tag.slice(1); +} + +function readJson(path, label) { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`cannot read ${label}: ${detail}`); + } +} + +function quotedTomlValue(block, key, label) { + const matches = [...block.matchAll(new RegExp(`^${key}\\s*=\\s*"([^"]+)"\\s*(?:#.*)?$`, "gm"))]; + if (matches.length !== 1) { + throw new Error(`${label} must contain exactly one quoted ${key}`); + } + return matches[0][1]; +} + +function cargoPackageBlock(cargoToml, label) { + const match = /^\[package\]\s*$/m.exec(cargoToml); + if (!match) { + throw new Error(`${label} has no [package] table`); + } + const start = match.index + match[0].length; + const rest = cargoToml.slice(start); + const nextTable = /^\s*\[/m.exec(rest); + return nextTable ? rest.slice(0, nextTable.index) : rest; +} + +function cargoLockPackageVersion(cargoLock, packageName) { + const blocks = cargoLock.split(/^\[\[package\]\]\s*$/m).slice(1); + const matches = blocks.filter((block) => { + try { + return quotedTomlValue(block, "name", "Cargo.lock package") === packageName; + } catch { + return false; + } + }); + if (matches.length !== 1) { + throw new Error(`Cargo.lock must contain exactly one ${packageName} package block`); + } + return quotedTomlValue(matches[0], "version", `Cargo.lock ${packageName} package`); +} + +export function readReleaseSourceVersions(sourceRoot) { + const root = resolve(sourceRoot); + const packageJson = readJson(join(root, "package.json"), "package.json"); + const packageLock = readJson(join(root, "package-lock.json"), "package-lock.json"); + const tauriConfig = readJson( + join(root, "src-tauri", "tauri.conf.json"), + "src-tauri/tauri.conf.json", + ); + const cargoTomlPath = join(root, "src-tauri", "Cargo.toml"); + const cargoLockPath = join(root, "src-tauri", "Cargo.lock"); + const cargoToml = readFileSync(cargoTomlPath, "utf8"); + const cargoLock = readFileSync(cargoLockPath, "utf8"); + + return [ + ["package.json#version", packageJson.version], + ["package-lock.json#version", packageLock.version], + ['package-lock.json#packages[""].version', packageLock.packages?.[""]?.version], + ["src-tauri/tauri.conf.json#version", tauriConfig.version], + [ + "src-tauri/Cargo.toml#[package].version", + quotedTomlValue( + cargoPackageBlock(cargoToml, "src-tauri/Cargo.toml"), + "version", + "src-tauri/Cargo.toml [package]", + ), + ], + [ + 'src-tauri/Cargo.lock#[[package]] name="codex-app-manager".version', + cargoLockPackageVersion(cargoLock, "codex-app-manager"), + ], + ]; +} + +export function assertReleaseSourceVersions(releaseTag, sourceRoot) { + const expected = releaseVersionFromTag(releaseTag); + const entries = readReleaseSourceVersions(sourceRoot); + const mismatches = entries.filter(([, version]) => version !== expected); + if (mismatches.length > 0) { + const detail = mismatches + .map(([label, version]) => `${label}=${JSON.stringify(version)}`) + .join(", "); + throw new Error( + `release tag ${releaseTag} requires application version ${expected}; mismatched source versions: ${detail}`, + ); + } + return { entries, version: expected }; +} + +export function assertLocalReleaseArtifactNames(releaseTag, artifactsDir) { + const version = releaseVersionFromTag(releaseTag); + const expected = requiredReleaseAssetNames(releaseTag).filter((name) => name !== "latest.json"); + const dir = resolve(artifactsDir); + const files = readdirSync(dir, { withFileTypes: true }) + .filter((entry) => entry.isFile()) + .map((entry) => entry.name); + const missing = expected.filter((name) => { + const path = join(dir, name); + return !existsSync(path) || !statSync(path).isFile() || statSync(path).size <= 0; + }); + if (missing.length > 0) { + throw new Error( + `release ${releaseTag} is missing exact local artifact names: ${missing.join(", ")}`, + ); + } + + const updaterInstallers = files.filter((name) => + /_(?:x64|arm64)-setup\.(?:exe|nsis\.zip)(?:\.sig)?$/.test(name), + ); + const wrongVersion = updaterInstallers.filter( + (name) => !name.startsWith(`CodexAppManager_${version}_`), + ); + if (wrongVersion.length > 0) { + throw new Error( + `release ${releaseTag} contains installer artifacts for another version: ${wrongVersion.join(", ")}`, + ); + } + + const unexpected = files.filter( + (name) => name.startsWith("CodexAppManager") && !expected.includes(name), + ); + if (unexpected.length > 0) { + throw new Error( + `release ${releaseTag} contains unexpected local artifact names: ${unexpected.join(", ")}`, + ); + } + + return { expected, version }; +} + +const isCli = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (isCli) { + const [, , mode, releaseTag, path] = process.argv; + try { + if (!mode || !releaseTag || !path || !["source", "artifacts"].includes(mode)) { + throw new Error( + "usage: check-release-version.mjs ", + ); + } + if (mode === "source") { + const result = assertReleaseSourceVersions(releaseTag, path); + console.log( + `[version] ${releaseTag} matches all ${result.entries.length} application version declarations`, + ); + } else { + const result = assertLocalReleaseArtifactNames(releaseTag, path); + console.log( + `[version] ${releaseTag} matches all ${result.expected.length} exact local release artifact names`, + ); + } + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + console.error(`::error::[version] ${detail}`); + process.exitCode = 1; + } +} diff --git a/scripts/release-workflow.test.mjs b/scripts/release-workflow.test.mjs index d223a6f..113f341 100644 --- a/scripts/release-workflow.test.mjs +++ b/scripts/release-workflow.test.mjs @@ -1,4 +1,5 @@ -import { readFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -8,6 +9,11 @@ import { assertImmutableReleasesEnabled, verifyImmutableReleases, } from "./check-immutable-releases.mjs"; +import { requiredReleaseAssetNames } from "./check-release-reuse.mjs"; +import { + assertLocalReleaseArtifactNames, + assertReleaseSourceVersions, +} from "./check-release-version.mjs"; const repoRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); const workflow = await readFile(join(repoRoot, ".github/workflows/release.yml"), "utf8"); @@ -21,6 +27,93 @@ describe("release workflow recovery invariants", () => { ); }); + it("rejects every manual dispatch outside the full default-branch ref before checkout or secrets", () => { + const dispatchInputs = workflow.slice( + workflow.indexOf(" workflow_dispatch:\n"), + workflow.indexOf("\npermissions:\n"), + ); + const reject = workflow.slice( + workflow.indexOf(" reject_untrusted_target_dispatch:\n"), + workflow.indexOf(" preflight:\n"), + ); + const preflight = workflow.slice( + workflow.indexOf(" preflight:\n"), + workflow.indexOf(" prepare:\n"), + ); + const prepare = workflow.slice( + workflow.indexOf(" prepare:\n"), + workflow.indexOf(" build:\n"), + ); + + expect(reject).toContain("github.event_name == 'workflow_dispatch'"); + expect(reject).toContain( + "github.ref != format('refs/heads/{0}', github.event.repository.default_branch)", + ); + expect(reject).not.toContain("inputs.target_tag != ''"); + expect(reject).not.toContain("actions/checkout@"); + expect(reject).not.toContain("environment:"); + expect(reject).not.toContain("${{ secrets."); + expect(dispatchInputs).toMatch(/target_tag:\n\s+description:.*\n\s+required: true/); + + expect(preflight).toContain( + "github.ref == format('refs/heads/{0}', github.event.repository.default_branch)", + ); + expect(preflight).toContain("workflow_dispatch requires a non-empty target_tag"); + expect(preflight).not.toContain("actions/checkout@"); + expect(preflight).not.toContain("environment:"); + expect(preflight).not.toContain("${{ secrets."); + expect(prepare).toContain("needs: preflight"); + expect(prepare).toContain("needs.preflight.result == 'success'"); + }); + + it("binds the release tag to all six source versions and exact local artifact names", async () => { + const packageJson = JSON.parse(await readFile(join(repoRoot, "package.json"), "utf8")); + const releaseTag = `v${packageJson.version}`; + const source = assertReleaseSourceVersions(releaseTag, repoRoot); + expect(source.entries).toHaveLength(6); + expect(() => assertReleaseSourceVersions("v9.9.9", repoRoot)).toThrow( + "mismatched source versions", + ); + + const artifactsDir = await mkdtemp(join(tmpdir(), "cam-release-artifacts-")); + try { + const expected = requiredReleaseAssetNames(releaseTag).filter( + (name) => name !== "latest.json", + ); + await Promise.all(expected.map((name) => writeFile(join(artifactsDir, name), "x"))); + expect(assertLocalReleaseArtifactNames(releaseTag, artifactsDir).expected).toEqual( + expected, + ); + + await writeFile(join(artifactsDir, "CodexAppManager_9.9.9_x64-setup.exe"), "x"); + expect(() => assertLocalReleaseArtifactNames(releaseTag, artifactsDir)).toThrow( + "installer artifacts for another version", + ); + } finally { + await rm(artifactsDir, { force: true, recursive: true }); + } + + const prepare = workflow.slice( + workflow.indexOf(" prepare:\n"), + workflow.indexOf(" build:\n"), + ); + expect(prepare).toContain('ref: ${{ env.RELEASE_TAG }}'); + expect(prepare).toContain( + 'node scripts/check-release-version.mjs source "$RELEASE_TAG" release-source', + ); + expect(releaseJob).toContain( + 'node scripts/check-release-version.mjs artifacts "$RELEASE_TAG" dist', + ); + const manifestStep = releaseJob.slice( + releaseJob.indexOf("- name: Generate updater manifest (latest.json)"), + releaseJob.indexOf( + "- name: Verify local updater signatures before immutable publication", + ), + ); + expect(manifestStep).toContain("node scripts/validate-release-manifest.mjs"); + expect(manifestStep).not.toContain('if [[ "$RELEASE_TAG" != *-* ]]'); + }); + it("enforces the single-writer boundary for the unconditional IHEP follower", () => { expect(workflow).toContain( "correctness boundary for the unconditional IHEP follower", @@ -112,6 +205,11 @@ describe("release workflow recovery invariants", () => { const localVerify = releaseJob.indexOf( "- name: Verify local updater signatures before immutable publication", ); + const attest = releaseJob.indexOf("- name: Attest fresh build provenance"); + const verifyExisting = releaseJob.indexOf( + "- name: Verify existing immutable release provenance", + ); + const provenance = releaseJob.indexOf("- name: Resolve provenance gate"); const stage = releaseJob.indexOf("- name: Stage CDN mirror candidate"); const mirrorVerify = releaseJob.indexOf( "- name: Verify staged CDN mirror before immutable publication", @@ -121,24 +219,27 @@ describe("release workflow recovery invariants", () => { const publishedVerify = releaseJob.indexOf( "- name: Verify published immutable Release and asset digests", ); - const attest = releaseJob.indexOf("- name: Attest build provenance"); const promote = releaseJob.indexOf("- name: Promote CDN mirror latest"); const winget = releaseJob.indexOf("- name: Trigger winget submission"); const summary = releaseJob.indexOf("- name: Write release summary"); expect(localVerify).toBeGreaterThan(-1); - expect(stage).toBeGreaterThan(localVerify); + expect(attest).toBeGreaterThan(localVerify); + expect(verifyExisting).toBeGreaterThan(attest); + expect(provenance).toBeGreaterThan(verifyExisting); + expect(stage).toBeGreaterThan(provenance); expect(mirrorVerify).toBeGreaterThan(stage); expect(upload).toBeGreaterThan(mirrorVerify); expect(upload).toBeGreaterThan(-1); expect(publish).toBeGreaterThan(upload); expect(publishedVerify).toBeGreaterThan(publish); - expect(attest).toBeGreaterThan(publishedVerify); - expect(promote).toBeGreaterThan(attest); + expect(promote).toBeGreaterThan(publishedVerify); const uploadStep = releaseJob.slice(upload, publish); const publishStep = releaseJob.slice(publish, publishedVerify); - const verifyStep = releaseJob.slice(publishedVerify, attest); - const attestStep = releaseJob.slice(attest, promote); + const verifyStep = releaseJob.slice(publishedVerify, promote); + const attestStep = releaseJob.slice(attest, verifyExisting); + const existingStep = releaseJob.slice(verifyExisting, provenance); + const provenanceStep = releaseJob.slice(provenance, stage); const promoteStep = releaseJob.slice(promote, winget); const wingetStep = releaseJob.slice(winget, summary); const localVerifyStep = releaseJob.slice(localVerify, stage); @@ -147,40 +248,54 @@ describe("release workflow recovery invariants", () => { expect(mirrorVerifyStep).toContain("MIRROR_PHASE: verify"); expect(mirrorVerifyStep).toContain("bash scripts/sync-mirror.sh dist"); expect(uploadStep).toContain("draft: true"); + expect(uploadStep).toContain("steps.provenance.outputs.ready == 'true'"); expect(uploadStep).toContain("prerelease: ${{ contains(env.RELEASE_TAG, '-') }}"); expect(uploadStep).toContain("files: |"); expect(publishStep).not.toMatch(/^\s+draft:/m); expect(publishStep).not.toContain("files: |"); + expect(publishStep).toContain("steps.provenance.outputs.ready == 'true'"); expect(verifyStep).toContain("node scripts/check-release-reuse.mjs"); expect(verifyStep).toContain("did not become immutable with canonical asset digests"); - expect(attestStep).toContain( - "steps.publish_release.outcome == 'success' || steps.release_source.outputs.existing == 'true'", - ); + expect(verifyStep).toContain("published immutable digest does not match attested local bytes"); + expect(attestStep).toContain("steps.release_source.outputs.existing != 'true'"); expect(attestStep).toContain("actions/attest-build-provenance@"); expect(attestStep).not.toContain("continue-on-error: true"); - expect(promoteStep).toContain("steps.attest.outcome == 'success'"); - expect(wingetStep).toContain("steps.attest.outcome == 'success'"); + expect(existingStep).toContain("gh attestation verify"); + expect(existingStep).toContain('--signer-workflow "$signer_workflow"'); + expect(existingStep).toContain('--source-ref "refs/tags/$RELEASE_TAG"'); + expect(existingStep).toContain('--source-digest "$RELEASE_SOURCE_SHA"'); + expect(provenanceStep).toContain('echo "ready=true" >> "$GITHUB_OUTPUT"'); + expect(promoteStep).toContain("steps.provenance.outputs.ready == 'true'"); + expect(promoteStep).not.toContain("steps.attest_fresh.outcome"); + expect(wingetStep).toContain("steps.provenance.outputs.ready == 'true'"); + expect(wingetStep).not.toContain("steps.attest_fresh.outcome"); expect(releaseJob.slice(0, upload)).toContain( "rm -f dist/latest.mirror.json dist/latest.json", ); }); - it("attests only immutable Release bytes on a failed-job reuse", () => { + it("verifies existing immutable provenance without minting a rerun attestation", () => { const source = releaseJob.indexOf("- name: Resolve immutable release artifact source"); const validate = releaseJob.indexOf("- name: Validate final release artifacts"); - const attest = releaseJob.indexOf("- name: Attest build provenance"); - const promote = releaseJob.indexOf("- name: Promote CDN mirror latest"); + const attest = releaseJob.indexOf("- name: Attest fresh build provenance"); + const verifyExisting = releaseJob.indexOf( + "- name: Verify existing immutable release provenance", + ); + const provenance = releaseJob.indexOf("- name: Resolve provenance gate"); const sourceStep = releaseJob.slice(source, validate); - const attestStep = releaseJob.slice(attest, promote); + const attestStep = releaseJob.slice(attest, verifyExisting); + const existingStep = releaseJob.slice(verifyExisting, provenance); expect(sourceStep).toContain("gh release download"); expect(sourceStep).toContain("--pattern 'CodexAppManager*'"); expect(sourceStep).toContain("--pattern 'latest.json'"); expect(sourceStep).toContain('actual_digest="sha256:$(sha256sum "$file"'); expect(sourceStep).toContain('if [[ "$actual_digest" != "$expected_digest" ]]'); - expect(attestStep).toContain("dist/*"); - expect(attestStep).toContain("latest.json"); - expect(attestStep).not.toContain("SHA256SUMS"); - expect(attestStep).not.toContain("release-assets/*"); + expect(attestStep).toContain("steps.release_source.outputs.existing != 'true'"); + expect(attestStep).not.toContain("steps.release_source.outputs.existing == 'true'"); + expect(existingStep).toContain("steps.release_source.outputs.existing == 'true'"); + expect(existingStep).toContain("for file in dist/* latest.json"); + expect(existingStep).toContain("gh attestation verify"); + expect(existingStep).toContain("--deny-self-hosted-runners"); }); }); diff --git a/scripts/write-release-summary.mjs b/scripts/write-release-summary.mjs index 0c14802..8785983 100644 --- a/scripts/write-release-summary.mjs +++ b/scripts/write-release-summary.mjs @@ -176,7 +176,8 @@ rows.push(`| Mirror pre-publish verify | ${env.MIRROR_VERIFY_OUTCOME || "skipped rows.push(`| Mirror promote | ${env.MIRROR_PROMOTE_OUTCOME || "skipped"} |`); rows.push(`| winget dispatch | ${env.WINGET_OUTCOME || "skipped"} |`); rows.push(`| SBOM | ${env.SBOM_OUTCOME || "skipped"} |`); -rows.push(`| provenance attestation | ${env.ATTESTATION_OUTCOME || "skipped"} |`); +rows.push(`| fresh provenance attestation | ${env.ATTESTATION_OUTCOME || "skipped"} |`); +rows.push(`| provenance mode | ${env.PROVENANCE_MODE || "blocked"} |`); rows.push(""); rows.push( "winget dispatch only starts the downstream submission workflow; until the first package is accepted in microsoft/winget-pkgs, downstream failures are expected noise.", From a80c6ed455ea73e1e32b5499ccd2e3a206437672 Mon Sep 17 00:00:00 2001 From: wangnov <48670012+Wangnov@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:20:30 +0800 Subject: [PATCH 5/8] fix(release): protect immutable release tags --- .github/workflows/release.yml | 69 ++++++-- scripts/check-release-tag-protection.mjs | 162 +++++++++++++++++ scripts/release-workflow.test.mjs | 216 +++++++++++++++++++---- 3 files changed, 399 insertions(+), 48 deletions(-) create mode 100644 scripts/check-release-tag-protection.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1b46d7e..f489ae8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,6 +32,8 @@ name: Release # round trips; promotion never relies on its ignored conditional headers. # GitHub Immutable Releases must be enabled before any release starts; the # workflow verifies the repository setting with IMMUTABLE_RELEASES_READ_TOKEN. +# An active repository tag ruleset must also forbid update and deletion for +# refs/tags/v*; the same read-only token verifies the policy and live peeled SHA. # Every reused asset is also pinned to the API-provided sha256 digest. # The old MANAGER_*_ACCESS_KEY_ID / SECRET_ACCESS_KEY names must stay deleted; # otherwise a historical workflow revision could still perform unconditional writes. @@ -135,18 +137,18 @@ jobs: release_reusable: ${{ steps.lookup.outputs.release_reusable }} release_asset_digests: ${{ steps.lookup.outputs.release_asset_digests }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: 20 # Verify the release source rather than the workflow source. For a # target_tag recovery the workflow itself comes from the protected default # branch, while the six application version declarations come from the tag. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ env.RELEASE_TAG }} path: release-source @@ -162,6 +164,12 @@ jobs: echo "::error::release tag $RELEASE_TAG moved away from triggering commit $GITHUB_SHA" exit 1 fi + echo "RELEASE_SOURCE_SHA=$release_source_sha" >> "$GITHUB_ENV" + + - name: Require protected immutable release tag + env: + GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }} + run: node scripts/check-release-tag-protection.mjs "$RELEASE_TAG" "$RELEASE_SOURCE_SHA" - name: Require GitHub Immutable Releases env: @@ -213,24 +221,32 @@ jobs: # Intel build on GitHub's current x86_64 image — macos-13 retired # 2025-12; macos-15-intel is the last Intel image (until ~2027-08). - { platform: macos-15-intel, target: x86_64-apple-darwin, os: macos } - - { platform: windows-latest, target: x86_64-pc-windows-msvc, os: windows } - - { platform: windows-latest, target: aarch64-pc-windows-msvc, os: windows } + - { + platform: windows-latest, + target: x86_64-pc-windows-msvc, + os: windows, + } + - { + platform: windows-latest, + target: aarch64-pc-windows-msvc, + os: windows, + } runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: 20 cache: npm - - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: targets: ${{ matrix.target }} - - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: workspaces: src-tauri -> target @@ -446,7 +462,7 @@ jobs: fi echo "::endgroup::" - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: bundle-${{ matrix.target }}-${{ github.run_id }}-${{ github.run_attempt }} path: dist-artifacts/* @@ -527,7 +543,7 @@ jobs: attestations: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -548,14 +564,14 @@ jobs: exit 1 fi - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: 20 # Re-run this gate inside every release-job attempt. GitHub's "Re-run # failed jobs" does not re-run prepare, and no immutable publication may # rely only on a previous attempt's version decision. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ env.RELEASE_TAG }} path: release-source @@ -791,7 +807,7 @@ jobs: - name: Attest fresh build provenance id: attest_fresh if: ${{ steps.release_source.outputs.existing != 'true' }} - uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 with: subject-path: | dist/* @@ -928,7 +944,7 @@ jobs: fi cat SHA256SUMS - - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - name: Generate SBOMs id: sbom @@ -960,13 +976,22 @@ jobs: exit "$status" + # The repository-level ruleset closes the long validation-to-publication + # window by forbidding v* tag updates/deletions. Re-read both the policy and + # the peeled commit immediately before creating the canonical draft. + - name: Re-check protected release tag before draft upload + if: ${{ steps.provenance.outputs.ready == 'true' && steps.release_source.outputs.existing != 'true' }} + env: + GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }} + run: node scripts/check-release-tag-protection.mjs "$RELEASE_TAG" "$RELEASE_SOURCE_SHA" + # Explicit draft-first publication keeps asset upload ahead of publication # for both stable and prerelease tags. This is required when GitHub immutable # releases are enabled: a published release can no longer accept assets. - name: Upload GitHub Release draft id: upload_release_draft if: ${{ steps.provenance.outputs.ready == 'true' && steps.release_source.outputs.existing != 'true' }} - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3 + uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3 with: tag_name: ${{ env.RELEASE_TAG }} draft: true @@ -983,12 +1008,20 @@ jobs: SHA256SUMS release-assets/* + # Upload can take long enough for a repository policy regression to matter. + # Fence the transition from mutable draft to immutable publication again. + - name: Re-check protected release tag before publication + if: ${{ steps.provenance.outputs.ready == 'true' && steps.release_source.outputs.existing != 'true' }} + env: + GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }} + run: node scripts/check-release-tag-protection.mjs "$RELEASE_TAG" "$RELEASE_SOURCE_SHA" + # Omitting `draft` tells action-gh-release v3 to publish the existing draft # after all uploads succeeded while retaining its stable/prerelease metadata. - name: Publish GitHub Release id: publish_release if: ${{ steps.provenance.outputs.ready == 'true' && steps.release_source.outputs.existing != 'true' }} - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3 + uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3 with: tag_name: ${{ env.RELEASE_TAG }} @@ -1094,7 +1127,7 @@ jobs: - name: Upload release audit records if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: release-audit-${{ env.RELEASE_TAG }}-${{ github.run_attempt }} path: | diff --git a/scripts/check-release-tag-protection.mjs b/scripts/check-release-tag-protection.mjs new file mode 100644 index 0000000..ecc4fc7 --- /dev/null +++ b/scripts/check-release-tag-protection.mjs @@ -0,0 +1,162 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +const API_VERSION = "2026-03-10"; +const REPOSITORY_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const RELEASE_TAG_PATTERN = + /^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$/; +const SHA_PATTERN = /^[0-9a-f]{40}$/; +const REQUIRED_REF_PATTERN = "refs/tags/v*"; + +function parseJson(stdout, label) { + try { + return JSON.parse(stdout); + } catch (error) { + throw new Error(`${label} was not valid JSON: ${error.message}`); + } +} + +function errorDetail(result) { + return String( + result.error?.message || result.stderr || result.stdout || "unknown error", + ) + .replace(/\s+/g, " ") + .trim() + .slice(0, 500); +} + +export function assertReleaseTagRuleset(rulesets) { + if (!Array.isArray(rulesets)) { + throw new Error("GitHub tag rulesets response must be an array"); + } + + const matching = rulesets.find((ruleset) => { + const includes = ruleset?.conditions?.ref_name?.include; + const excludes = ruleset?.conditions?.ref_name?.exclude; + const ruleTypes = new Set( + Array.isArray(ruleset?.rules) + ? ruleset.rules.map((rule) => rule?.type) + : [], + ); + const bypassActors = ruleset?.bypass_actors; + return ( + ruleset?.target === "tag" && + ruleset?.enforcement === "active" && + Array.isArray(includes) && + includes.includes(REQUIRED_REF_PATTERN) && + Array.isArray(excludes) && + excludes.length === 0 && + ruleTypes.has("update") && + ruleTypes.has("deletion") && + (!Array.isArray(bypassActors) || bypassActors.length === 0) + ); + }); + + if (!matching) { + throw new Error( + `an active tag ruleset must protect ${REQUIRED_REF_PATTERN} from update and deletion without exclusions or visible bypass actors`, + ); + } + + return { id: matching.id, name: matching.name }; +} + +export function assertReleaseTagCommit(actualSha, expectedSha) { + if (!SHA_PATTERN.test(expectedSha)) { + throw new Error( + "expected release source SHA must be a lowercase 40-character commit SHA", + ); + } + if (!SHA_PATTERN.test(actualSha)) { + throw new Error("live release tag did not peel to a commit SHA"); + } + if (actualSha !== expectedSha) { + throw new Error( + `release tag moved after validation: expected ${expectedSha}, found ${actualSha}`, + ); + } + return { sha: actualSha }; +} + +export function verifyReleaseTagProtection({ + repository = process.env.GITHUB_REPOSITORY || "", + releaseTag = process.argv[2] || process.env.RELEASE_TAG || "", + expectedSha = process.argv[3] || process.env.RELEASE_SOURCE_SHA || "", + token = process.env.GH_TOKEN || "", + runner = spawnSync, +} = {}) { + if (!REPOSITORY_PATTERN.test(repository)) { + throw new Error("GITHUB_REPOSITORY must be an owner/repository name"); + } + if (!RELEASE_TAG_PATTERN.test(releaseTag)) { + throw new Error("release tag must be a semantic vX.Y.Z tag"); + } + if (!SHA_PATTERN.test(expectedSha)) { + throw new Error( + "expected release source SHA must be a lowercase 40-character commit SHA", + ); + } + if (!token.trim()) { + throw new Error( + "IMMUTABLE_RELEASES_READ_TOKEN is missing; release cannot verify tag protection", + ); + } + + const api = (endpoint) => { + const result = runner( + "gh", + ["api", "-H", `X-GitHub-Api-Version: ${API_VERSION}`, endpoint], + { + encoding: "utf8", + env: { ...process.env, GH_TOKEN: token }, + }, + ); + if (result.error || result.status !== 0) { + throw new Error( + `could not verify GitHub release tag protection: ${errorDetail(result)}`, + ); + } + return parseJson(result.stdout, `GitHub API response for ${endpoint}`); + }; + + const summaries = api( + `repos/${repository}/rulesets?targets=tag&per_page=100`, + ); + if (!Array.isArray(summaries)) { + throw new Error("GitHub tag rulesets response must be an array"); + } + const details = summaries + .filter( + (ruleset) => + ruleset?.target === "tag" && ruleset?.enforcement === "active", + ) + .map((ruleset) => api(`repos/${repository}/rulesets/${ruleset.id}`)); + const ruleset = assertReleaseTagRuleset(details); + + let object = api(`repos/${repository}/git/ref/tags/${releaseTag}`).object; + for (let depth = 0; object?.type === "tag" && depth < 5; depth += 1) { + object = api(`repos/${repository}/git/tags/${object.sha}`).object; + } + if (object?.type !== "commit") { + throw new Error("live release tag did not peel to a commit"); + } + const commit = assertReleaseTagCommit(object.sha, expectedSha); + + return { commit, ruleset }; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + try { + const result = verifyReleaseTagProtection(); + console.log( + `Release tag protection verified by ${result.ruleset.name || result.ruleset.id}: ${result.commit.sha}`, + ); + } catch (error) { + console.error( + `::error::${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + } +} diff --git a/scripts/release-workflow.test.mjs b/scripts/release-workflow.test.mjs index 113f341..fddc868 100644 --- a/scripts/release-workflow.test.mjs +++ b/scripts/release-workflow.test.mjs @@ -9,6 +9,11 @@ import { assertImmutableReleasesEnabled, verifyImmutableReleases, } from "./check-immutable-releases.mjs"; +import { + assertReleaseTagCommit, + assertReleaseTagRuleset, + verifyReleaseTagProtection, +} from "./check-release-tag-protection.mjs"; import { requiredReleaseAssetNames } from "./check-release-reuse.mjs"; import { assertLocalReleaseArtifactNames, @@ -16,8 +21,14 @@ import { } from "./check-release-version.mjs"; const repoRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); -const workflow = await readFile(join(repoRoot, ".github/workflows/release.yml"), "utf8"); -const mirrorRelease = await readFile(join(repoRoot, "scripts/mirror-release.mjs"), "utf8"); +const workflow = await readFile( + join(repoRoot, ".github/workflows/release.yml"), + "utf8", +); +const mirrorRelease = await readFile( + join(repoRoot, "scripts/mirror-release.mjs"), + "utf8", +); const releaseJob = workflow.slice(workflow.indexOf(" release:\n")); describe("release workflow recovery invariants", () => { @@ -53,12 +64,16 @@ describe("release workflow recovery invariants", () => { expect(reject).not.toContain("actions/checkout@"); expect(reject).not.toContain("environment:"); expect(reject).not.toContain("${{ secrets."); - expect(dispatchInputs).toMatch(/target_tag:\n\s+description:.*\n\s+required: true/); + expect(dispatchInputs).toMatch( + /target_tag:\n\s+description:.*\n\s+required: true/, + ); expect(preflight).toContain( "github.ref == format('refs/heads/{0}', github.event.repository.default_branch)", ); - expect(preflight).toContain("workflow_dispatch requires a non-empty target_tag"); + expect(preflight).toContain( + "workflow_dispatch requires a non-empty target_tag", + ); expect(preflight).not.toContain("actions/checkout@"); expect(preflight).not.toContain("environment:"); expect(preflight).not.toContain("${{ secrets."); @@ -67,7 +82,9 @@ describe("release workflow recovery invariants", () => { }); it("binds the release tag to all six source versions and exact local artifact names", async () => { - const packageJson = JSON.parse(await readFile(join(repoRoot, "package.json"), "utf8")); + const packageJson = JSON.parse( + await readFile(join(repoRoot, "package.json"), "utf8"), + ); const releaseTag = `v${packageJson.version}`; const source = assertReleaseSourceVersions(releaseTag, repoRoot); expect(source.entries).toHaveLength(6); @@ -75,20 +92,27 @@ describe("release workflow recovery invariants", () => { "mismatched source versions", ); - const artifactsDir = await mkdtemp(join(tmpdir(), "cam-release-artifacts-")); + const artifactsDir = await mkdtemp( + join(tmpdir(), "cam-release-artifacts-"), + ); try { const expected = requiredReleaseAssetNames(releaseTag).filter( (name) => name !== "latest.json", ); - await Promise.all(expected.map((name) => writeFile(join(artifactsDir, name), "x"))); - expect(assertLocalReleaseArtifactNames(releaseTag, artifactsDir).expected).toEqual( - expected, + await Promise.all( + expected.map((name) => writeFile(join(artifactsDir, name), "x")), ); + expect( + assertLocalReleaseArtifactNames(releaseTag, artifactsDir).expected, + ).toEqual(expected); - await writeFile(join(artifactsDir, "CodexAppManager_9.9.9_x64-setup.exe"), "x"); - expect(() => assertLocalReleaseArtifactNames(releaseTag, artifactsDir)).toThrow( - "installer artifacts for another version", + await writeFile( + join(artifactsDir, "CodexAppManager_9.9.9_x64-setup.exe"), + "x", ); + expect(() => + assertLocalReleaseArtifactNames(releaseTag, artifactsDir), + ).toThrow("installer artifacts for another version"); } finally { await rm(artifactsDir, { force: true, recursive: true }); } @@ -97,7 +121,7 @@ describe("release workflow recovery invariants", () => { workflow.indexOf(" prepare:\n"), workflow.indexOf(" build:\n"), ); - expect(prepare).toContain('ref: ${{ env.RELEASE_TAG }}'); + expect(prepare).toContain("ref: ${{ env.RELEASE_TAG }}"); expect(prepare).toContain( 'node scripts/check-release-version.mjs source "$RELEASE_TAG" release-source', ); @@ -110,7 +134,9 @@ describe("release workflow recovery invariants", () => { "- name: Verify local updater signatures before immutable publication", ), ); - expect(manifestStep).toContain("node scripts/validate-release-manifest.mjs"); + expect(manifestStep).toContain( + "node scripts/validate-release-manifest.mjs", + ); expect(manifestStep).not.toContain('if [[ "$RELEASE_TAG" != *-* ]]'); }); @@ -153,11 +179,15 @@ describe("release workflow recovery invariants", () => { }); it("uses the target tag updater trust root without executing historical scripts", () => { - const refresh = releaseJob.indexOf("- name: Refresh immutable release state"); + const refresh = releaseJob.indexOf( + "- name: Refresh immutable release state", + ); const resolveTrust = releaseJob.indexOf( "- name: Resolve updater trust root for release tag", ); - const download = releaseJob.indexOf("- name: Download canonical build artifacts"); + const download = releaseJob.indexOf( + "- name: Download canonical build artifacts", + ); const localVerify = releaseJob.indexOf( "- name: Verify local updater signatures before immutable publication", ); @@ -179,7 +209,9 @@ describe("release workflow recovery invariants", () => { }); it("fails closed when immutable settings are disabled or cannot be queried", () => { - expect(assertImmutableReleasesEnabled({ enabled: true })).toEqual({ enabled: true }); + expect(assertImmutableReleasesEnabled({ enabled: true })).toEqual({ + enabled: true, + }); expect(() => assertImmutableReleasesEnabled({ enabled: false })).toThrow( "GitHub Immutable Releases are disabled", ); @@ -196,9 +228,111 @@ describe("release workflow recovery invariants", () => { workflow.indexOf(" build:\n"), ); expect(prepare).toContain("environment: release"); - expect(prepare).toContain("GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }}"); + expect(prepare).toContain( + "GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }}", + ); expect(prepare).toContain("run: node scripts/check-immutable-releases.mjs"); - expect(releaseJob).toContain("run: node scripts/check-immutable-releases.mjs"); + expect(releaseJob).toContain( + "run: node scripts/check-immutable-releases.mjs", + ); + }); + + it("requires immutable release tags and rechecks the live peeled commit at publication", () => { + const protectedRuleset = { + id: 1, + name: "immutable release tags", + target: "tag", + enforcement: "active", + bypass_actors: [], + conditions: { ref_name: { include: ["refs/tags/v*"], exclude: [] } }, + rules: [{ type: "update" }, { type: "deletion" }], + }; + expect(assertReleaseTagRuleset([protectedRuleset])).toEqual({ + id: 1, + name: "immutable release tags", + }); + expect(() => + assertReleaseTagRuleset([ + { ...protectedRuleset, rules: [{ type: "deletion" }] }, + ]), + ).toThrow("must protect refs/tags/v*"); + expect(() => + assertReleaseTagRuleset([ + { + ...protectedRuleset, + bypass_actors: [{ actor_type: "User", actor_id: 1 }], + }, + ]), + ).toThrow("visible bypass actors"); + + const expectedSha = "a".repeat(40); + expect(assertReleaseTagCommit(expectedSha, expectedSha)).toEqual({ + sha: expectedSha, + }); + expect(() => assertReleaseTagCommit("b".repeat(40), expectedSha)).toThrow( + "release tag moved after validation", + ); + + const responses = new Map([ + [ + "repos/owner/repo/rulesets?targets=tag&per_page=100", + [protectedRuleset], + ], + ["repos/owner/repo/rulesets/1", protectedRuleset], + [ + "repos/owner/repo/git/ref/tags/v1.2.3", + { object: { type: "tag", sha: "b".repeat(40) } }, + ], + [ + `repos/owner/repo/git/tags/${"b".repeat(40)}`, + { object: { type: "commit", sha: expectedSha } }, + ], + ]); + const runner = (_command, args) => ({ + status: 0, + stderr: "", + stdout: JSON.stringify(responses.get(args.at(-1))), + }); + expect( + verifyReleaseTagProtection({ + repository: "owner/repo", + releaseTag: "v1.2.3", + expectedSha, + token: "read-token", + runner, + }), + ).toEqual({ + commit: { sha: expectedSha }, + ruleset: { id: 1, name: "immutable release tags" }, + }); + + const prepare = workflow.slice( + workflow.indexOf(" prepare:\n"), + workflow.indexOf(" build:\n"), + ); + expect(prepare).toContain("node scripts/check-release-tag-protection.mjs"); + expect(prepare).toContain('echo "RELEASE_SOURCE_SHA=$release_source_sha"'); + + const upload = releaseJob.indexOf("- name: Upload GitHub Release draft"); + const publish = releaseJob.indexOf("- name: Publish GitHub Release"); + const beforeUpload = releaseJob.lastIndexOf( + "- name: Re-check protected release tag before draft upload", + upload, + ); + const beforePublish = releaseJob.lastIndexOf( + "- name: Re-check protected release tag before publication", + publish, + ); + expect(beforeUpload).toBeGreaterThan(-1); + expect(beforeUpload).toBeLessThan(upload); + expect(beforePublish).toBeGreaterThan(upload); + expect(beforePublish).toBeLessThan(publish); + expect(releaseJob.slice(beforeUpload, upload)).toContain( + 'node scripts/check-release-tag-protection.mjs "$RELEASE_TAG" "$RELEASE_SOURCE_SHA"', + ); + expect(releaseJob.slice(beforePublish, publish)).toContain( + 'node scripts/check-release-tag-protection.mjs "$RELEASE_TAG" "$RELEASE_SOURCE_SHA"', + ); }); it("uploads stable and prerelease assets to a draft before publishing", () => { @@ -244,20 +378,30 @@ describe("release workflow recovery invariants", () => { const wingetStep = releaseJob.slice(winget, summary); const localVerifyStep = releaseJob.slice(localVerify, stage); const mirrorVerifyStep = releaseJob.slice(mirrorVerify, upload); - expect(localVerifyStep).toContain("node scripts/verify-release-artifacts.mjs"); + expect(localVerifyStep).toContain( + "node scripts/verify-release-artifacts.mjs", + ); expect(mirrorVerifyStep).toContain("MIRROR_PHASE: verify"); expect(mirrorVerifyStep).toContain("bash scripts/sync-mirror.sh dist"); expect(uploadStep).toContain("draft: true"); expect(uploadStep).toContain("steps.provenance.outputs.ready == 'true'"); - expect(uploadStep).toContain("prerelease: ${{ contains(env.RELEASE_TAG, '-') }}"); + expect(uploadStep).toContain( + "prerelease: ${{ contains(env.RELEASE_TAG, '-') }}", + ); expect(uploadStep).toContain("files: |"); expect(publishStep).not.toMatch(/^\s+draft:/m); expect(publishStep).not.toContain("files: |"); expect(publishStep).toContain("steps.provenance.outputs.ready == 'true'"); expect(verifyStep).toContain("node scripts/check-release-reuse.mjs"); - expect(verifyStep).toContain("did not become immutable with canonical asset digests"); - expect(verifyStep).toContain("published immutable digest does not match attested local bytes"); - expect(attestStep).toContain("steps.release_source.outputs.existing != 'true'"); + expect(verifyStep).toContain( + "did not become immutable with canonical asset digests", + ); + expect(verifyStep).toContain( + "published immutable digest does not match attested local bytes", + ); + expect(attestStep).toContain( + "steps.release_source.outputs.existing != 'true'", + ); expect(attestStep).toContain("actions/attest-build-provenance@"); expect(attestStep).not.toContain("continue-on-error: true"); expect(existingStep).toContain("gh attestation verify"); @@ -275,8 +419,12 @@ describe("release workflow recovery invariants", () => { }); it("verifies existing immutable provenance without minting a rerun attestation", () => { - const source = releaseJob.indexOf("- name: Resolve immutable release artifact source"); - const validate = releaseJob.indexOf("- name: Validate final release artifacts"); + const source = releaseJob.indexOf( + "- name: Resolve immutable release artifact source", + ); + const validate = releaseJob.indexOf( + "- name: Validate final release artifacts", + ); const attest = releaseJob.indexOf("- name: Attest fresh build provenance"); const verifyExisting = releaseJob.indexOf( "- name: Verify existing immutable release provenance", @@ -290,10 +438,18 @@ describe("release workflow recovery invariants", () => { expect(sourceStep).toContain("--pattern 'CodexAppManager*'"); expect(sourceStep).toContain("--pattern 'latest.json'"); expect(sourceStep).toContain('actual_digest="sha256:$(sha256sum "$file"'); - expect(sourceStep).toContain('if [[ "$actual_digest" != "$expected_digest" ]]'); - expect(attestStep).toContain("steps.release_source.outputs.existing != 'true'"); - expect(attestStep).not.toContain("steps.release_source.outputs.existing == 'true'"); - expect(existingStep).toContain("steps.release_source.outputs.existing == 'true'"); + expect(sourceStep).toContain( + 'if [[ "$actual_digest" != "$expected_digest" ]]', + ); + expect(attestStep).toContain( + "steps.release_source.outputs.existing != 'true'", + ); + expect(attestStep).not.toContain( + "steps.release_source.outputs.existing == 'true'", + ); + expect(existingStep).toContain( + "steps.release_source.outputs.existing == 'true'", + ); expect(existingStep).toContain("for file in dist/* latest.json"); expect(existingStep).toContain("gh attestation verify"); expect(existingStep).toContain("--deny-self-hosted-runners"); From ad3bad18b5fd9054a3b4bc9a3e6fb4f296acf661 Mon Sep 17 00:00:00 2001 From: wangnov <48670012+Wangnov@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:41:19 +0800 Subject: [PATCH 6/8] fix(release): authorize release tag creation --- .github/workflows/release.yml | 5 +-- docs/release.md | 3 ++ scripts/check-release-tag-protection.mjs | 46 ++++++++++++++++++++++-- scripts/release-workflow.test.mjs | 38 +++++++++++++++++++- 4 files changed, 87 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f489ae8..e1c51b8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,8 +32,9 @@ name: Release # round trips; promotion never relies on its ignored conditional headers. # GitHub Immutable Releases must be enabled before any release starts; the # workflow verifies the repository setting with IMMUTABLE_RELEASES_READ_TOKEN. -# An active repository tag ruleset must also forbid update and deletion for -# refs/tags/v*; the same read-only token verifies the policy and live peeled SHA. +# Separate active tag rulesets must restrict refs/tags/v* creation to the named +# release publisher and forbid every update/deletion; the same read-only token +# verifies those policies and the live peeled SHA. # Every reused asset is also pinned to the API-provided sha256 digest. # The old MANAGER_*_ACCESS_KEY_ID / SECRET_ACCESS_KEY names must stay deleted; # otherwise a historical workflow revision could still perform unconditional writes. diff --git a/docs/release.md b/docs/release.md index 3e3489e..228994c 100644 --- a/docs/release.md +++ b/docs/release.md @@ -131,6 +131,9 @@ Before `prepare` permits any build, and again at the start of every release-job attempt, the workflow queries the repository Immutable Releases setting with a dedicated fine-grained, read-only token. A missing token, failed query, or `enabled: false` response fails closed before draft upload or mirror publication. +Two separate active tag rulesets restrict `refs/tags/v*` creation to the named +release publisher and forbid update/deletion for everyone (including that +publisher). The workflow also re-peels the live tag before publication. Same-tag reruns reuse the artifacts and `latest.json` attached to a complete, published GitHub Release only when GitHub reports `immutable: true` and a canonical diff --git a/scripts/check-release-tag-protection.mjs b/scripts/check-release-tag-protection.mjs index ecc4fc7..d12ffc2 100644 --- a/scripts/check-release-tag-protection.mjs +++ b/scripts/check-release-tag-protection.mjs @@ -63,6 +63,47 @@ export function assertReleaseTagRuleset(rulesets) { return { id: matching.id, name: matching.name }; } +export function assertReleaseTagCreationRuleset(rulesets) { + if (!Array.isArray(rulesets)) { + throw new Error("GitHub tag rulesets response must be an array"); + } + const matching = rulesets.find((ruleset) => { + const includes = ruleset?.conditions?.ref_name?.include; + const excludes = ruleset?.conditions?.ref_name?.exclude; + const ruleTypes = new Set( + Array.isArray(ruleset?.rules) + ? ruleset.rules.map((rule) => rule?.type) + : [], + ); + // GitHub intentionally omits bypass_actors from read-only responses. When + // visible (local/admin verification), require exactly the configured + // Wangnov publisher identity and reject a broad role/team bypass. + const bypassActors = ruleset?.bypass_actors; + const visibleBypassIsAuthorized = + !Array.isArray(bypassActors) || + (bypassActors.length === 1 && + bypassActors[0]?.actor_type === "User" && + bypassActors[0]?.actor_id === 48670012 && + bypassActors[0]?.bypass_mode === "always"); + return ( + ruleset?.target === "tag" && + ruleset?.enforcement === "active" && + Array.isArray(includes) && + includes.includes(REQUIRED_REF_PATTERN) && + Array.isArray(excludes) && + excludes.length === 0 && + ruleTypes.has("creation") && + visibleBypassIsAuthorized + ); + }); + if (!matching) { + throw new Error( + `an active tag ruleset must restrict creation of ${REQUIRED_REF_PATTERN} to the authorized release publisher`, + ); + } + return { id: matching.id, name: matching.name }; +} + export function assertReleaseTagCommit(actualSha, expectedSha) { if (!SHA_PATTERN.test(expectedSha)) { throw new Error( @@ -134,6 +175,7 @@ export function verifyReleaseTagProtection({ ) .map((ruleset) => api(`repos/${repository}/rulesets/${ruleset.id}`)); const ruleset = assertReleaseTagRuleset(details); + const creationRuleset = assertReleaseTagCreationRuleset(details); let object = api(`repos/${repository}/git/ref/tags/${releaseTag}`).object; for (let depth = 0; object?.type === "tag" && depth < 5; depth += 1) { @@ -144,14 +186,14 @@ export function verifyReleaseTagProtection({ } const commit = assertReleaseTagCommit(object.sha, expectedSha); - return { commit, ruleset }; + return { commit, creationRuleset, ruleset }; } if (process.argv[1] === fileURLToPath(import.meta.url)) { try { const result = verifyReleaseTagProtection(); console.log( - `Release tag protection verified by ${result.ruleset.name || result.ruleset.id}: ${result.commit.sha}`, + `Release tag creation/immutability verified by ${result.creationRuleset.name || result.creationRuleset.id} and ${result.ruleset.name || result.ruleset.id}: ${result.commit.sha}`, ); } catch (error) { console.error( diff --git a/scripts/release-workflow.test.mjs b/scripts/release-workflow.test.mjs index fddc868..f006be0 100644 --- a/scripts/release-workflow.test.mjs +++ b/scripts/release-workflow.test.mjs @@ -10,6 +10,7 @@ import { verifyImmutableReleases, } from "./check-immutable-releases.mjs"; import { + assertReleaseTagCreationRuleset, assertReleaseTagCommit, assertReleaseTagRuleset, verifyReleaseTagProtection, @@ -247,6 +248,21 @@ describe("release workflow recovery invariants", () => { conditions: { ref_name: { include: ["refs/tags/v*"], exclude: [] } }, rules: [{ type: "update" }, { type: "deletion" }], }; + const creationRuleset = { + id: 2, + name: "authorized release tag creation", + target: "tag", + enforcement: "active", + bypass_actors: [ + { + actor_id: 48670012, + actor_type: "User", + bypass_mode: "always", + }, + ], + conditions: { ref_name: { include: ["refs/tags/v*"], exclude: [] } }, + rules: [{ type: "creation" }], + }; expect(assertReleaseTagRuleset([protectedRuleset])).toEqual({ id: 1, name: "immutable release tags", @@ -264,6 +280,24 @@ describe("release workflow recovery invariants", () => { }, ]), ).toThrow("visible bypass actors"); + expect(assertReleaseTagCreationRuleset([creationRuleset])).toEqual({ + id: 2, + name: "authorized release tag creation", + }); + expect(() => + assertReleaseTagCreationRuleset([ + { + ...creationRuleset, + bypass_actors: [ + { + actor_id: 5, + actor_type: "RepositoryRole", + bypass_mode: "always", + }, + ], + }, + ]), + ).toThrow("authorized release publisher"); const expectedSha = "a".repeat(40); expect(assertReleaseTagCommit(expectedSha, expectedSha)).toEqual({ @@ -276,9 +310,10 @@ describe("release workflow recovery invariants", () => { const responses = new Map([ [ "repos/owner/repo/rulesets?targets=tag&per_page=100", - [protectedRuleset], + [protectedRuleset, creationRuleset], ], ["repos/owner/repo/rulesets/1", protectedRuleset], + ["repos/owner/repo/rulesets/2", creationRuleset], [ "repos/owner/repo/git/ref/tags/v1.2.3", { object: { type: "tag", sha: "b".repeat(40) } }, @@ -303,6 +338,7 @@ describe("release workflow recovery invariants", () => { }), ).toEqual({ commit: { sha: expectedSha }, + creationRuleset: { id: 2, name: "authorized release tag creation" }, ruleset: { id: 1, name: "immutable release tags" }, }); From 244d14a8c8b799a3a2eeccef7346f884f03b328c Mon Sep 17 00:00:00 2001 From: wangnov <48670012+Wangnov@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:29:02 +0800 Subject: [PATCH 7/8] fix(release): run publication from trusted main --- .github/workflows/ci.yml | 3 +- .github/workflows/release-source.yml | 33 ++ .github/workflows/release.yml | 440 +++++++++++++++++++---- docs/release.md | 40 ++- docs/releases/TEMPLATE.md | 3 +- scripts/check-release-reuse.mjs | 6 +- scripts/check-release-source-ancestor.sh | 42 +++ scripts/check-release-tag-protection.mjs | 19 +- scripts/check-release-version.mjs | 4 +- scripts/mirror-release.test.mjs | 2 +- scripts/release-binding.mjs | 264 ++++++++++++++ scripts/release-workflow.test.mjs | 373 +++++++++++++++++-- 12 files changed, 1127 insertions(+), 102 deletions(-) create mode 100644 .github/workflows/release-source.yml create mode 100644 scripts/check-release-source-ancestor.sh create mode 100644 scripts/release-binding.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da040fe..faa91db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,8 @@ name: CI # Fast pull-request gate: type-check, unit tests and a cross-platform Rust # clippy/test pass. The heavy signed/notarized bundle build stays in -# release.yml (tags only) — this never builds or signs the .app. +# the default-branch release.yml (signalled by v* tags) — this never builds or +# signs the .app. on: pull_request: push: diff --git a/.github/workflows/release-source.yml b/.github/workflows/release-source.yml new file mode 100644 index 0000000..70c893f --- /dev/null +++ b/.github/workflows/release-source.yml @@ -0,0 +1,33 @@ +name: Release source + +# This workflow is only an unprivileged signal. Tag workflows are loaded from +# the tagged commit, so this file must never receive an environment, secrets, +# write permissions, checkout, or artifacts. The credentialed Release workflow +# is triggered with workflow_run and is therefore loaded from the default branch. +on: + push: + tags: ["v*"] + +permissions: {} + +jobs: + signal: + runs-on: ubuntu-latest + permissions: {} + steps: + - name: Validate release signal shape + shell: bash + env: + RELEASE_TAG: ${{ github.ref_name }} + RELEASE_SOURCE_SHA: ${{ github.sha }} + run: | + set -euo pipefail + if [[ ! "$RELEASE_TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then + echo "::error::release tag must be a semantic vX.Y.Z tag: $RELEASE_TAG" + exit 1 + fi + if [[ ! "$RELEASE_SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::release source must be a full commit SHA" + exit 1 + fi + echo "Unprivileged release signal accepted for $RELEASE_TAG at $RELEASE_SOURCE_SHA" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e1c51b8..1f3e8ec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,11 +1,11 @@ name: Release -# Tag-driven cross-platform release: +# Default-branch, tag-signalled cross-platform release: # build (unsigned) → macOS finalize (adaptive icon + inside-out Developer ID # sign + notarize + staple + repackage updater/dmg) → collect → publish a # GitHub Release with the Tauri updater manifest (latest.json). # -# Required repo secrets (Settings ▸ Secrets and variables ▸ Actions / release env): +# Required `release` environment secrets (never repository/org secrets): # APPLE_CERTIFICATE base64 of your "Developer ID Application" .p12 # APPLE_CERTIFICATE_PASSWORD password for that .p12 # APPLE_SIGNING_IDENTITY "Developer ID Application: NAME (TEAMID)" @@ -36,6 +36,9 @@ name: Release # release publisher and forbid every update/deletion; the same read-only token # verifies those policies and the live peeled SHA. # Every reused asset is also pinned to the API-provided sha256 digest. +# Configure the `release` environment for the protected default branch only; +# never allow v* tags. A tagged revision may change release-source.yml, but it +# cannot claim this environment and the signal itself has no credentials. # The old MANAGER_*_ACCESS_KEY_ID / SECRET_ACCESS_KEY names must stay deleted; # otherwise a historical workflow revision could still perform unconditional writes. # @@ -46,8 +49,12 @@ name: Release # vars.WINDOWS_TIMESTAMP_URL optional RFC3161 timestamp URL override on: - push: - tags: ["v*"] + # `workflow_run` always loads this credentialed workflow from the repository + # default branch. Never put credentials in release-source.yml: tag-triggered + # workflow definitions are loaded from the tagged commit. + workflow_run: + workflows: ["Release source"] + types: [completed] workflow_dispatch: inputs: target_tag: @@ -69,7 +76,11 @@ permissions: contents: read env: - RELEASE_TAG: ${{ inputs.target_tag || github.ref_name }} + AUTHORIZED_RELEASE_ACTOR_LOGIN: Wangnov + AUTHORIZED_RELEASE_ACTOR_ID: "48670012" + RELEASE_TAG: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch || inputs.target_tag }} + RELEASE_SOURCE_SHA: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || '' }} + TRUSTED_RELEASE_INVOCATION: ${{ github.event_name == 'workflow_run' && 'tag-signal' || 'manual-recovery' }} # Serialize every credentialed mirror writer through one release lane. This is a # correctness boundary for the unconditional IHEP follower, not an optimization. @@ -81,51 +92,107 @@ concurrency: queue: max jobs: - # A manual dispatch may execute only from the protected default branch and - # must name an existing target_tag. This also rejects an empty-input dispatch - # from an attacker-controlled branch named like a release tag. - # Keep the rejecting path free of checkout, environments, permissions, and - # secrets so an untrusted ref cannot reach any credential-bearing job. Compare - # the complete ref, not ref_name: a tag named like the default branch must fail. - reject_untrusted_target_dispatch: - if: ${{ github.event_name == 'workflow_dispatch' && github.ref != format('refs/heads/{0}', github.event.repository.default_branch) }} - runs-on: ubuntu-latest - permissions: {} - steps: - - name: Reject manual dispatch outside the default branch - env: - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - DISPATCH_REF: ${{ github.ref }} - run: | - echo "::error::target_tag must be dispatched from refs/heads/$DEFAULT_BRANCH, not $DISPATCH_REF" - exit 1 - + # This is the only path from the unprivileged tag signal to credentialed jobs. + # It deliberately has no checkout, environment, write permission, or secrets. + # Every field is copied through the environment so event data is never + # interpolated into shell source. preflight: - if: ${{ github.event_name != 'workflow_dispatch' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} runs-on: ubuntu-latest permissions: {} + outputs: + release_tag: ${{ steps.validate.outputs.release_tag }} + release_source_sha: ${{ steps.validate.outputs.release_source_sha }} + trusted_invocation: ${{ steps.validate.outputs.trusted_invocation }} steps: - name: Validate release invocation without secrets + id: validate shell: bash env: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + CURRENT_TRIGGERING_ACTOR_LOGIN: ${{ github.triggering_actor }} DISPATCH_REF: ${{ github.ref }} + DISPATCH_ACTOR_LOGIN: ${{ github.actor }} + DISPATCH_ACTOR_ID: ${{ github.actor_id }} + DISPATCH_TRIGGERING_ACTOR_LOGIN: ${{ github.triggering_actor }} EVENT_NAME: ${{ github.event_name }} REQUESTED_TARGET_TAG: ${{ inputs.target_tag || '' }} + UPSTREAM_ACTOR_LOGIN: ${{ github.event.workflow_run.actor.login || '' }} + UPSTREAM_ACTOR_ID: ${{ github.event.workflow_run.actor.id || '' }} + UPSTREAM_CONCLUSION: ${{ github.event.workflow_run.conclusion || '' }} + UPSTREAM_EVENT: ${{ github.event.workflow_run.event || '' }} + UPSTREAM_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch || '' }} + UPSTREAM_HEAD_REPOSITORY: ${{ github.event.workflow_run.head_repository.full_name || '' }} + UPSTREAM_HEAD_SHA: ${{ github.event.workflow_run.head_sha || '' }} + UPSTREAM_PATH: ${{ github.event.workflow_run.path || '' }} + UPSTREAM_REPOSITORY: ${{ github.event.workflow_run.repository.full_name || '' }} + UPSTREAM_TRIGGERING_ACTOR_LOGIN: ${{ github.event.workflow_run.triggering_actor.login || '' }} + UPSTREAM_TRIGGERING_ACTOR_ID: ${{ github.event.workflow_run.triggering_actor.id || '' }} + UPSTREAM_WORKFLOW_NAME: ${{ github.event.workflow_run.name || '' }} run: | set -euo pipefail - if [[ "$EVENT_NAME" == "workflow_dispatch" && "$DISPATCH_REF" != "refs/heads/$DEFAULT_BRANCH" ]]; then - echo "::error::workflow_dispatch must run from refs/heads/$DEFAULT_BRANCH, not $DISPATCH_REF" + [[ "$CURRENT_TRIGGERING_ACTOR_LOGIN" == "$AUTHORIZED_RELEASE_ACTOR_LOGIN" ]] || { + echo "::error::release workflow rerun actor is not the authorized publisher" exit 1 - fi - if [[ "$EVENT_NAME" == "workflow_dispatch" && -z "$REQUESTED_TARGET_TAG" ]]; then - echo "::error::workflow_dispatch requires a non-empty target_tag" + } + if [[ "$EVENT_NAME" == "workflow_run" ]]; then + [[ "$UPSTREAM_WORKFLOW_NAME" == "Release source" ]] || { + echo "::error::unexpected upstream workflow: $UPSTREAM_WORKFLOW_NAME" + exit 1 + } + [[ "$UPSTREAM_PATH" == ".github/workflows/release-source.yml" ]] || { + echo "::error::unexpected upstream workflow path: $UPSTREAM_PATH" + exit 1 + } + [[ "$UPSTREAM_EVENT" == "push" && "$UPSTREAM_CONCLUSION" == "success" ]] || { + echo "::error::release source must be a successful push workflow" + exit 1 + } + [[ "$UPSTREAM_REPOSITORY" == "$GITHUB_REPOSITORY" && "$UPSTREAM_HEAD_REPOSITORY" == "$GITHUB_REPOSITORY" ]] || { + echo "::error::release source must originate in $GITHUB_REPOSITORY" + exit 1 + } + [[ "$UPSTREAM_ACTOR_LOGIN" == "$AUTHORIZED_RELEASE_ACTOR_LOGIN" && "$UPSTREAM_ACTOR_ID" == "$AUTHORIZED_RELEASE_ACTOR_ID" ]] || { + echo "::error::release source actor is not the authorized publisher" + exit 1 + } + [[ "$UPSTREAM_TRIGGERING_ACTOR_LOGIN" == "$AUTHORIZED_RELEASE_ACTOR_LOGIN" && "$UPSTREAM_TRIGGERING_ACTOR_ID" == "$AUTHORIZED_RELEASE_ACTOR_ID" ]] || { + echo "::error::release source rerun actor is not the authorized publisher" + exit 1 + } + [[ "$UPSTREAM_HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || { + echo "::error::release source head_sha must be a full commit SHA" + exit 1 + } + RELEASE_TAG="$UPSTREAM_HEAD_BRANCH" + RELEASE_SOURCE_SHA="$UPSTREAM_HEAD_SHA" + TRUSTED_RELEASE_INVOCATION="tag-signal" + elif [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then + [[ "$DISPATCH_REF" == "refs/heads/$DEFAULT_BRANCH" ]] || { + echo "::error::workflow_dispatch must run from refs/heads/$DEFAULT_BRANCH, not $DISPATCH_REF" + exit 1 + } + [[ "$DISPATCH_ACTOR_LOGIN" == "$AUTHORIZED_RELEASE_ACTOR_LOGIN" && "$DISPATCH_ACTOR_ID" == "$AUTHORIZED_RELEASE_ACTOR_ID" && "$DISPATCH_TRIGGERING_ACTOR_LOGIN" == "$AUTHORIZED_RELEASE_ACTOR_LOGIN" ]] || { + echo "::error::workflow_dispatch actor is not the authorized publisher" + exit 1 + } + [[ -n "$REQUESTED_TARGET_TAG" ]] || { + echo "::error::workflow_dispatch requires a non-empty target_tag" + exit 1 + } + RELEASE_TAG="$REQUESTED_TARGET_TAG" + RELEASE_SOURCE_SHA="" + TRUSTED_RELEASE_INVOCATION="manual-recovery" + else + echo "::error::unsupported release event: $EVENT_NAME" exit 1 fi if [[ ! "$RELEASE_TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then echo "::error::release tag must be a semantic vX.Y.Z tag: $RELEASE_TAG" exit 1 fi + printf 'release_tag=%s\n' "$RELEASE_TAG" >> "$GITHUB_OUTPUT" + printf 'release_source_sha=%s\n' "$RELEASE_SOURCE_SHA" >> "$GITHUB_OUTPUT" + printf 'trusted_invocation=%s\n' "$TRUSTED_RELEASE_INVOCATION" >> "$GITHUB_OUTPUT" prepare: needs: preflight @@ -137,7 +204,19 @@ jobs: outputs: release_reusable: ${{ steps.lookup.outputs.release_reusable }} release_asset_digests: ${{ steps.lookup.outputs.release_asset_digests }} + release_source_sha: ${{ steps.tag_source.outputs.release_source_sha }} steps: + - name: Authorize credentialed job rerun actor + shell: bash + env: + CURRENT_TRIGGERING_ACTOR_LOGIN: ${{ github.triggering_actor }} + run: | + set -euo pipefail + [[ "$CURRENT_TRIGGERING_ACTOR_LOGIN" == "$AUTHORIZED_RELEASE_ACTOR_LOGIN" ]] || { + echo "::error::credentialed release job rerun actor is not the authorized publisher" + exit 1 + } + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -146,31 +225,46 @@ jobs: with: node-version: 20 - # Verify the release source rather than the workflow source. For a - # target_tag recovery the workflow itself comes from the protected default - # branch, while the six application version declarations come from the tag. + # Resolve the live tag through the API before any tag checkout. A + # workflow_run must still point at the exact SHA carried by the successful + # signal; an authorized manual recovery may resolve the immutable tag here. + - name: Resolve protected immutable release tag + id: tag_source + env: + ALLOW_RELEASE_SHA_RESOLUTION: ${{ needs.preflight.outputs.trusted_invocation == 'manual-recovery' && '1' || '0' }} + GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }} + RELEASE_SOURCE_SHA: ${{ needs.preflight.outputs.release_source_sha }} + run: node scripts/check-release-tag-protection.mjs "$RELEASE_TAG" "$RELEASE_SOURCE_SHA" + + # The root checkout contains only the trusted default-branch workflow and + # release scripts. Check out application source separately by the already + # verified commit SHA, never by a mutable tag name. - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - ref: ${{ env.RELEASE_TAG }} + ref: ${{ steps.tag_source.outputs.release_source_sha }} path: release-source persist-credentials: false + fetch-depth: 0 - name: Match release tag to every application version declaration shell: bash + env: + EXPECTED_RELEASE_SOURCE_SHA: ${{ steps.tag_source.outputs.release_source_sha }} run: | set -euo pipefail node scripts/check-release-version.mjs source "$RELEASE_TAG" release-source release_source_sha="$(git -C release-source rev-parse HEAD)" - if [[ "$GITHUB_EVENT_NAME" == "push" && "$release_source_sha" != "$GITHUB_SHA" ]]; then - echo "::error::release tag $RELEASE_TAG moved away from triggering commit $GITHUB_SHA" + if [[ "$release_source_sha" != "$EXPECTED_RELEASE_SOURCE_SHA" ]]; then + echo "::error::release checkout differs from verified commit $EXPECTED_RELEASE_SOURCE_SHA" exit 1 fi echo "RELEASE_SOURCE_SHA=$release_source_sha" >> "$GITHUB_ENV" - - name: Require protected immutable release tag + - name: Require release source merged into live default branch env: - GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }} - run: node scripts/check-release-tag-protection.mjs "$RELEASE_TAG" "$RELEASE_SOURCE_SHA" + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + VERIFIED_RELEASE_SOURCE_SHA: ${{ steps.tag_source.outputs.release_source_sha }} + run: bash scripts/check-release-source-ancestor.sh release-source "$VERIFIED_RELEASE_SOURCE_SHA" "$DEFAULT_BRANCH" - name: Require GitHub Immutable Releases env: @@ -207,13 +301,18 @@ jobs: fi build: - needs: prepare - if: ${{ needs.prepare.result == 'success' && needs.prepare.outputs.release_reusable != 'true' && github.ref_type == 'tag' && startsWith(github.ref_name, 'v') && !(github.event_name == 'workflow_dispatch' && inputs.target_tag != '') }} - # Repo admins should configure the release environment and copy signing - # secrets there before moving them off repo-level Actions secrets. + needs: [preflight, prepare] + if: ${{ needs.prepare.result == 'success' && needs.prepare.outputs.release_reusable != 'true' && needs.preflight.outputs.trusted_invocation == 'tag-signal' }} + # All signing secrets live only in the default-branch-restricted release + # environment. Never duplicate them as repository or organization secrets. environment: release + env: + RELEASE_SOURCE_SHA: ${{ needs.prepare.outputs.release_source_sha }} permissions: contents: read + defaults: + run: + working-directory: release-source strategy: fail-fast: false matrix: @@ -234,14 +333,49 @@ jobs: } runs-on: ${{ matrix.platform }} steps: + - name: Authorize credentialed job rerun actor + shell: bash + env: + CURRENT_TRIGGERING_ACTOR_LOGIN: ${{ github.triggering_actor }} + run: | + set -euo pipefail + [[ "$CURRENT_TRIGGERING_ACTOR_LOGIN" == "$AUTHORIZED_RELEASE_ACTOR_LOGIN" ]] || { + echo "::error::credentialed release job rerun actor is not the authorized publisher" + exit 1 + } + + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: + ref: ${{ needs.prepare.outputs.release_source_sha }} + path: release-source persist-credentials: false + fetch-depth: 0 + + # A successful prepare may belong to an earlier attempt. Re-fetch the live + # default branch before executing application-controlled build code or + # exposing signing credentials to later steps. + - name: Re-check release source before build + shell: bash + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + [[ "$(git rev-parse HEAD)" == "$RELEASE_SOURCE_SHA" ]] || { + echo "::error::release checkout differs from verified commit $RELEASE_SOURCE_SHA" + exit 1 + } + node ../scripts/check-release-version.mjs source "$RELEASE_TAG" . + bash ../scripts/check-release-source-ancestor.sh . "$RELEASE_SOURCE_SHA" "$DEFAULT_BRANCH" - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: 20 cache: npm + cache-dependency-path: release-source/package-lock.json - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -249,7 +383,7 @@ jobs: - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: - workspaces: src-tauri -> target + workspaces: release-source/src-tauri -> target - run: npm ci @@ -532,11 +666,14 @@ jobs: echo "Canonical artifacts: $artifact_names" release: - if: ${{ always() && needs.prepare.result == 'success' && needs.select_artifacts.result == 'success' && startsWith(inputs.target_tag || github.ref_name, 'v') }} - needs: [prepare, build, select_artifacts] - # Repo admins should configure the release environment and copy mirror - # secrets there before moving them off repo-level Actions secrets. + if: ${{ always() && needs.preflight.result == 'success' && needs.prepare.result == 'success' && needs.select_artifacts.result == 'success' }} + needs: [preflight, prepare, build, select_artifacts] + # All mirror credentials live only in the default-branch-restricted release + # environment. Never duplicate them as repository or organization secrets. environment: release + env: + TRUSTED_WORKFLOW_SHA: ${{ github.workflow_sha }} + TRUSTED_WORKFLOW_SOURCE_SHA: ${{ github.sha }} permissions: contents: write actions: write # dispatch winget.yml after publishing (release events from GITHUB_TOKEN don't cascade) @@ -544,6 +681,17 @@ jobs: attestations: write runs-on: ubuntu-latest steps: + - name: Authorize credentialed job rerun actor + shell: bash + env: + CURRENT_TRIGGERING_ACTOR_LOGIN: ${{ github.triggering_actor }} + run: | + set -euo pipefail + [[ "$CURRENT_TRIGGERING_ACTOR_LOGIN" == "$AUTHORIZED_RELEASE_ACTOR_LOGIN" ]] || { + echo "::error::credentialed release job rerun actor is not the authorized publisher" + exit 1 + } + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -569,25 +717,41 @@ jobs: with: node-version: 20 + # Re-resolve tag policy on every release-job attempt. For manual recovery + # this independently resolves the immutable tag; a signal run must still + # match the SHA carried by the original successful push workflow. + - name: Re-resolve protected immutable release tag + id: tag_source + env: + ALLOW_RELEASE_SHA_RESOLUTION: ${{ needs.preflight.outputs.trusted_invocation == 'manual-recovery' && '1' || '0' }} + GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }} + RELEASE_SOURCE_SHA: ${{ needs.preflight.outputs.release_source_sha }} + run: node scripts/check-release-tag-protection.mjs "$RELEASE_TAG" "$RELEASE_SOURCE_SHA" + # Re-run this gate inside every release-job attempt. GitHub's "Re-run # failed jobs" does not re-run prepare, and no immutable publication may - # rely only on a previous attempt's version decision. + # rely only on a previous attempt's source or ancestry decision. - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - ref: ${{ env.RELEASE_TAG }} + ref: ${{ steps.tag_source.outputs.release_source_sha }} path: release-source persist-credentials: false + fetch-depth: 0 - name: Re-check release tag against application versions shell: bash + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + EXPECTED_RELEASE_SOURCE_SHA: ${{ steps.tag_source.outputs.release_source_sha }} run: | set -euo pipefail node scripts/check-release-version.mjs source "$RELEASE_TAG" release-source release_source_sha="$(git -C release-source rev-parse HEAD)" - if [[ "$GITHUB_EVENT_NAME" == "push" && "$release_source_sha" != "$GITHUB_SHA" ]]; then - echo "::error::release tag $RELEASE_TAG moved away from triggering commit $GITHUB_SHA" + if [[ "$release_source_sha" != "$EXPECTED_RELEASE_SOURCE_SHA" ]]; then + echo "::error::release checkout differs from verified commit $EXPECTED_RELEASE_SOURCE_SHA" exit 1 fi + bash scripts/check-release-source-ancestor.sh release-source "$release_source_sha" "$DEFAULT_BRANCH" echo "RELEASE_SOURCE_SHA=$release_source_sha" >> "$GITHUB_ENV" # Re-check on every release-job attempt. `prepare` is not rerun by @@ -642,7 +806,7 @@ jobs: gh api --method GET \ -H "Accept: application/vnd.github.raw+json" \ "repos/$GITHUB_REPOSITORY/contents/src-tauri/tauri.conf.json" \ - -f ref="$RELEASE_TAG" >"$release_config" + -f ref="$RELEASE_SOURCE_SHA" >"$release_config" if ! updater_public_key="$( jq -er ' .plugins.updater.pubkey @@ -666,6 +830,7 @@ jobs: run: | set -euo pipefail rm -rf dist + rm -f release-binding.json mkdir -p dist jq -er '.[]' <<<"$ARTIFACT_NAMES_JSON" | while IFS= read -r artifact; do gh run download "$GITHUB_RUN_ID" \ @@ -693,7 +858,8 @@ jobs: --repo "$GITHUB_REPOSITORY" \ --dir "$published" \ --pattern 'CodexAppManager*' \ - --pattern 'latest.json' + --pattern 'latest.json' \ + --pattern 'release-binding.json' test -f "$published/latest.json" || { echo "::error::published $RELEASE_TAG has no latest.json" exit 1 @@ -717,6 +883,8 @@ jobs: done < <(jq -r 'to_entries[] | [.key, .value] | @tsv' <<<"$RELEASE_ASSET_DIGESTS") cp "$published/latest.json" "$RUNNER_TEMP/published-latest.json" rm "$published/latest.json" + cp "$published/release-binding.json" release-binding.json + rm "$published/release-binding.json" rm -rf dist mv "$published" dist echo "existing=true" >> "$GITHUB_OUTPUT" @@ -801,6 +969,37 @@ jobs: dist \ "$RELEASE_TAURI_CONFIG" + # A workflow_run attestation identifies the trusted default-branch + # release.yml, not the tag ref. Bind the separately verified tag commit to + # the exact subject digests in a custom, immutable predicate. + - name: Resolve signed release binding + id: binding + shell: bash + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + shopt -s nullglob + subjects=(dist/* latest.json) + if [[ "${{ steps.release_source.outputs.existing }}" != "true" ]]; then + node scripts/release-binding.mjs create \ + release-binding.json \ + "$RELEASE_TAG" \ + "$RELEASE_SOURCE_SHA" \ + "$GITHUB_REPOSITORY" \ + "$DEFAULT_BRANCH" \ + "$TRUSTED_WORKFLOW_SHA" \ + "$TRUSTED_WORKFLOW_SOURCE_SHA" \ + "${subjects[@]}" + fi + node scripts/release-binding.mjs verify \ + release-binding.json \ + "$RELEASE_TAG" \ + "$RELEASE_SOURCE_SHA" \ + "$GITHUB_REPOSITORY" \ + "$DEFAULT_BRANCH" \ + "${subjects[@]}" + # Create provenance only for bytes produced by this release run, before a # draft can become immutable. A later failed-job rerun must never mint a # new statement that falsely attributes already-published bytes to the @@ -814,27 +1013,90 @@ jobs: dist/* latest.json + - name: Attest fresh release source binding + id: attest_binding + if: ${{ steps.release_source.outputs.existing != 'true' }} + uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + with: + subject-path: | + dist/* + latest.json + predicate-type: https://codexapp.agentsmirror.com/attestations/release-binding/v1 + predicate-path: release-binding.json + + # Verify the just-created bundle before any draft upload. The standard + # workflow_run identity is the trusted main workflow/source; the custom + # predicate and exact subject set provide the independent tag binding. + - name: Verify fresh release source binding + id: verify_fresh_binding + if: ${{ steps.release_source.outputs.existing != 'true' }} + shell: bash + env: + ATTESTATION_BUNDLE: ${{ steps.attest_binding.outputs.bundle-path }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + signer_workflow="$GITHUB_REPOSITORY/.github/workflows/release.yml" + verification="$RUNNER_TEMP/fresh-release-binding-verification.json" + gh attestation verify latest.json \ + --bundle "$ATTESTATION_BUNDLE" \ + --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$signer_workflow" \ + --signer-digest "$TRUSTED_WORKFLOW_SHA" \ + --source-ref "refs/heads/$DEFAULT_BRANCH" \ + --source-digest "$TRUSTED_WORKFLOW_SOURCE_SHA" \ + --predicate-type "https://codexapp.agentsmirror.com/attestations/release-binding/v1" \ + --deny-self-hosted-runners \ + --format json > "$verification" + node scripts/release-binding.mjs attestation "$verification" release-binding.json + # Existing immutable releases are accepted only when their downloaded, # API-digest-pinned bytes already carry provenance from this repository's - # release workflow and exact tag ref. This is verification, not a new OIDC - # statement; missing historical provenance fails closed. + # trusted default-branch release workflow revision. The custom predicate + # separately binds the immutable tag and peeled source SHA to every digest. + # This is verification, not a new OIDC statement; missing historical + # provenance or binding fails closed. - name: Verify existing immutable release provenance id: verify_existing_provenance if: ${{ steps.release_source.outputs.existing == 'true' }} shell: bash env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} GH_TOKEN: ${{ github.token }} + RELEASE_SIGNER_SHA: ${{ steps.binding.outputs.signer_sha }} + RELEASE_WORKFLOW_SOURCE_SHA: ${{ steps.binding.outputs.source_sha }} run: | set -euo pipefail signer_workflow="$GITHUB_REPOSITORY/.github/workflows/release.yml" + predicate_type="https://codexapp.agentsmirror.com/attestations/release-binding/v1" + gh release verify "$RELEASE_TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --format json > "$RUNNER_TEMP/immutable-release-attestation.json" + for file in dist/* latest.json release-binding.json; do + gh release verify-asset "$RELEASE_TAG" "$file" \ + --repo "$GITHUB_REPOSITORY" >/dev/null + done for file in dist/* latest.json; do gh attestation verify "$file" \ --repo "$GITHUB_REPOSITORY" \ --signer-workflow "$signer_workflow" \ - --source-ref "refs/tags/$RELEASE_TAG" \ - --source-digest "$RELEASE_SOURCE_SHA" \ + --signer-digest "$RELEASE_SIGNER_SHA" \ + --source-ref "refs/heads/$DEFAULT_BRANCH" \ + --source-digest "$RELEASE_WORKFLOW_SOURCE_SHA" \ --deny-self-hosted-runners >/dev/null - echo "Verified existing provenance: $(basename "$file")" + custom_json="$RUNNER_TEMP/release-binding-$(basename "$file").json" + gh attestation verify "$file" \ + --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$signer_workflow" \ + --signer-digest "$RELEASE_SIGNER_SHA" \ + --source-ref "refs/heads/$DEFAULT_BRANCH" \ + --source-digest "$RELEASE_WORKFLOW_SOURCE_SHA" \ + --predicate-type "$predicate_type" \ + --deny-self-hosted-runners \ + --format json > "$custom_json" + node scripts/release-binding.mjs attestation "$custom_json" release-binding.json + echo "Verified existing provenance and release binding: $(basename "$file")" done - name: Resolve provenance gate @@ -842,16 +1104,22 @@ jobs: shell: bash env: FRESH_ATTESTATION_OUTCOME: ${{ steps.attest_fresh.outcome }} + FRESH_BINDING_ATTESTATION_OUTCOME: ${{ steps.attest_binding.outcome }} + FRESH_BINDING_VERIFICATION_OUTCOME: ${{ steps.verify_fresh_binding.outcome }} EXISTING_PROVENANCE_OUTCOME: ${{ steps.verify_existing_provenance.outcome }} RELEASE_REUSED: ${{ steps.release_source.outputs.existing }} run: | set -euo pipefail if [[ "$RELEASE_REUSED" == "true" ]]; then [[ "$FRESH_ATTESTATION_OUTCOME" == "skipped" ]] + [[ "$FRESH_BINDING_ATTESTATION_OUTCOME" == "skipped" ]] + [[ "$FRESH_BINDING_VERIFICATION_OUTCOME" == "skipped" ]] [[ "$EXISTING_PROVENANCE_OUTCOME" == "success" ]] echo "mode=verified-existing-immutable-release" >> "$GITHUB_OUTPUT" else [[ "$FRESH_ATTESTATION_OUTCOME" == "success" ]] + [[ "$FRESH_BINDING_ATTESTATION_OUTCOME" == "success" ]] + [[ "$FRESH_BINDING_VERIFICATION_OUTCOME" == "success" ]] [[ "$EXISTING_PROVENANCE_OUTCOME" == "skipped" ]] echo "mode=fresh-build-attested-before-publication" >> "$GITHUB_OUTPUT" fi @@ -925,7 +1193,7 @@ jobs: # empty. GitHub's auto "What's Changed" section is appended either way. - name: Prepare release notes run: | - NOTES="docs/releases/$RELEASE_TAG.md" + NOTES="release-source/docs/releases/$RELEASE_TAG.md" if [ -f "$NOTES" ]; then cp "$NOTES" "$RUNNER_TEMP/release-notes.md" else @@ -943,6 +1211,7 @@ jobs: if [ ! -f dist/latest.json ]; then sha256sum latest.json >> SHA256SUMS fi + sha256sum release-binding.json >> SHA256SUMS cat SHA256SUMS - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @@ -956,7 +1225,7 @@ jobs: status=0 : > release-assets/sbom-status.txt - if npm sbom --package-lock-only --sbom-format cyclonedx > release-assets/sbom-npm.cdx.json; then + if (cd release-source && npm sbom --package-lock-only --sbom-format cyclonedx) > release-assets/sbom-npm.cdx.json; then echo "npm SBOM: generated" >> release-assets/sbom-status.txt else echo "::warning::npm SBOM generation failed" @@ -965,7 +1234,7 @@ jobs: status=1 fi - if cargo metadata --manifest-path src-tauri/Cargo.toml --format-version=1 > "$RUNNER_TEMP/cargo-metadata.json" \ + if cargo metadata --manifest-path release-source/src-tauri/Cargo.toml --format-version=1 > "$RUNNER_TEMP/cargo-metadata.json" \ && node scripts/cargo-metadata-to-cyclonedx.mjs "$RUNNER_TEMP/cargo-metadata.json" release-assets/sbom-cargo.cdx.json; then echo "cargo SBOM: generated" >> release-assets/sbom-status.txt else @@ -980,11 +1249,14 @@ jobs: # The repository-level ruleset closes the long validation-to-publication # window by forbidding v* tag updates/deletions. Re-read both the policy and # the peeled commit immediately before creating the canonical draft. - - name: Re-check protected release tag before draft upload + - name: Re-check protected release source before draft upload if: ${{ steps.provenance.outputs.ready == 'true' && steps.release_source.outputs.existing != 'true' }} env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }} - run: node scripts/check-release-tag-protection.mjs "$RELEASE_TAG" "$RELEASE_SOURCE_SHA" + run: | + bash scripts/check-release-source-ancestor.sh release-source "$RELEASE_SOURCE_SHA" "$DEFAULT_BRANCH" + node scripts/check-release-tag-protection.mjs "$RELEASE_TAG" "$RELEASE_SOURCE_SHA" # Explicit draft-first publication keeps asset upload ahead of publication # for both stable and prerelease tags. This is required when GitHub immutable @@ -1006,16 +1278,20 @@ jobs: files: | dist/* latest.json + release-binding.json SHA256SUMS release-assets/* # Upload can take long enough for a repository policy regression to matter. # Fence the transition from mutable draft to immutable publication again. - - name: Re-check protected release tag before publication + - name: Re-check protected release source before publication if: ${{ steps.provenance.outputs.ready == 'true' && steps.release_source.outputs.existing != 'true' }} env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }} - run: node scripts/check-release-tag-protection.mjs "$RELEASE_TAG" "$RELEASE_SOURCE_SHA" + run: | + bash scripts/check-release-source-ancestor.sh release-source "$RELEASE_SOURCE_SHA" "$DEFAULT_BRANCH" + node scripts/check-release-tag-protection.mjs "$RELEASE_TAG" "$RELEASE_SOURCE_SHA" # Omitting `draft` tells action-gh-release v3 to publish the existing draft # after all uploads succeeded while retaining its stable/prerelease metadata. @@ -1054,13 +1330,30 @@ jobs: exit 1 fi cat "$verification_log" + release_attestation="$RUNNER_TEMP/immutable-release-attestation.json" + release_attestation_verified=false + for attempt in 1 2 3 4 5; do + if gh release verify "$RELEASE_TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --format json > "$release_attestation"; then + release_attestation_verified=true + break + fi + sleep 3 + done + if [[ "$release_attestation_verified" != "true" ]]; then + echo "::error::published $RELEASE_TAG has no valid GitHub immutable release attestation" + exit 1 + fi release_asset_digests="$(sed -n 's/^release_asset_digests=//p' "$verification_output")" jq -e 'type == "object" and length > 0' <<<"$release_asset_digests" >/dev/null || { echo "::error::published immutable digest evidence is missing" exit 1 } - for file in dist/* latest.json; do + for file in dist/* latest.json release-binding.json; do name="$(basename "$file")" + gh release verify-asset "$RELEASE_TAG" "$file" \ + --repo "$GITHUB_REPOSITORY" >/dev/null expected_digest="$(jq -er --arg name "$name" '.[$name]' <<<"$release_asset_digests")" actual_digest="sha256:$(sha256sum "$file" | cut -d' ' -f1)" if [[ "$actual_digest" != "$expected_digest" ]]; then @@ -1070,6 +1363,17 @@ jobs: echo "Verified published attested digest: $name" done + # Existing immutable releases skip draft publication, so fence the final + # credentialed mirror promotion for both fresh and recovery paths. + - name: Re-check protected release source before final promotion + if: ${{ success() && steps.provenance.outputs.ready == 'true' && !contains(env.RELEASE_TAG, '-') }} + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_TOKEN: ${{ secrets.IMMUTABLE_RELEASES_READ_TOKEN }} + run: | + bash scripts/check-release-source-ancestor.sh release-source "$RELEASE_SOURCE_SHA" "$DEFAULT_BRANCH" + node scripts/check-release-tag-protection.mjs "$RELEASE_TAG" "$RELEASE_SOURCE_SHA" + - name: Promote CDN mirror latest (R2 + IHEP S3) id: mirror_promote if: ${{ success() && steps.provenance.outputs.ready == 'true' && !contains(env.RELEASE_TAG, '-') }} @@ -1123,6 +1427,7 @@ jobs: WINGET_OUTCOME: ${{ steps.winget.outputs.status || steps.winget.outcome }} SBOM_OUTCOME: ${{ steps.sbom.outcome }} ATTESTATION_OUTCOME: ${{ steps.attest_fresh.outcome }} + BINDING_ATTESTATION_OUTCOME: ${{ steps.attest_binding.outcome }} PROVENANCE_MODE: ${{ steps.provenance.outputs.mode }} run: node scripts/write-release-summary.mjs @@ -1138,5 +1443,6 @@ jobs: mirror-verification-summary.json mirror-promotion-summary.json SHA256SUMS + release-binding.json if-no-files-found: warn retention-days: 90 diff --git a/docs/release.md b/docs/release.md index 228994c..c288c7b 100644 --- a/docs/release.md +++ b/docs/release.md @@ -3,13 +3,18 @@ ## Release notes 每个版本的 release note 写在 `docs/releases/v.md`,随版本号 bump 一起进发版 PR; -tag 推送后 `release.yml` 按 tag 名取用该文件作为 GitHub Release 正文,并自动追加 +tag 推送后无权限的 `release-source.yml` 发出信号,默认分支的 `release.yml` 按 tag +名取用该文件作为 GitHub Release 正文,并自动追加 "What's Changed" 与 Full Changelog。文件缺失时回退到 `docs/releases/FALLBACK.md` (安装表 + 升级说明),正文永远不会为空。写法与双语风格见 `docs/releases/TEMPLATE.md`。 -Cross-platform release is tag-driven via [`.github/workflows/release.yml`](../.github/workflows/release.yml). -Push a `v*` tag and CI builds, signs, notarizes, and publishes a GitHub Release -with the Tauri updater manifest. +Cross-platform release starts with the unprivileged +[`release-source.yml`](../.github/workflows/release-source.yml) tag signal. Its +successful `workflow_run` invokes the current default-branch +[`release.yml`](../.github/workflows/release.yml), which builds, signs, +notarizes, and publishes the GitHub Release with the Tauri updater manifest. +The signal has no checkout, secrets, environment, write permission, or artifact +handoff; all credentialed jobs remain in the trusted default-branch workflow. ## macOS signing pipeline @@ -133,7 +138,19 @@ dedicated fine-grained, read-only token. A missing token, failed query, or `enabled: false` response fails closed before draft upload or mirror publication. Two separate active tag rulesets restrict `refs/tags/v*` creation to the named release publisher and forbid update/deletion for everyone (including that -publisher). The workflow also re-peels the live tag before publication. +publisher). The workflow also re-peels the live tag before publication. The +peeled commit must be an ancestor of the freshly fetched live default branch; +tagging an unmerged release-bump branch fails before build/signing and is checked +again in each build, release-job attempt, publication boundary, and final mirror +promotion. Every checkout after resolution uses that exact commit SHA, not the +tag name. + +The `release` environment must allow the protected default branch only. Do not +allow `v*` deployment refs and do not duplicate release credentials as repository +or organization secrets: tag-triggered workflow definitions are read from the +tagged commit, whereas the credentialed `workflow_run` workflow is read from the +default branch. `release-source.yml` must stay an unprivileged signal and must +never download or pass artifacts. Same-tag reruns reuse the artifacts and `latest.json` attached to a complete, published GitHub Release only when GitHub reports `immutable: true` and a canonical @@ -151,7 +168,18 @@ an immutable version key. New releases are uploaded as drafts first and publishe only after all assets succeed, including prereleases. Immediately after publish, the workflow requires the Release itself to report `immutable: true` and canonical digests for every required asset before any mirror pointer can advance. A -historical Actions run still executes its historical workflow revision, so the +Each fresh release also publishes `release-binding.json` and a custom GitHub +attestation. `workflow_run` OIDC identifies `refs/heads/` rather than the +tag, so verification separately pins the trusted signer workflow digest and the +default-branch source digest, then requires the signed predicate and attestation +subject set to match the target tag, peeled source SHA, and every canonical +subject digest exactly. Fresh bundles are self-verified before draft upload. +Immutable reuse downloads the API-digest-pinned binding and verifies both SLSA +provenance and this exact custom predicate. Both fresh publication and reuse also +require `gh release verify` to accept GitHub's immutable release attestation; an +old immutable release without the binding is deliberately not reusable. + +Historical Actions runs execute their historical workflow revision, so the release environment intentionally uses new `*_PROMOTION_*` credential names. The four legacy access-key secrets listed below must be deleted; the current workflow has a hard gate that rejects them if they are reintroduced. diff --git a/docs/releases/TEMPLATE.md b/docs/releases/TEMPLATE.md index 8c81d84..90cc7ce 100644 --- a/docs/releases/TEMPLATE.md +++ b/docs/releases/TEMPLATE.md @@ -1,6 +1,7 @@