From 0fc251fd2c5e6888e3c3fc231a07b5c3ed8c371d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 29 Jul 2026 17:17:16 -0700 Subject: [PATCH 1/2] fix(ci): scope the Next.js build cache sticky disk per branch The Turbopack persistent build cache disk was keyed on github.event_name alone, so every open PR shared one mutable volume. A sticky disk mount clones the last committed snapshot and commits back last-write-wins, so each PR build restored a cache produced by a different branch. Measured on the same staging commit, two runs minutes apart: the single-writer push disk compiled in 9.3 min, the shared pull_request disk in 14.0 min. Across 22 recent runs, push builds land at 3.5-11 min and PR builds at 13.5-17.8 min. Correctness matters more than the minutes here. turbopackFileSystemCacheForBuild is beta and cross-commit restore is not a documented-supported mode - vercel/next.js#87283 reports stale HTML from a cache built at another commit, with no maintainer answer. Every other cache layer in this repo already isolates by branch: GitHub's cache cannot read sibling branches, and Blacksmith's cache product branch-scopes by default. Only this key didn't. Adds pre/post-build cache size reporting, because whether the cache was warm is otherwise invisible - the disk mounts either way and turbo buffers the build log. Adds ci-cache-cleanup.yml to delete a branch's disk when its PR closes, so per-branch disks track open PRs instead of accumulating at ~5 GB each. Also corrects the 105s-cold/22s-warm comment: those were local numbers and have never reproduced in CI. --- .github/workflows/ci-cache-cleanup.yml | 38 ++++++++++++++++++ .github/workflows/test-build.yml | 55 +++++++++++++++++++++++--- 2 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/ci-cache-cleanup.yml diff --git a/.github/workflows/ci-cache-cleanup.yml b/.github/workflows/ci-cache-cleanup.yml new file mode 100644 index 00000000000..bf1e2f2890f --- /dev/null +++ b/.github/workflows/ci-cache-cleanup.yml @@ -0,0 +1,38 @@ +name: CI Cache Cleanup + +# The Next.js build cache sticky disk in test-build.yml is keyed per branch, so +# every PR leaves a ~5 GB volume behind. Branches are short-lived; the disks +# aren't. Delete a branch's disk when its PR closes so storage tracks open PRs +# rather than every branch the repo has ever seen. +# +# Only the pull_request-event disk is deleted. The push-event disks belong to +# main/staging/dev and must stay warm. + +on: + pull_request: + types: [closed] + +concurrency: + group: ci-cache-cleanup-${{ github.head_ref }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + delete-nextjs-cache: + name: Delete Next.js build cache disk + # Sticky disks only exist on Blacksmith; in GitHub break-glass mode the + # cache-mount fallback uses actions/cache, which expires on its own. + if: vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith' + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + + steps: + # Key must stay byte-identical to the Mount Next.js build cache key in + # test-build.yml, or this silently deletes nothing and the disks pile up. + - name: Delete sticky disk + uses: useblacksmith/stickydisk-delete@b41313d28b8647d72114c9ba3c96bb04061562b6 # v1 + continue-on-error: true + with: + delete-key: ${{ github.repository }}-nextjs-cache-pull_request${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-${{ github.head_ref }} diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index c50e076a317..8bbd775db73 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -265,14 +265,31 @@ jobs: # Turbopack's persistent build cache (NEXT_TURBOPACK_BUILD_CACHE below) # writes ~5 GB into .next/cache — a sticky disk mounts it in ~1s where an - # actions/cache round-trip would eat the warm-build win. Same event/fork - # namespacing as the other mounts; the GitHub fallback inside cache-mount - # still uses actions/cache with a run_id-suffixed key. + # actions/cache round-trip would eat the warm-build win. + # + # Scoped per branch, not just per event. A sticky disk is a single mutable + # volume per key: mounting clones the last committed snapshot and the job + # commits back last-write-wins. Keyed on event alone, every open PR shared + # one volume, so each run cloned a snapshot built from a *different* + # branch. Measured on the same staging commit, minutes apart: the push + # disk (single-writer, genuinely warm) compiled in 9.3 min while the + # shared pull_request disk took 14.0 min. + # + # Branch scoping is also the correctness-preserving choice — + # turbopackFileSystemCacheForBuild is beta, and cross-commit restore is + # not a documented-supported mode (vercel/next.js#87283 reports stale HTML + # from a cache built at another commit). Every other cache layer here + # already isolates by branch: GitHub's cache can't read sibling branches, + # and Blacksmith's own cache product branch-scopes by default. Only this + # key didn't. + # + # Cost of per-branch disks is bounded by ci-cache-cleanup.yml, which + # deletes a branch's disk when its PR closes. - name: Mount Next.js build cache uses: ./.github/actions/cache-mount with: provider: ${{ vars.CI_PROVIDER }} - key: ${{ github.repository }}-nextjs-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + key: ${{ github.repository }}-nextjs-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-${{ github.head_ref || github.ref_name }} path: ./apps/sim/.next/cache # Running out of RAM kills the whole VM and surfaces only as "the runner @@ -293,6 +310,20 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile --ignore-scripts + # Whether the Turbopack cache was warm is otherwise invisible: the disk + # mounts successfully either way, and turbo buffers the build log so + # compile time only surfaces at the end. Without this, "is the cache + # working" can only be answered by diffing compile times across runs and + # guessing. Reported, never gated — this is a measurement, not a check. + - name: Report Next.js cache state (pre-build) + run: | + CACHE_DIR=apps/sim/.next/cache + if [ -d "$CACHE_DIR" ] && [ -n "$(ls -A "$CACHE_DIR" 2>/dev/null)" ]; then + echo "PRE_BUILD_CACHE=warm ($(du -sh "$CACHE_DIR" | cut -f1), $(find "$CACHE_DIR" -type f | wc -l) files)" + else + echo 'PRE_BUILD_CACHE=cold (empty or absent — expect a full compile)' + fi + - name: Build application env: NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' @@ -304,7 +335,19 @@ jobs: AWS_REGION: 'us-west-2' ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only TURBO_CACHE_DIR: .turbo - # Opt into Turbopack's persistent build cache (beta) for this CI - # check build only — measured 105s cold vs 22s warm locally. + # Opt into Turbopack's persistent build cache (beta) for this CI check + # build only. The 105s-cold/22s-warm figures behind this were measured + # locally and have never reproduced in CI, where compile has ranged + # 3.5-17.8 min — treat them as local numbers, not a CI target. NEXT_TURBOPACK_BUILD_CACHE: '1' run: bunx turbo run build --filter=sim + + - name: Report Next.js cache state (post-build) + if: always() + run: | + CACHE_DIR=apps/sim/.next/cache + if [ -d "$CACHE_DIR" ]; then + echo "POST_BUILD_CACHE=$(du -sh "$CACHE_DIR" | cut -f1) committed to the sticky disk for the next run on this branch" + else + echo 'POST_BUILD_CACHE=absent — the build wrote no cache' + fi From cb77e1eebe012c3ea2c7637c1aeae0030aded762 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 29 Jul 2026 17:20:34 -0700 Subject: [PATCH 2/2] refactor(ci): trim the cache-key comments to the load-bearing facts Keeps the mechanism (one mutable volume per key, clone-on-mount, last-write-wins) and the two numbers, drops the restated reasoning. Both report steps collapse to a single du, dropping a second full walk of a ~5 GB tree. Cleanup workflow loses a concurrency group a once-per-PR delete never needed. --- .github/workflows/ci-cache-cleanup.yml | 24 ++++------ .github/workflows/test-build.yml | 61 ++++++++------------------ 2 files changed, 28 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci-cache-cleanup.yml b/.github/workflows/ci-cache-cleanup.yml index bf1e2f2890f..d6426b250c5 100644 --- a/.github/workflows/ci-cache-cleanup.yml +++ b/.github/workflows/ci-cache-cleanup.yml @@ -1,36 +1,30 @@ name: CI Cache Cleanup -# The Next.js build cache sticky disk in test-build.yml is keyed per branch, so -# every PR leaves a ~5 GB volume behind. Branches are short-lived; the disks -# aren't. Delete a branch's disk when its PR closes so storage tracks open PRs -# rather than every branch the repo has ever seen. -# -# Only the pull_request-event disk is deleted. The push-event disks belong to -# main/staging/dev and must stay warm. +# test-build.yml keys the Next.js build cache sticky disk per branch, so every PR +# leaves a ~5 GB volume behind. Branches are short-lived; the disks aren't. Only +# the pull_request disks are reclaimed — the push disks belong to main/staging/dev +# and must stay warm. on: pull_request: types: [closed] -concurrency: - group: ci-cache-cleanup-${{ github.head_ref }} - cancel-in-progress: false - permissions: contents: read jobs: delete-nextjs-cache: name: Delete Next.js build cache disk - # Sticky disks only exist on Blacksmith; in GitHub break-glass mode the - # cache-mount fallback uses actions/cache, which expires on its own. + # Sticky disks only exist on Blacksmith; the GitHub break-glass path uses + # actions/cache, which expires on its own. if: vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith' runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 5 steps: - # Key must stay byte-identical to the Mount Next.js build cache key in - # test-build.yml, or this silently deletes nothing and the disks pile up. + # Must stay byte-identical to the Mount Next.js build cache key in + # test-build.yml, or this deletes nothing and the disks accumulate. + # Non-blocking: PRs skipped by ci.yml's paths-ignore never made a disk. - name: Delete sticky disk uses: useblacksmith/stickydisk-delete@b41313d28b8647d72114c9ba3c96bb04061562b6 # v1 continue-on-error: true diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 8bbd775db73..192337458aa 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -267,24 +267,14 @@ jobs: # writes ~5 GB into .next/cache — a sticky disk mounts it in ~1s where an # actions/cache round-trip would eat the warm-build win. # - # Scoped per branch, not just per event. A sticky disk is a single mutable - # volume per key: mounting clones the last committed snapshot and the job - # commits back last-write-wins. Keyed on event alone, every open PR shared - # one volume, so each run cloned a snapshot built from a *different* - # branch. Measured on the same staging commit, minutes apart: the push - # disk (single-writer, genuinely warm) compiled in 9.3 min while the - # shared pull_request disk took 14.0 min. - # - # Branch scoping is also the correctness-preserving choice — - # turbopackFileSystemCacheForBuild is beta, and cross-commit restore is - # not a documented-supported mode (vercel/next.js#87283 reports stale HTML - # from a cache built at another commit). Every other cache layer here - # already isolates by branch: GitHub's cache can't read sibling branches, - # and Blacksmith's own cache product branch-scopes by default. Only this - # key didn't. - # - # Cost of per-branch disks is bounded by ci-cache-cleanup.yml, which - # deletes a branch's disk when its PR closes. + # Keyed per branch, not just per event. A sticky disk is one mutable volume + # per key: mounting clones the last committed snapshot, job end commits back + # last-write-wins. An event-only key had every open PR restoring a cache + # built from a different branch — 14.0 min vs 9.3 min for the single-writer + # push disk on the same commit. Branch scoping also keeps us off + # cross-commit restore, which turbopackFileSystemCacheForBuild (beta) does + # not document as supported (vercel/next.js#87283: stale HTML from a cache + # built at another commit). ci-cache-cleanup.yml reclaims the disks. - name: Mount Next.js build cache uses: ./.github/actions/cache-mount with: @@ -310,19 +300,12 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile --ignore-scripts - # Whether the Turbopack cache was warm is otherwise invisible: the disk - # mounts successfully either way, and turbo buffers the build log so - # compile time only surfaces at the end. Without this, "is the cache - # working" can only be answered by diffing compile times across runs and - # guessing. Reported, never gated — this is a measurement, not a check. - - name: Report Next.js cache state (pre-build) - run: | - CACHE_DIR=apps/sim/.next/cache - if [ -d "$CACHE_DIR" ] && [ -n "$(ls -A "$CACHE_DIR" 2>/dev/null)" ]; then - echo "PRE_BUILD_CACHE=warm ($(du -sh "$CACHE_DIR" | cut -f1), $(find "$CACHE_DIR" -type f | wc -l) files)" - else - echo 'PRE_BUILD_CACHE=cold (empty or absent — expect a full compile)' - fi + # The disk mounts successfully whether or not it carried anything and turbo + # buffers the build log, so cache warmth is otherwise unobservable — #5859 + # shipped a cache that carried almost nothing and it took a PR to notice. + # Reported, never gated. + - name: Report Next.js cache size (pre-build) + run: du -sh apps/sim/.next/cache 2>/dev/null || echo 'cold — no cache restored' - name: Build application env: @@ -336,18 +319,12 @@ jobs: ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only TURBO_CACHE_DIR: .turbo # Opt into Turbopack's persistent build cache (beta) for this CI check - # build only. The 105s-cold/22s-warm figures behind this were measured - # locally and have never reproduced in CI, where compile has ranged - # 3.5-17.8 min — treat them as local numbers, not a CI target. + # build only. #5869's 105s-cold/22s-warm was measured locally and has + # never reproduced in CI (compile has ranged 3.5-17.8 min) — local + # numbers, not a CI target. NEXT_TURBOPACK_BUILD_CACHE: '1' run: bunx turbo run build --filter=sim - - name: Report Next.js cache state (post-build) + - name: Report Next.js cache size (post-build) if: always() - run: | - CACHE_DIR=apps/sim/.next/cache - if [ -d "$CACHE_DIR" ]; then - echo "POST_BUILD_CACHE=$(du -sh "$CACHE_DIR" | cut -f1) committed to the sticky disk for the next run on this branch" - else - echo 'POST_BUILD_CACHE=absent — the build wrote no cache' - fi + run: du -sh apps/sim/.next/cache 2>/dev/null || echo 'no cache written'