From afec72ea7a25c93cf9c38020b7201a594b63bb62 Mon Sep 17 00:00:00 2001 From: marksverdhei Date: Wed, 18 Mar 2026 09:17:02 +0100 Subject: [PATCH 1/7] ci: add arm64 build cache and self-hosted runner cleanup - Registry-based build cache for arm64 QEMU cross-compilation - Cleanup steps on both jobs to prevent garbage accumulation - Remove temp ci-build image after amd64 push - Prune buildx cache and dangling images older than 72h Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/docker.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 27458fe..d5fb9e2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -42,6 +42,12 @@ jobs: docker push "$tag" done + - name: Cleanup + if: always() + run: | + docker rmi ${{ env.IMAGE_NAME }}:ci-build 2>/dev/null || true + docker image prune -f --filter "label=org.opencontainers.image.source=https://github.com/marksverdhei/bakery" --filter "until=72h" 2>/dev/null || true + build-arm64: runs-on: [self-hosted, linux, x64] needs: build-amd64 @@ -82,3 +88,11 @@ jobs: provenance: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache-arm64 + cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache-arm64,mode=max + + - name: Cleanup + if: always() + run: | + docker buildx prune -f --filter "until=72h" 2>/dev/null || true + docker image prune -f --filter "until=72h" 2>/dev/null || true From 8343ae7cca340fbe7251a9382833f03ceafad6ec Mon Sep 17 00:00:00 2001 From: marksverdhei Date: Wed, 18 Mar 2026 09:59:15 +0100 Subject: [PATCH 2/7] fix: arm64 buildx 400 Bad Request from DockerHub Force Docker V2 media types (oci-mediatypes=false), disable SBOM attestations, and use gzip compression to avoid DockerHub rejecting large layer blobs from buildx. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d5fb9e2..32ab111 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -86,10 +86,12 @@ jobs: platforms: linux/arm64 push: true provenance: false + sbom: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache-arm64 cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache-arm64,mode=max + outputs: type=image,oci-mediatypes=false,push=true,compression=gzip,force-compression=true - name: Cleanup if: always() From 83e6a72ec690b2e4fe88ddc23eab4299902f9395 Mon Sep 17 00:00:00 2001 From: marksverdhei Date: Wed, 18 Mar 2026 10:40:17 +0100 Subject: [PATCH 3/7] fix: use cache mode=min to avoid pushing huge base layers mode=max tries to push all layers (including multi-GB CUDA base) as cache, triggering DockerHub 400 errors. mode=min only caches final image layers. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 32ab111..e8bd34f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -90,8 +90,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache-arm64 - cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache-arm64,mode=max - outputs: type=image,oci-mediatypes=false,push=true,compression=gzip,force-compression=true + cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache-arm64,mode=min - name: Cleanup if: always() From 58488cca5677f193d2d1438d98d4dedeb7378719 Mon Sep 17 00:00:00 2001 From: marksverdhei Date: Wed, 18 Mar 2026 11:00:20 +0100 Subject: [PATCH 4/7] fix: remove registry cache to isolate 400 error The successful arm64 builds on Mar 16 had no registry cache. Removing cache-from/cache-to to verify the image push works without cache interference. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/docker.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e8bd34f..23f90eb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -89,8 +89,6 @@ jobs: sbom: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache-arm64 - cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache-arm64,mode=min - name: Cleanup if: always() From 8e8574bcca19809057fca1e52139b7b8165f4eb3 Mon Sep 17 00:00:00 2001 From: marksverdhei Date: Wed, 18 Mar 2026 11:30:48 +0100 Subject: [PATCH 5/7] fix: use GitHub Actions cache instead of registry cache Registry cache (type=registry) triggers DockerHub 400 errors on large CUDA layers. GHA cache (type=gha) stores build cache in GitHub's blob storage, avoiding DockerHub entirely. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 23f90eb..9d86582 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -89,6 +89,8 @@ jobs: sbom: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=arm64 + cache-to: type=gha,scope=arm64,mode=max - name: Cleanup if: always() From 3e931f4a5a8ed8653267928b88a353b6c84d65d0 Mon Sep 17 00:00:00 2001 From: marksverdhei Date: Wed, 18 Mar 2026 12:29:48 +0100 Subject: [PATCH 6/7] fix: use local disk cache instead of registry cache for arm64 Registry cache triggers DockerHub 400 errors (buildx#271). Local cache on the self-hosted runner avoids network roundtrips entirely. - BuildKit GC capped at 20GB via maxUsedSpace - Cache rotated after each build to prevent unbounded growth - buildx prune --keep-storage 20g as safety net Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/docker.yml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9d86582..00b282c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -8,6 +8,7 @@ on: env: IMAGE_NAME: marksverdhei/bakery + ARM64_CACHE_DIR: /var/lib/buildx-cache/bakery-arm64 jobs: build-amd64: @@ -61,6 +62,12 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + buildkitd-config-inline: | + [worker.oci] + [[worker.oci.gcpolicy]] + maxUsedSpace = "20GB" + keepDuration = 604800 - name: Login to DockerHub uses: docker/login-action@v3 @@ -78,6 +85,9 @@ jobs: type=semver,pattern={{version}},suffix=-arm64 type=sha,prefix=,suffix=-arm64 + - name: Prepare cache + run: mkdir -p ${{ env.ARM64_CACHE_DIR }} + - name: Build and push (arm64) uses: docker/build-push-action@v6 with: @@ -89,11 +99,19 @@ jobs: sbom: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha,scope=arm64 - cache-to: type=gha,scope=arm64,mode=max + cache-from: type=local,src=${{ env.ARM64_CACHE_DIR }} + cache-to: type=local,dest=${{ env.ARM64_CACHE_DIR }}-new,mode=max + + - name: Rotate cache + if: always() + run: | + rm -rf ${{ env.ARM64_CACHE_DIR }} + if [ -d "${{ env.ARM64_CACHE_DIR }}-new" ]; then + mv ${{ env.ARM64_CACHE_DIR }}-new ${{ env.ARM64_CACHE_DIR }} + fi - name: Cleanup if: always() run: | - docker buildx prune -f --filter "until=72h" 2>/dev/null || true + docker buildx prune -f --keep-storage 20g 2>/dev/null || true docker image prune -f --filter "until=72h" 2>/dev/null || true From 6ef0228002a2db1df3b78f6bc35f4a5462ee374f Mon Sep 17 00:00:00 2001 From: marksverdhei Date: Wed, 18 Mar 2026 16:38:35 +0100 Subject: [PATCH 7/7] feat: add wandb and flash-linear-attention to Docker images Co-Authored-By: Claude Opus 4.6 (1M context) --- Dockerfile | 2 +- Dockerfile.arm64 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index cdc1883..48352ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv WORKDIR /opt/bakery COPY pyproject.toml uv.lock README.md LICENSE ./ COPY src/ src/ -RUN uv pip install ".[qlora]" +RUN uv pip install ".[qlora]" wandb flash-linear-attention # Verify installation RUN python3 -c "import bakery; import torch; print(f'bakery OK, torch={torch.__version__}, cuda_built={torch.version.cuda}')" diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index 73e984d..d8d72d8 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -28,7 +28,7 @@ RUN uv pip install torch --index-url https://download.pytorch.org/whl/cu128 WORKDIR /opt/bakery COPY pyproject.toml uv.lock README.md LICENSE ./ COPY src/ src/ -RUN uv pip install ".[qlora]" +RUN uv pip install ".[qlora]" wandb flash-linear-attention # Verify installation RUN python3 -c "import bakery; import torch; print(f'bakery OK, torch={torch.__version__}, cuda_built={torch.version.cuda}')"