diff --git a/.changeset/patch-add-arm64-container-builds.md b/.changeset/patch-add-arm64-container-builds.md new file mode 100644 index 0000000000..0f14a9b247 --- /dev/null +++ b/.changeset/patch-add-arm64-container-builds.md @@ -0,0 +1,4 @@ +--- +"gh-aw": patch +--- +Updated the Docker build flow to use buildx and production dist layout so linux/amd64 and linux/arm64 images are produced and tested. diff --git a/.changeset/patch-arm64-container-build.md b/.changeset/patch-arm64-container-build.md new file mode 100644 index 0000000000..dc573a7b82 --- /dev/null +++ b/.changeset/patch-arm64-container-build.md @@ -0,0 +1,4 @@ +--- +"gh-aw": patch +--- +Updated the Docker/CI flow so the dist/ directories feed multi-platform linux/amd64 and linux/arm64 images built via buildx and validated via the release/CI scripts. diff --git a/.changeset/patch-document-arm64-container-builds.md b/.changeset/patch-document-arm64-container-builds.md new file mode 100644 index 0000000000..15c2b99ff1 --- /dev/null +++ b/.changeset/patch-document-arm64-container-builds.md @@ -0,0 +1,4 @@ +--- +"gh-aw": patch +--- +Updated the release, Makefile, CI workflows, and Docker build scripts so linux/amd64 and linux/arm64 container images are produced and validated via buildx. diff --git a/.changeset/patch-fix-alpine-dockerfile.md b/.changeset/patch-fix-alpine-dockerfile.md new file mode 100644 index 0000000000..169f5fa823 --- /dev/null +++ b/.changeset/patch-fix-alpine-dockerfile.md @@ -0,0 +1,5 @@ +--- +"gh-aw": patch +--- + +Fix Alpine Dockerfile compilation by using Alpine 3.19 and correct gh package name diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json index 0a95331f84..e86e50dd3a 100644 --- a/.github/aw/actions-lock.json +++ b/.github/aw/actions-lock.json @@ -150,6 +150,11 @@ "version": "v3.12.0", "sha": "8d2750c68a42422c14e847fe6c8ac0403b4cbd6f" }, + "docker/setup-qemu-action@v3": { + "repo": "docker/setup-qemu-action", + "version": "v3", + "sha": "c7c53464625b32c7a7e944ae62b3e17d2b600130" + }, "erlef/setup-beam@v1": { "repo": "erlef/setup-beam", "version": "v1.20.4", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00720c7694..170d5f579a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1773,11 +1773,23 @@ jobs: - name: Build Linux binary for Alpine run: make build-linux + - name: Prepare dist directory for Docker build + run: | + echo "Preparing dist directory structure..." + mkdir -p dist + cp gh-aw-linux-amd64 dist/linux-amd64 + cp gh-aw-linux-arm64 dist/linux-arm64 + echo "✅ dist directory prepared" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build Alpine Docker image run: | echo "Building Alpine Docker image..." - docker build -t gh-aw-alpine:test \ - --build-arg BINARY=gh-aw-linux-amd64 \ + docker buildx build --platform linux/amd64 \ + -t gh-aw-alpine:test \ + --load \ -f Dockerfile . echo "✅ Alpine Docker image built successfully" @@ -1852,6 +1864,158 @@ jobs: run: | rm -rf test-workspace + alpine-container-test-arm64: + name: Alpine Container Test (ARM64) + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-alpine-container-arm64 + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Download dependencies with retry + run: | + set -e + MAX_RETRIES=3 + RETRY_DELAY=5 + + for i in $(seq 1 $MAX_RETRIES); do + echo "Attempt $i of $MAX_RETRIES: Downloading Go modules..." + if go mod download; then + echo "✅ Successfully downloaded Go modules" + break + else + if [ $i -eq $MAX_RETRIES ]; then + echo "❌ Failed to download Go modules after $MAX_RETRIES attempts" + echo "This may indicate that proxy.golang.org is unreachable" + echo "Please check network connectivity or consider vendoring dependencies" + exit 1 + fi + echo "⚠️ Download failed, retrying in ${RETRY_DELAY}s..." + sleep $RETRY_DELAY + fi + done + + - name: Verify dependencies + run: go mod verify + + - name: Build Linux binary for Alpine + run: make build-linux + + - name: Prepare dist directory for Docker build + run: | + echo "Preparing dist directory structure..." + mkdir -p dist + cp gh-aw-linux-amd64 dist/linux-amd64 + cp gh-aw-linux-arm64 dist/linux-arm64 + echo "✅ dist directory prepared" + + - name: Set up QEMU for ARM64 emulation + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Alpine Docker image (ARM64) + run: | + echo "Building Alpine Docker image for ARM64..." + docker buildx build --platform linux/arm64 \ + -t gh-aw-alpine:test-arm64 \ + --load \ + -f Dockerfile . + echo "✅ Alpine Docker image (ARM64) built successfully" + + - name: Test Docker image basic commands (ARM64) + run: | + echo "Testing Docker image basic commands on ARM64..." + docker run --rm --platform linux/arm64 gh-aw-alpine:test-arm64 --version + docker run --rm --platform linux/arm64 gh-aw-alpine:test-arm64 --help + echo "✅ Basic commands work on ARM64" + + - name: Create test workflow in container + run: | + echo "Creating test workflow file..." + mkdir -p test-workspace/.github/workflows + cat > test-workspace/.github/workflows/test-alpine-arm64.md << 'EOF' + --- + on: push + engine: copilot + --- + # Test Workflow for Alpine Container (ARM64) + + This is a simple test workflow to verify the compile command works correctly in Alpine container on ARM64. + + ## Task + Echo hello from Alpine container (ARM64). + EOF + echo "✅ Test workflow created" + + - name: Run compile through Alpine container (ARM64) + run: | + echo "Running compile command through Alpine container (ARM64)..." + docker run --rm --platform linux/arm64 \ + -v "$(pwd)/test-workspace:/workspace" \ + -w /workspace \ + gh-aw-alpine:test-arm64 compile test-alpine-arm64 --verbose + + echo "✅ Compile command executed on ARM64" + + - name: Verify lock file generation + run: | + echo "Verifying lock file was generated..." + if [ -f "test-workspace/.github/workflows/test-alpine-arm64.lock.yml" ]; then + echo "✅ Lock file generated successfully" + echo "" + echo "Lock file contents:" + head -20 test-workspace/.github/workflows/test-alpine-arm64.lock.yml + else + echo "❌ Lock file not found" + ls -la test-workspace/.github/workflows/ + exit 1 + fi + + - name: Generate test summary + if: always() + run: | + echo "## Alpine Container Test Results (ARM64)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "This test verifies that:" >> $GITHUB_STEP_SUMMARY + echo "1. The Alpine Docker image can be built for ARM64 architecture" >> $GITHUB_STEP_SUMMARY + echo "2. The gh-aw binary works correctly in Alpine Linux on ARM64" >> $GITHUB_STEP_SUMMARY + echo "3. The compile command can process workflows in the ARM64 container" >> $GITHUB_STEP_SUMMARY + echo "4. Lock files are generated correctly on ARM64" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + if [ -f "test-workspace/.github/workflows/test-alpine-arm64.lock.yml" ]; then + echo "✅ All ARM64 tests passed successfully" >> $GITHUB_STEP_SUMMARY + else + echo "❌ Lock file generation failed on ARM64" >> $GITHUB_STEP_SUMMARY + fi + + - name: Clean up test files + if: always() + run: | + rm -rf test-workspace + safe-outputs-conformance: runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/release.lock.yml b/.github/workflows/release.lock.yml index 858f6f13bd..700b54bc9d 100644 --- a/.github/workflows/release.lock.yml +++ b/.github/workflows/release.lock.yml @@ -27,7 +27,7 @@ # Imports: # - shared/mood.md # -# frontmatter-hash: cc2b8c6b51e275896e8102fe283fdb0e690e468cd8e5693425a35d14bb622645 +# frontmatter-hash: 5ab9279679b4dcbf80e9fa430e222569d154c38e53b56d06959fa443aaf617ed name: "Release" "on": @@ -1183,17 +1183,19 @@ jobs: echo "✓ Binaries built successfully" env: RELEASE_TAG: ${{ needs.config.outputs.release_tag }} + - name: Set up QEMU for multi-platform builds + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + with: + platforms: arm64 - name: Setup Docker Buildx (pre-validation) uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Build Docker image (validation only) uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: - build-args: | - BINARY=dist/linux-amd64 cache-from: type=gha context: . load: false - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: false - name: Create GitHub release id: get_release @@ -1273,17 +1275,15 @@ jobs: type=semver,pattern={{major}} type=sha,format=long type=raw,value=latest,enable={{is_default_branch}} - - name: Build and push Docker image (amd64) + - name: Build and push Docker image (multi-platform) id: build uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: - build-args: | - BINARY=dist/linux-amd64 cache-from: type=gha cache-to: type=gha,mode=max context: . labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 provenance: mode=max push: true sbom: true diff --git a/.github/workflows/release.md b/.github/workflows/release.md index eb1d4b864e..ba90061845 100644 --- a/.github/workflows/release.md +++ b/.github/workflows/release.md @@ -185,6 +185,11 @@ jobs: bash scripts/build-release.sh "$RELEASE_TAG" echo "✓ Binaries built successfully" + - name: Set up QEMU for multi-platform builds + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - name: Setup Docker Buildx (pre-validation) uses: docker/setup-buildx-action@v3 @@ -192,11 +197,9 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: false load: false - build-args: | - BINARY=dist/linux-amd64 cache-from: type=gha - name: Create GitHub release @@ -287,17 +290,15 @@ jobs: type=sha,format=long type=raw,value=latest,enable={{is_default_branch}} - - name: Build and push Docker image (amd64) + - name: Build and push Docker image (multi-platform) id: build uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - build-args: | - BINARY=dist/linux-amd64 cache-from: type=gha cache-to: type=gha,mode=max sbom: true diff --git a/Dockerfile b/Dockerfile index c0681ff33b..d1a0a75026 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,38 @@ # Dockerfile for GitHub Agentic Workflows compiler # Provides a minimal container with gh-aw, gh CLI, git, and jq -# Use Alpine for minimal size (official distribution) -FROM alpine:3.21 +# Use Alpine 3.19 for minimal size (3.20+ removed gh CLI due to Python 3.12 compatibility) +FROM alpine:3.19 # Install required dependencies -RUN apk add --no-cache \ +RUN apk update && apk add --no-cache \ git \ jq \ bash \ curl \ ca-certificates \ - github-cli + gh -# Accept build argument for binary name (defaults to linux-amd64) -ARG BINARY=gh-aw-linux-amd64 +# Docker Buildx automatically provides these ARGs for multi-platform builds +# Expected values: TARGETOS=linux, TARGETARCH=amd64|arm64 +# For local builds without buildx, these must be provided explicitly: +# docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 ... +# Default to linux/amd64 if not provided +ARG TARGETOS=linux +ARG TARGETARCH=amd64 # Create a directory for the binary WORKDIR /usr/local/bin -# Copy the gh-aw binary from build context -COPY ${BINARY} /usr/local/bin/gh-aw +# Copy the appropriate binary based on target platform +# TARGETOS=linux, TARGETARCH=amd64 -> dist/linux-amd64 +# TARGETOS=linux, TARGETARCH=arm64 -> dist/linux-arm64 +COPY dist/${TARGETOS}-${TARGETARCH} /usr/local/bin/gh-aw -# Ensure the binary is executable -RUN chmod +x /usr/local/bin/gh-aw +# Ensure the binary is executable and verify it exists +RUN chmod +x /usr/local/bin/gh-aw && \ + /usr/local/bin/gh-aw --version || \ + (echo "Error: gh-aw binary not found or not executable" && exit 1) # Configure git to trust all directories to avoid "dubious ownership" errors # This is necessary when the container runs with mounted volumes owned by different users diff --git a/Makefile b/Makefile index b1838ee540..4a5f47c89a 100644 --- a/Makefile +++ b/Makefile @@ -270,11 +270,22 @@ docker-build: build-linux echo "Error: Docker is not installed."; \ exit 1; \ fi - @# Build for linux/amd64 by default for local testing - docker build -t $(DOCKER_IMAGE):$(VERSION) \ - --build-arg BINARY=$(BINARY_NAME)-linux-amd64 \ + @# Prepare dist directory structure for Docker build + @mkdir -p dist + @cp $(BINARY_NAME)-linux-amd64 dist/linux-amd64 + @cp $(BINARY_NAME)-linux-arm64 dist/linux-arm64 + @# Check if buildx is available + @if ! docker buildx version >/dev/null 2>&1; then \ + echo "Error: Docker buildx is not available."; \ + echo "Install with: docker buildx install"; \ + exit 1; \ + fi + @# Build for linux/amd64 by default for local testing using buildx + docker buildx build --platform linux/amd64 \ + -t $(DOCKER_IMAGE):$(VERSION) \ + -t $(DOCKER_IMAGE):latest \ + --load \ -f Dockerfile . - @docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):latest @echo "✓ Docker image built: $(DOCKER_IMAGE):$(VERSION)" @echo "✓ Docker image tagged: $(DOCKER_IMAGE):latest" @@ -291,6 +302,10 @@ docker-build-multiarch: build-linux echo "Install with: docker buildx install"; \ exit 1; \ fi + @# Prepare dist directory structure for Docker build + @mkdir -p dist + @cp $(BINARY_NAME)-linux-amd64 dist/linux-amd64 + @cp $(BINARY_NAME)-linux-arm64 dist/linux-arm64 @# Create buildx builder if it doesn't exist @docker buildx create --use --name gh-aw-builder 2>/dev/null || docker buildx use gh-aw-builder @# Build for multiple platforms diff --git a/pkg/cli/docker_build_integration_test.go b/pkg/cli/docker_build_integration_test.go index d09120f5fb..7c88bbca1c 100644 --- a/pkg/cli/docker_build_integration_test.go +++ b/pkg/cli/docker_build_integration_test.go @@ -35,13 +35,15 @@ func TestDockerfile_Exists(t *testing.T) { // Verify essential components are present requiredComponents := []string{ - "FROM alpine:", // Alpine base image - "github-cli", // GitHub CLI package - "git", // Git package - "jq", // jq package - "bash", // Bash package - "ARG BINARY", // Build argument for binary - "ENTRYPOINT [\"gh-aw\"]", // Entrypoint + "FROM alpine:", // Alpine base image + "github-cli", // GitHub CLI package + "git", // Git package + "jq", // jq package + "bash", // Bash package + "ARG TARGETOS", // Build argument for target OS + "ARG TARGETARCH", // Build argument for target architecture + "dist/${TARGETOS}-${TARGETARCH}", // Binary path pattern + "ENTRYPOINT [\"gh-aw\"]", // Entrypoint } for _, component := range requiredComponents { @@ -124,13 +126,18 @@ func TestDockerBuild_WithMake(t *testing.T) { t.Fatalf("Failed to build Linux binary: %v", err) } - // Verify the Linux binary was created - binaryPath := filepath.Join(repoRoot, "gh-aw-linux-amd64") - if _, err := os.Stat(binaryPath); os.IsNotExist(err) { - t.Fatalf("Linux binary not found at %s", binaryPath) + // Verify the Linux binaries were created in repo root + binaryPathAmd64 := filepath.Join(repoRoot, "gh-aw-linux-amd64") + if _, err := os.Stat(binaryPathAmd64); os.IsNotExist(err) { + t.Fatalf("Linux amd64 binary not found at %s", binaryPathAmd64) } - // Build Docker image using Makefile + binaryPathArm64 := filepath.Join(repoRoot, "gh-aw-linux-arm64") + if _, err := os.Stat(binaryPathArm64); os.IsNotExist(err) { + t.Fatalf("Linux arm64 binary not found at %s", binaryPathArm64) + } + + // Build Docker image using Makefile (should create dist/ directory) t.Log("Building Docker image with make docker-build...") dockerBuildCmd := exec.Command("make", "docker-build") dockerBuildCmd.Dir = repoRoot @@ -140,6 +147,17 @@ func TestDockerBuild_WithMake(t *testing.T) { t.Fatalf("Failed to build Docker image: %v", err) } + // Verify dist directory structure was created + distAmd64Path := filepath.Join(repoRoot, "dist", "linux-amd64") + if _, err := os.Stat(distAmd64Path); os.IsNotExist(err) { + t.Errorf("dist/linux-amd64 binary not found at %s", distAmd64Path) + } + + distArm64Path := filepath.Join(repoRoot, "dist", "linux-arm64") + if _, err := os.Stat(distArm64Path); os.IsNotExist(err) { + t.Errorf("dist/linux-arm64 binary not found at %s", distArm64Path) + } + t.Log("Docker image built successfully") // Clean up Docker image after test diff --git a/pkg/workflow/data/action_pins.json b/pkg/workflow/data/action_pins.json index 0a95331f84..e86e50dd3a 100644 --- a/pkg/workflow/data/action_pins.json +++ b/pkg/workflow/data/action_pins.json @@ -150,6 +150,11 @@ "version": "v3.12.0", "sha": "8d2750c68a42422c14e847fe6c8ac0403b4cbd6f" }, + "docker/setup-qemu-action@v3": { + "repo": "docker/setup-qemu-action", + "version": "v3", + "sha": "c7c53464625b32c7a7e944ae62b3e17d2b600130" + }, "erlef/setup-beam@v1": { "repo": "erlef/setup-beam", "version": "v1.20.4",