From d944c5d81f1a256575ab48832bd2ff7d7215db03 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Apr 2025 10:17:45 -0700 Subject: [PATCH 01/14] Pointing to private registry --- .github/workflows/publish_docker.yaml | 58 +++++++++++++-------------- noxfiles/constants_nox.py | 4 +- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index c5ae771a2ca..4d405a80fb5 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -150,52 +150,52 @@ jobs: - name: Create and Push Dev Manifest if: needs.ParseTags.outputs.prod_tag == 'false' && needs.ParseTags.outputs.rc_tag == 'false' && needs.ParseTags.outputs.beta_tag == 'false' && needs.ParseTags.outputs.alpha_tag == 'false' run: | - docker manifest create ethyca/${{ matrix.image_name }}:dev \ - --amend ethyca/${{ matrix.image_name }}:dev-amd64 \ - --amend ethyca/${{ matrix.image_name }}:dev-arm64 - docker manifest push ethyca/${{ matrix.image_name }}:dev + docker manifest create thegalvanizer/${{ matrix.image_name }}:dev \ + --amend thegalvanizer/${{ matrix.image_name }}:dev-amd64 \ + --amend thegalvanizer/${{ matrix.image_name }}:dev-arm64 + docker manifest push thegalvanizer/${{ matrix.image_name }}:dev - name: Create and Push Prod Manifest if: needs.ParseTags.outputs.prod_tag == 'true' run: | # Create and push version manifest - docker manifest create ethyca/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} \ - --amend ethyca/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-amd64 \ - --amend ethyca/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-arm64 - docker manifest push ethyca/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} + docker manifest create thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} \ + --amend thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-amd64 \ + --amend thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-arm64 + docker manifest push thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} # Create and push latest manifest - docker manifest create ethyca/${{ matrix.image_name }}:latest \ - --amend ethyca/${{ matrix.image_name }}:latest-amd64 \ - --amend ethyca/${{ matrix.image_name }}:latest-arm64 - docker manifest push ethyca/${{ matrix.image_name }}:latest + docker manifest create thegalvanizer/${{ matrix.image_name }}:latest \ + --amend thegalvanizer/${{ matrix.image_name }}:latest-amd64 \ + --amend thegalvanizer/${{ matrix.image_name }}:latest-arm64 + docker manifest push thegalvanizer/${{ matrix.image_name }}:latest - name: Create and Push RC Manifest if: needs.ParseTags.outputs.rc_tag == 'true' run: | # Create and push version manifest - docker manifest create ethyca/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} \ - --amend ethyca/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-amd64 \ - --amend ethyca/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-arm64 - docker manifest push ethyca/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} + docker manifest create thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} \ + --amend thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-amd64 \ + --amend thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-arm64 + docker manifest push thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} # Create and push rc manifest - docker manifest create ethyca/${{ matrix.image_name }}:rc \ - --amend ethyca/${{ matrix.image_name }}:rc-amd64 \ - --amend ethyca/${{ matrix.image_name }}:rc-arm64 - docker manifest push ethyca/${{ matrix.image_name }}:rc + docker manifest create thegalvanizer/${{ matrix.image_name }}:rc \ + --amend thegalvanizer/${{ matrix.image_name }}:rc-amd64 \ + --amend thegalvanizer/${{ matrix.image_name }}:rc-arm64 + docker manifest push thegalvanizer/${{ matrix.image_name }}:rc - name: Create and Push Prerelease Manifest if: needs.ParseTags.outputs.alpha_tag == 'true' || needs.ParseTags.outputs.beta_tag == 'true' run: | # Create and push version manifest - docker manifest create ethyca/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} \ - --amend ethyca/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-amd64 \ - --amend ethyca/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-arm64 - docker manifest push ethyca/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} + docker manifest create thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} \ + --amend thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-amd64 \ + --amend thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-arm64 + docker manifest push thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} # Create and push prerelease manifest - docker manifest create ethyca/${{ matrix.image_name }}:prerelease \ - --amend ethyca/${{ matrix.image_name }}:prerelease-amd64 \ - --amend ethyca/${{ matrix.image_name }}:prerelease-arm64 - docker manifest push ethyca/${{ matrix.image_name }}:prerelease + docker manifest create thegalvanizer/${{ matrix.image_name }}:prerelease \ + --amend thegalvanizer/${{ matrix.image_name }}:prerelease-amd64 \ + --amend thegalvanizer/${{ matrix.image_name }}:prerelease-arm64 + docker manifest push thegalvanizer/${{ matrix.image_name }}:prerelease NotifyRedeploy: runs-on: ubuntu-latest @@ -207,5 +207,5 @@ jobs: uses: peter-evans/repository-dispatch@v3 with: event-type: trigger-fidesinfra-deploy-fides-rc - repository: ethyca/fidesinfra + repository: thegalvanizer/fidesinfra token: ${{ secrets.DISPATCH_ACCESS_TOKEN }} diff --git a/noxfiles/constants_nox.py b/noxfiles/constants_nox.py index afc4422c9d2..fde50d96648 100644 --- a/noxfiles/constants_nox.py +++ b/noxfiles/constants_nox.py @@ -22,13 +22,13 @@ } # Image Names & Tags -REGISTRY = "ethyca" +REGISTRY = "thegalvanizer" IMAGE_NAME = "fides" CONTAINER_NAME = "fides" COMPOSE_SERVICE_NAME = "fides" # Image Names & Tags -REGISTRY = "ethyca" +REGISTRY = "thegalvanizer" IMAGE_NAME = "fides" IMAGE = f"{REGISTRY}/{IMAGE_NAME}" IMAGE_LOCAL = f"{IMAGE}:local" From 514680ecbac0e6ca534fabc04e552f0558ea157a Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Apr 2025 10:27:39 -0700 Subject: [PATCH 02/14] Focusing on sample_app --- .github/workflows/publish_docker.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index 4d405a80fb5..d40f1c5e940 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -63,7 +63,8 @@ jobs: needs: ParseTags strategy: matrix: - application: ["fides", "sample_app", "privacy_center"] + # application: ["fides", "sample_app", "privacy_center"] + application: ["sample_app"] arch: ["amd64", "arm64"] include: - arch: "amd64" @@ -132,7 +133,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - application: ["fides", "sample_app", "privacy_center"] + # application: ["fides", "sample_app", "privacy_center"] + application: ["sample_app"] include: - application: "fides" image_name: "fides" From 6d8af061a539c31c7480fb075cd8416a74355947 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Apr 2025 13:53:35 -0700 Subject: [PATCH 03/14] Testing new workflow --- .github/workflows/publish_docker.yaml | 181 +++++++++++--------------- 1 file changed, 79 insertions(+), 102 deletions(-) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index d40f1c5e940..e52d8320ca5 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -8,10 +8,11 @@ on: - "*" env: - # Docker auth with read-write (publish) permissions. Set as env in workflow root as auth is required in multiple jobs. DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} DEFAULT_PYTHON_VERSION: "3.10.16" + REGISTRY: docker.io + IMAGE_NAME: thegalvanizer jobs: ParseTags: @@ -23,6 +24,7 @@ jobs: beta_tag: ${{ steps.check-beta-tag.outputs.match }} version: ${{ github.ref_name }} steps: + # Your existing tag parsing steps remain the same - name: Check Prod Tag id: check-prod-tag run: | @@ -59,29 +61,37 @@ jobs: echo "match=false" >> $GITHUB_OUTPUT fi - Push: + BuildAndPush: needs: ParseTags strategy: + fail-fast: false matrix: - # application: ["fides", "sample_app", "privacy_center"] - application: ["sample_app"] - arch: ["amd64", "arm64"] + application: ["sample_app"] # Expand to ["fides", "sample_app", "privacy_center"] as needed + platform: [linux/amd64, linux/arm64] include: - - arch: "amd64" - runner: "ubuntu-latest" - - arch: "arm64" - runner: "ubuntu-24.04-arm" + - platform: linux/amd64 + runner: ubuntu-latest + - platform: linux/arm64 + runner: ubuntu-24.04-arm + - application: "sample_app" + image_name: "fides-sample-app" + build_context: "clients/sample-app" + build_target: "prod" runs-on: ${{ matrix.runner }} steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 with: - fetch-depth: 0 # This is required to properly tag images + fetch-depth: 0 - - name: Set Up Python - uses: actions/setup-python@v5 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 with: - python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - cache: "pip" + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub uses: docker/login-action@v3 @@ -89,123 +99,90 @@ jobs: username: ${{ env.DOCKER_USER }} password: ${{ env.DOCKER_TOKEN }} - - name: Install Dev Requirements - run: pip install -r dev-requirements.txt + - name: Build and push by digest + id: build + uses: docker/build-push-action@v6 + with: + context: ${{ matrix.build_context }} + target: ${{ matrix.build_target }} + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }},push-by-digest=true - # if neither prod, rc, beta or alpha git tag, then push images with the ":dev" tag - # these dev images do not need a versioned/git-tagged image - - name: Push Fides Dev Tag - if: needs.ParseTags.outputs.prod_tag == 'false' && needs.ParseTags.outputs.rc_tag == 'false' && needs.ParseTags.outputs.beta_tag == 'false' && needs.ParseTags.outputs.alpha_tag == 'false' - env: - DOCKER_PLATFORMS: linux/${{ matrix.arch }} - IMAGE_SUFFIX: -${{ matrix.arch }} - run: nox -s "push(${{ matrix.application }},dev)" - - # if a prod git tag, then we run the prod job to publish images tagged with the version number and a constant ":latest" tag - # prod pushes a versioned image, git-tagged images not needed - - name: Push Fides Prod Tags - if: needs.ParseTags.outputs.prod_tag == 'true' - env: - DOCKER_PLATFORMS: linux/${{ matrix.arch }} - IMAGE_SUFFIX: -${{ matrix.arch }} - run: nox -s "push(${{ matrix.application }},prod)" - - # if an RC git tag, then we run the rc job to publish images with an ":rc" tag - # git-tagged images are also pushed - - name: Push Fides RC Tags - if: needs.ParseTags.outputs.rc_tag == 'true' - env: - DOCKER_PLATFORMS: linux/${{ matrix.arch }} - IMAGE_SUFFIX: -${{ matrix.arch }} - run: nox -s "push(${{ matrix.application }},rc)" -- git_tag - - # if an alpha or beta git tag, then we run the prerelease job to publish images with an ":prerelease" tag - # git-tagged images are also pushed - - name: Push Fides prerelease Tags - if: needs.ParseTags.outputs.alpha_tag == 'true' || needs.ParseTags.outputs.beta_tag == 'true' - env: - DOCKER_PLATFORMS: linux/${{ matrix.arch }} - IMAGE_SUFFIX: -${{ matrix.arch }} - run: nox -s "push(${{ matrix.application }},prerelease)" -- git_tag + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.application }}-${{ matrix.platform }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 CreateManifests: - needs: [Push, ParseTags] - runs-on: ubuntu-latest + needs: [ParseTags, BuildAndPush] strategy: matrix: - # application: ["fides", "sample_app", "privacy_center"] - application: ["sample_app"] + application: ["sample_app"] # Expand to ["fides", "sample_app", "privacy_center"] as needed include: - - application: "fides" - image_name: "fides" - application: "sample_app" image_name: "fides-sample-app" - - application: "privacy_center" - image_name: "fides-privacy-center" + runs-on: ubuntu-latest steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-${{ matrix.application }}-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub uses: docker/login-action@v3 with: username: ${{ env.DOCKER_USER }} password: ${{ env.DOCKER_TOKEN }} - - name: Create and Push Dev Manifest + - name: Create manifest lists and push (dev) if: needs.ParseTags.outputs.prod_tag == 'false' && needs.ParseTags.outputs.rc_tag == 'false' && needs.ParseTags.outputs.beta_tag == 'false' && needs.ParseTags.outputs.alpha_tag == 'false' + working-directory: /tmp/digests run: | - docker manifest create thegalvanizer/${{ matrix.image_name }}:dev \ - --amend thegalvanizer/${{ matrix.image_name }}:dev-amd64 \ - --amend thegalvanizer/${{ matrix.image_name }}:dev-arm64 - docker manifest push thegalvanizer/${{ matrix.image_name }}:dev + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '{"tags":["${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:dev"]}') \ + $(ls -1 | sort | sed -e 's/^/${{ env.REGISTRY }}\/${{ env.IMAGE_NAME }}\/${{ matrix.image_name }}@sha256:/') - - name: Create and Push Prod Manifest + - name: Create manifest lists and push (prod) if: needs.ParseTags.outputs.prod_tag == 'true' + working-directory: /tmp/digests run: | - # Create and push version manifest - docker manifest create thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} \ - --amend thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-amd64 \ - --amend thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-arm64 - docker manifest push thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} - # Create and push latest manifest - docker manifest create thegalvanizer/${{ matrix.image_name }}:latest \ - --amend thegalvanizer/${{ matrix.image_name }}:latest-amd64 \ - --amend thegalvanizer/${{ matrix.image_name }}:latest-arm64 - docker manifest push thegalvanizer/${{ matrix.image_name }}:latest - - - name: Create and Push RC Manifest + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '{"tags":["${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}","${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:latest"]}') \ + $(ls -1 | sort | sed -e 's/^/${{ env.REGISTRY }}\/${{ env.IMAGE_NAME }}\/${{ matrix.image_name }}@sha256:/') + + - name: Create manifest lists and push (rc) if: needs.ParseTags.outputs.rc_tag == 'true' + working-directory: /tmp/digests run: | - # Create and push version manifest - docker manifest create thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} \ - --amend thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-amd64 \ - --amend thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-arm64 - docker manifest push thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} - # Create and push rc manifest - docker manifest create thegalvanizer/${{ matrix.image_name }}:rc \ - --amend thegalvanizer/${{ matrix.image_name }}:rc-amd64 \ - --amend thegalvanizer/${{ matrix.image_name }}:rc-arm64 - docker manifest push thegalvanizer/${{ matrix.image_name }}:rc - - - name: Create and Push Prerelease Manifest + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '{"tags":["${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}","${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:rc"]}') \ + $(ls -1 | sort | sed -e 's/^/${{ env.REGISTRY }}\/${{ env.IMAGE_NAME }}\/${{ matrix.image_name }}@sha256:/') + + - name: Create manifest lists and push (prerelease) if: needs.ParseTags.outputs.alpha_tag == 'true' || needs.ParseTags.outputs.beta_tag == 'true' + working-directory: /tmp/digests run: | - # Create and push version manifest - docker manifest create thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} \ - --amend thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-amd64 \ - --amend thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}-arm64 - docker manifest push thegalvanizer/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }} - # Create and push prerelease manifest - docker manifest create thegalvanizer/${{ matrix.image_name }}:prerelease \ - --amend thegalvanizer/${{ matrix.image_name }}:prerelease-amd64 \ - --amend thegalvanizer/${{ matrix.image_name }}:prerelease-arm64 - docker manifest push thegalvanizer/${{ matrix.image_name }}:prerelease + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '{"tags":["${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}","${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:prerelease"]}') \ + $(ls -1 | sort | sed -e 's/^/${{ env.REGISTRY }}\/${{ env.IMAGE_NAME }}\/${{ matrix.image_name }}@sha256:/') NotifyRedeploy: runs-on: ubuntu-latest needs: [CreateManifests] + if: needs.ParseTags.outputs.rc_tag == 'true' steps: - # if an RC git tag, also notify Fidesinfra to trigger a redeploy of rc env, to pick up our newly published images - name: Send Repository Dispatch Event (RC redeploy) - if: needs.ParseTags.outputs.rc_tag == 'true' uses: peter-evans/repository-dispatch@v3 with: event-type: trigger-fidesinfra-deploy-fides-rc From bd2c5aa7293c440751a077cddf06c3ada46b687c Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Apr 2025 14:01:05 -0700 Subject: [PATCH 04/14] Setting arch as the key --- .github/workflows/publish_docker.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index e52d8320ca5..7828e0ead7b 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -67,12 +67,14 @@ jobs: fail-fast: false matrix: application: ["sample_app"] # Expand to ["fides", "sample_app", "privacy_center"] as needed - platform: [linux/amd64, linux/arm64] + arch: ["amd64", "arm64"] include: - - platform: linux/amd64 + - arch: "amd64" runner: ubuntu-latest - - platform: linux/arm64 + platform: linux/amd64 + - arch: "arm64" runner: ubuntu-24.04-arm + platform: linux/arm64 - application: "sample_app" image_name: "fides-sample-app" build_context: "clients/sample-app" @@ -118,7 +120,7 @@ jobs: - name: Upload digest uses: actions/upload-artifact@v4 with: - name: digests-${{ matrix.application }}-${{ matrix.platform }} + name: digests-${{ matrix.application }}-${{ matrix.arch }} path: /tmp/digests/* if-no-files-found: error retention-days: 1 From 46a7d07017b93ccc0233583cfecf0ea72061b78f Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Apr 2025 14:23:19 -0700 Subject: [PATCH 05/14] Adding push and name-canonical --- .github/workflows/publish_docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index 7828e0ead7b..d695c5f230b 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -109,7 +109,7 @@ jobs: target: ${{ matrix.build_target }} platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }},push-by-digest=true + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=true - name: Export digest run: | From ffde7846df93f358906d97e9094c07b0ac95d7cf Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Apr 2025 14:40:00 -0700 Subject: [PATCH 06/14] Trying runner.temp and without name --- .github/workflows/publish_docker.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index d695c5f230b..3ccc2b9f8d8 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -109,19 +109,19 @@ jobs: target: ${{ matrix.build_target }} platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=true + outputs: type=image,push-by-digest=true,name-canonical=true,push=true - name: Export digest run: | - mkdir -p /tmp/digests + mkdir -p ${{ runner.temp }}/digests digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" - name: Upload digest uses: actions/upload-artifact@v4 with: name: digests-${{ matrix.application }}-${{ matrix.arch }} - path: /tmp/digests/* + path: ${{ runner.temp }}/digests/* if-no-files-found: error retention-days: 1 @@ -138,7 +138,7 @@ jobs: - name: Download digests uses: actions/download-artifact@v4 with: - path: /tmp/digests + path: ${{ runner.temp }}/digests pattern: digests-${{ matrix.application }}-* merge-multiple: true @@ -153,28 +153,28 @@ jobs: - name: Create manifest lists and push (dev) if: needs.ParseTags.outputs.prod_tag == 'false' && needs.ParseTags.outputs.rc_tag == 'false' && needs.ParseTags.outputs.beta_tag == 'false' && needs.ParseTags.outputs.alpha_tag == 'false' - working-directory: /tmp/digests + working-directory: ${{ runner.temp }}/digests run: | docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '{"tags":["${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:dev"]}') \ $(ls -1 | sort | sed -e 's/^/${{ env.REGISTRY }}\/${{ env.IMAGE_NAME }}\/${{ matrix.image_name }}@sha256:/') - name: Create manifest lists and push (prod) if: needs.ParseTags.outputs.prod_tag == 'true' - working-directory: /tmp/digests + working-directory: ${{ runner.temp }}/digests run: | docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '{"tags":["${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}","${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:latest"]}') \ $(ls -1 | sort | sed -e 's/^/${{ env.REGISTRY }}\/${{ env.IMAGE_NAME }}\/${{ matrix.image_name }}@sha256:/') - name: Create manifest lists and push (rc) if: needs.ParseTags.outputs.rc_tag == 'true' - working-directory: /tmp/digests + working-directory: ${{ runner.temp }}/digests run: | docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '{"tags":["${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}","${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:rc"]}') \ $(ls -1 | sort | sed -e 's/^/${{ env.REGISTRY }}\/${{ env.IMAGE_NAME }}\/${{ matrix.image_name }}@sha256:/') - name: Create manifest lists and push (prerelease) if: needs.ParseTags.outputs.alpha_tag == 'true' || needs.ParseTags.outputs.beta_tag == 'true' - working-directory: /tmp/digests + working-directory: ${{ runner.temp }}/digests run: | docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '{"tags":["${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}","${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:prerelease"]}') \ $(ls -1 | sort | sed -e 's/^/${{ env.REGISTRY }}\/${{ env.IMAGE_NAME }}\/${{ matrix.image_name }}@sha256:/') From d3dbb5189c579d1b665c6ef5aa19164eca6a5fbb Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Apr 2025 14:41:59 -0700 Subject: [PATCH 07/14] Re-adding name --- .github/workflows/publish_docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index 3ccc2b9f8d8..64124367292 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -109,7 +109,7 @@ jobs: target: ${{ matrix.build_target }} platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,push-by-digest=true,name-canonical=true,push=true + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=true - name: Export digest run: | From 094c7e7eadad6bc9b45c6b2f4ca768c920374568 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Apr 2025 15:12:16 -0700 Subject: [PATCH 08/14] Testing cleanup --- .github/workflows/publish_docker.yaml | 53 +++++++++++++-------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index 64124367292..219d7c3f62c 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -12,7 +12,7 @@ env: DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} DEFAULT_PYTHON_VERSION: "3.10.16" REGISTRY: docker.io - IMAGE_NAME: thegalvanizer + REPO_NAME: thegalvanizer jobs: ParseTags: @@ -24,7 +24,6 @@ jobs: beta_tag: ${{ steps.check-beta-tag.outputs.match }} version: ${{ github.ref_name }} steps: - # Your existing tag parsing steps remain the same - name: Check Prod Tag id: check-prod-tag run: | @@ -61,7 +60,7 @@ jobs: echo "match=false" >> $GITHUB_OUTPUT fi - BuildAndPush: + Push: needs: ParseTags strategy: fail-fast: false @@ -90,7 +89,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }} + images: ${{ env.REGISTRY }}/${{ env.REPO_NAME }}/${{ matrix.image_name }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -109,7 +108,7 @@ jobs: target: ${{ matrix.build_target }} platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=true + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.REPO_NAME }}/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=true - name: Export digest run: | @@ -151,33 +150,31 @@ jobs: username: ${{ env.DOCKER_USER }} password: ${{ env.DOCKER_TOKEN }} - - name: Create manifest lists and push (dev) - if: needs.ParseTags.outputs.prod_tag == 'false' && needs.ParseTags.outputs.rc_tag == 'false' && needs.ParseTags.outputs.beta_tag == 'false' && needs.ParseTags.outputs.alpha_tag == 'false' - working-directory: ${{ runner.temp }}/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '{"tags":["${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:dev"]}') \ - $(ls -1 | sort | sed -e 's/^/${{ env.REGISTRY }}\/${{ env.IMAGE_NAME }}\/${{ matrix.image_name }}@sha256:/') - - - name: Create manifest lists and push (prod) - if: needs.ParseTags.outputs.prod_tag == 'true' - working-directory: ${{ runner.temp }}/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '{"tags":["${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}","${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:latest"]}') \ - $(ls -1 | sort | sed -e 's/^/${{ env.REGISTRY }}\/${{ env.IMAGE_NAME }}\/${{ matrix.image_name }}@sha256:/') - - - name: Create manifest lists and push (rc) - if: needs.ParseTags.outputs.rc_tag == 'true' + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.REGISTRY }}/${{ env.REPO_NAME }}/${{ matrix.image_name }} + tags: | + type=raw,value=dev,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=latest,enable=${{ needs.ParseTags.outputs.prod_tag == 'true' }} + type=raw,value=rc,enable=${{ needs.ParseTags.outputs.rc_tag == 'true' }} + type=raw,value=prerelease,enable=${{ needs.ParseTags.outputs.alpha_tag == 'true' || needs.ParseTags.outputs.beta_tag == 'true' }} + type=ref,event=tag + type=semver,pattern={{version}} + + - name: Create manifest list and push working-directory: ${{ runner.temp }}/digests run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '{"tags":["${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}","${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:rc"]}') \ - $(ls -1 | sort | sed -e 's/^/${{ env.REGISTRY }}\/${{ env.IMAGE_NAME }}\/${{ matrix.image_name }}@sha256:/') + REGISTRY_IMAGE="${{ env.REGISTRY }}/${{ env.REPO_NAME }}/${{ matrix.image_name }}" + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '$REGISTRY_IMAGE@sha256:%s ' *) - - name: Create manifest lists and push (prerelease) - if: needs.ParseTags.outputs.alpha_tag == 'true' || needs.ParseTags.outputs.beta_tag == 'true' - working-directory: ${{ runner.temp }}/digests + - name: Inspect image run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '{"tags":["${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:${{ needs.ParseTags.outputs.version }}","${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.image_name }}:prerelease"]}') \ - $(ls -1 | sort | sed -e 's/^/${{ env.REGISTRY }}\/${{ env.IMAGE_NAME }}\/${{ matrix.image_name }}@sha256:/') + REGISTRY_IMAGE="${{ env.REGISTRY }}/${{ env.REPO_NAME }}/${{ matrix.image_name }}" + docker buildx imagetools inspect $REGISTRY_IMAGE:${{ steps.meta.outputs.version }} NotifyRedeploy: runs-on: ubuntu-latest From 0c5ff1f77a830d70b4e5026b39efe5b410670374 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Apr 2025 15:14:13 -0700 Subject: [PATCH 09/14] Fixing dependencies --- .github/workflows/publish_docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index 219d7c3f62c..0b317cf4eb1 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -125,7 +125,7 @@ jobs: retention-days: 1 CreateManifests: - needs: [ParseTags, BuildAndPush] + needs: [ParseTags, Push] strategy: matrix: application: ["sample_app"] # Expand to ["fides", "sample_app", "privacy_center"] as needed From 71daa362f52030fa1531719308e596a5a0ca4306 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Apr 2025 15:19:44 -0700 Subject: [PATCH 10/14] Fixing variable syntax --- .github/workflows/publish_docker.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index 0b317cf4eb1..2967a75f048 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -169,12 +169,12 @@ jobs: run: | REGISTRY_IMAGE="${{ env.REGISTRY }}/${{ env.REPO_NAME }}/${{ matrix.image_name }}" docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '$REGISTRY_IMAGE@sha256:%s ' *) + $(printf "${REGISTRY_IMAGE}@sha256:%s " *) - name: Inspect image run: | REGISTRY_IMAGE="${{ env.REGISTRY }}/${{ env.REPO_NAME }}/${{ matrix.image_name }}" - docker buildx imagetools inspect $REGISTRY_IMAGE:${{ steps.meta.outputs.version }} + docker buildx imagetools inspect ${REGISTRY_IMAGE}:${{ steps.meta.outputs.version }} NotifyRedeploy: runs-on: ubuntu-latest From 6ab5236e18d0e9ca2d0a43ce6be08785e4c31ea6 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Apr 2025 15:43:21 -0700 Subject: [PATCH 11/14] Tag fix --- .github/workflows/publish_docker.yaml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index 2967a75f048..5cd671c7874 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -157,12 +157,28 @@ jobs: images: | ${{ env.REGISTRY }}/${{ env.REPO_NAME }}/${{ matrix.image_name }} tags: | - type=raw,value=dev,enable=${{ github.ref == 'refs/heads/main' }} + # When the GitHub ref is a tag that matches the production pattern + # Tag as both the specific version and "latest" + type=semver,pattern={{version}},enable=${{ needs.ParseTags.outputs.prod_tag == 'true' }} type=raw,value=latest,enable=${{ needs.ParseTags.outputs.prod_tag == 'true' }} + + # When the GitHub ref is a tag that matches the RC pattern + # Tag as both the specific version and "rc" + type=semver,pattern={{version}},enable=${{ needs.ParseTags.outputs.rc_tag == 'true' }} type=raw,value=rc,enable=${{ needs.ParseTags.outputs.rc_tag == 'true' }} + + # When the GitHub ref is a tag that matches the alpha or beta pattern + # Tag as both the specific version and "prerelease" + type=semver,pattern={{version}},enable=${{ needs.ParseTags.outputs.alpha_tag == 'true' || needs.ParseTags.outputs.beta_tag == 'true' }} type=raw,value=prerelease,enable=${{ needs.ParseTags.outputs.alpha_tag == 'true' || needs.ParseTags.outputs.beta_tag == 'true' }} - type=ref,event=tag - type=semver,pattern={{version}} + + # When NOT a recognized tag (main branch push) + # Tag as "dev" + type=raw,value=dev,enable=${{ needs.ParseTags.outputs.prod_tag == 'false' && needs.ParseTags.outputs.rc_tag == 'false' && needs.ParseTags.outputs.beta_tag == 'false' && needs.ParseTags.outputs.alpha_tag == 'false' }} + + - name: Debug metadata tags + run: | + echo "Generated tags: ${{ steps.meta.outputs.tags }}" - name: Create manifest list and push working-directory: ${{ runner.temp }}/digests From 277797b9b734ee5a6a227fc854a75936106ca8bd Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Apr 2025 15:51:40 -0700 Subject: [PATCH 12/14] Adding missing tags --- .github/workflows/publish_docker.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index 5cd671c7874..389e910f8fb 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -159,17 +159,17 @@ jobs: tags: | # When the GitHub ref is a tag that matches the production pattern # Tag as both the specific version and "latest" - type=semver,pattern={{version}},enable=${{ needs.ParseTags.outputs.prod_tag == 'true' }} + type=raw,value=${{ needs.ParseTags.outputs.version }},enable=${{ needs.ParseTags.outputs.prod_tag == 'true' }} type=raw,value=latest,enable=${{ needs.ParseTags.outputs.prod_tag == 'true' }} # When the GitHub ref is a tag that matches the RC pattern # Tag as both the specific version and "rc" - type=semver,pattern={{version}},enable=${{ needs.ParseTags.outputs.rc_tag == 'true' }} + type=raw,value=${{ needs.ParseTags.outputs.version }},enable=${{ needs.ParseTags.outputs.rc_tag == 'true' }} type=raw,value=rc,enable=${{ needs.ParseTags.outputs.rc_tag == 'true' }} # When the GitHub ref is a tag that matches the alpha or beta pattern # Tag as both the specific version and "prerelease" - type=semver,pattern={{version}},enable=${{ needs.ParseTags.outputs.alpha_tag == 'true' || needs.ParseTags.outputs.beta_tag == 'true' }} + type=raw,value=${{ needs.ParseTags.outputs.version }},enable=${{ needs.ParseTags.outputs.alpha_tag == 'true' || needs.ParseTags.outputs.beta_tag == 'true' }} type=raw,value=prerelease,enable=${{ needs.ParseTags.outputs.alpha_tag == 'true' || needs.ParseTags.outputs.beta_tag == 'true' }} # When NOT a recognized tag (main branch push) From a77899ac28ea8cd1898d755336082ca9e4919c45 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 23 Apr 2025 16:21:37 -0700 Subject: [PATCH 13/14] Full run --- .github/workflows/publish_docker.yaml | 34 ++++++++++++--------------- noxfiles/docker_nox.py | 1 - noxfiles/test_docker_nox.py | 3 --- 3 files changed, 15 insertions(+), 23 deletions(-) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index 389e910f8fb..a67b589af7e 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -65,7 +65,7 @@ jobs: strategy: fail-fast: false matrix: - application: ["sample_app"] # Expand to ["fides", "sample_app", "privacy_center"] as needed + application: ["fides", "privacy_center", "sample_app"] arch: ["amd64", "arm64"] include: - arch: "amd64" @@ -74,6 +74,14 @@ jobs: - arch: "arm64" runner: ubuntu-24.04-arm platform: linux/arm64 + - application: "fides" + image_name: "fides" + build_context: "." + build_target: "prod" + - application: "privacy_center" + image_name: "fides-privacy-center" + build_context: "." + build_target: "prod_pc" - application: "sample_app" image_name: "fides-sample-app" build_context: "clients/sample-app" @@ -83,13 +91,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - fetch-depth: 0 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.REPO_NAME }}/${{ matrix.image_name }} + fetch-depth: 0 # This is required to properly tag images - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -107,7 +109,6 @@ jobs: context: ${{ matrix.build_context }} target: ${{ matrix.build_target }} platforms: ${{ matrix.platform }} - labels: ${{ steps.meta.outputs.labels }} outputs: type=image,name=${{ env.REGISTRY }}/${{ env.REPO_NAME }}/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=true - name: Export digest @@ -128,8 +129,12 @@ jobs: needs: [ParseTags, Push] strategy: matrix: - application: ["sample_app"] # Expand to ["fides", "sample_app", "privacy_center"] as needed + application: ["fides", "privacy_center", "sample_app"] include: + - application: "fides" + image_name: "fides" + - application: "privacy_center" + image_name: "fides-privacy-center" - application: "sample_app" image_name: "fides-sample-app" runs-on: ubuntu-latest @@ -176,10 +181,6 @@ jobs: # Tag as "dev" type=raw,value=dev,enable=${{ needs.ParseTags.outputs.prod_tag == 'false' && needs.ParseTags.outputs.rc_tag == 'false' && needs.ParseTags.outputs.beta_tag == 'false' && needs.ParseTags.outputs.alpha_tag == 'false' }} - - name: Debug metadata tags - run: | - echo "Generated tags: ${{ steps.meta.outputs.tags }}" - - name: Create manifest list and push working-directory: ${{ runner.temp }}/digests run: | @@ -187,11 +188,6 @@ jobs: docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ $(printf "${REGISTRY_IMAGE}@sha256:%s " *) - - name: Inspect image - run: | - REGISTRY_IMAGE="${{ env.REGISTRY }}/${{ env.REPO_NAME }}/${{ matrix.image_name }}" - docker buildx imagetools inspect ${REGISTRY_IMAGE}:${{ steps.meta.outputs.version }} - NotifyRedeploy: runs-on: ubuntu-latest needs: [CreateManifests] diff --git a/noxfiles/docker_nox.py b/noxfiles/docker_nox.py index 7747e8a2967..c836b985e8e 100644 --- a/noxfiles/docker_nox.py +++ b/noxfiles/docker_nox.py @@ -61,7 +61,6 @@ def generate_buildx_command( "buildx", "build", "--push", - "--provenance=false", f"--target={docker_build_target}", "--platform", DOCKER_PLATFORMS, diff --git a/noxfiles/test_docker_nox.py b/noxfiles/test_docker_nox.py index 5d4eb9b76d7..b410dbca637 100644 --- a/noxfiles/test_docker_nox.py +++ b/noxfiles/test_docker_nox.py @@ -13,7 +13,6 @@ def test_single_tag(self) -> None: "buildx", "build", "--push", - "--provenance=false", "--target=prod", "--platform", "linux/amd64,linux/arm64", @@ -34,7 +33,6 @@ def test_multiplte_tags(self) -> None: "buildx", "build", "--push", - "--provenance=false", "--target=prod", "--platform", "linux/amd64,linux/arm64", @@ -57,7 +55,6 @@ def test_different_path(self) -> None: "buildx", "build", "--push", - "--provenance=false", "--target=prod", "--platform", "linux/amd64,linux/arm64", From a25852f3026ab325262562255c7aa1c1c26af826 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 24 Apr 2025 09:40:49 -0700 Subject: [PATCH 14/14] Adding comments --- .github/workflows/publish_docker.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index a67b589af7e..8f16c247d1e 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -62,6 +62,8 @@ jobs: Push: needs: ParseTags + # This job builds platform-specific images and uploads their digests + # Each runner (amd64/arm64) builds its own platform image in isolation strategy: fail-fast: false matrix: @@ -102,6 +104,8 @@ jobs: username: ${{ env.DOCKER_USER }} password: ${{ env.DOCKER_TOKEN }} + # Build and push the image by digest only (no tags yet) + # Each platform (amd64/arm64) is built separately on appropriate runners - name: Build and push by digest id: build uses: docker/build-push-action@v6 @@ -111,12 +115,15 @@ jobs: platforms: ${{ matrix.platform }} outputs: type=image,name=${{ env.REGISTRY }}/${{ env.REPO_NAME }}/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=true + # Extract the image digest to make it available for the CreateManifests job - name: Export digest run: | mkdir -p ${{ runner.temp }}/digests digest="${{ steps.build.outputs.digest }}" touch "${{ runner.temp }}/digests/${digest#sha256:}" + # Share the digest between jobs via artifacts + # This is necessary because each runner (amd64/arm64) has its own isolated environment - name: Upload digest uses: actions/upload-artifact@v4 with: @@ -127,6 +134,8 @@ jobs: CreateManifests: needs: [ParseTags, Push] + # This job combines the individual platform images into multi-arch manifests + # It downloads digests from all platform-specific builds and creates tagged manifests strategy: matrix: application: ["fides", "privacy_center", "sample_app"] @@ -139,6 +148,8 @@ jobs: image_name: "fides-sample-app" runs-on: ubuntu-latest steps: + # Download all platform-specific digests built in the Push job + # These were uploaded as artifacts from each platform-specific runner - name: Download digests uses: actions/download-artifact@v4 with: @@ -181,6 +192,8 @@ jobs: # Tag as "dev" type=raw,value=dev,enable=${{ needs.ParseTags.outputs.prod_tag == 'false' && needs.ParseTags.outputs.rc_tag == 'false' && needs.ParseTags.outputs.beta_tag == 'false' && needs.ParseTags.outputs.alpha_tag == 'false' }} + # Create manifest lists by combining platform-specific image digests + # This step applies the tags generated by Docker meta to the multi-arch images - name: Create manifest list and push working-directory: ${{ runner.temp }}/digests run: |