From 2a0342c1b09dcdc833f8c8b6574f3bd8ca2dff67 Mon Sep 17 00:00:00 2001 From: Maurice Atrops Date: Sat, 10 May 2025 14:54:44 +0200 Subject: [PATCH 01/11] feat: upload docker image as artifact --- .github/workflows/ci-pipeline.yaml | 40 ++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-pipeline.yaml b/.github/workflows/ci-pipeline.yaml index 71ec0e9..5df368a 100644 --- a/.github/workflows/ci-pipeline.yaml +++ b/.github/workflows/ci-pipeline.yaml @@ -61,7 +61,8 @@ jobs: actions: read contents: read env: - PROJECT_IMAGE_NAME: tmp-image + PROJECT_IMAGE_NAME: shortwave6046/tmp-image:latest # TODO: Secret! + IMAGE_TARBALL_PATH: ${{ format('{0}/built_image.tar', runner.temp }} TRIVY_REPORT_DIR: /tmp/trivy_reports steps: - name: Checkout Repository @@ -77,10 +78,18 @@ jobs: echo "deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main" | sudo tee -a /etc/apt/sources.list.d/trivy.list sudo apt-get update sudo apt-get install trivy - + + # Setup a DockerCLI plugin for Build Kit. Used to export the cache + # Recommended in https://github.com/docker/build-push-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build Project Docker Image - run: | - docker build -t $PROJECT_IMAGE_NAME . + uses: docker/build-push-action@v6 + with: + push: false + tags: ${{ env.PROJECT_IMAGE_NAME }} + outputs: type=docker,dest=${{ env.IMAGE_TARBALL_PATH }} - name: Setup Trivy Reports Directory run: | @@ -106,4 +115,25 @@ jobs: with: sarif_file: trivy-dockerfile-report.sarif category: trivy-dockerfile-report - \ No newline at end of file + + - name: Upload Docker Image IMAGE_TARBALL_NAME + uses: actions/upload-artifact@v4 + with: + name: built_image + path: ${{ env.IMAGE_TARBALL_PATH }} + + push-image: + name: Push Docker Image + runs-on: ubuntu-latest + needs: [test-and-check, scan] + if: github.ref_name == 'main' + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Push Docker Image + run: | + echo "Hello World" + # Login to Docker Hub + # Push Image + # TODO: How to handle secrets? From 200b53b91d54621579bbc4c791334b6401deb3fa Mon Sep 17 00:00:00 2001 From: Maurice Atrops Date: Sat, 10 May 2025 14:55:40 +0200 Subject: [PATCH 02/11] feat: upload docker image as artifact --- .github/workflows/ci-pipeline.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pipeline.yaml b/.github/workflows/ci-pipeline.yaml index 5df368a..19f6b94 100644 --- a/.github/workflows/ci-pipeline.yaml +++ b/.github/workflows/ci-pipeline.yaml @@ -88,6 +88,7 @@ jobs: uses: docker/build-push-action@v6 with: push: false + load: true # Image is available locally for Trivy tags: ${{ env.PROJECT_IMAGE_NAME }} outputs: type=docker,dest=${{ env.IMAGE_TARBALL_PATH }} From f92cb48f8cd91a8e855c698328d2f61c5596ed68 Mon Sep 17 00:00:00 2001 From: Maurice Atrops Date: Sat, 10 May 2025 14:58:41 +0200 Subject: [PATCH 03/11] fix: tarball path --- .github/workflows/ci-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yaml b/.github/workflows/ci-pipeline.yaml index 19f6b94..b7eaa34 100644 --- a/.github/workflows/ci-pipeline.yaml +++ b/.github/workflows/ci-pipeline.yaml @@ -62,7 +62,7 @@ jobs: contents: read env: PROJECT_IMAGE_NAME: shortwave6046/tmp-image:latest # TODO: Secret! - IMAGE_TARBALL_PATH: ${{ format('{0}/built_image.tar', runner.temp }} + IMAGE_TARBALL_PATH: /tmp/built_image.tar' TRIVY_REPORT_DIR: /tmp/trivy_reports steps: - name: Checkout Repository From 663f8c97ba769ff7be133eed26e3fc643b3da14b Mon Sep 17 00:00:00 2001 From: Maurice Atrops Date: Sat, 10 May 2025 15:05:03 +0200 Subject: [PATCH 04/11] debug --- .github/workflows/ci-pipeline.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci-pipeline.yaml b/.github/workflows/ci-pipeline.yaml index b7eaa34..fe408aa 100644 --- a/.github/workflows/ci-pipeline.yaml +++ b/.github/workflows/ci-pipeline.yaml @@ -91,6 +91,10 @@ jobs: load: true # Image is available locally for Trivy tags: ${{ env.PROJECT_IMAGE_NAME }} outputs: type=docker,dest=${{ env.IMAGE_TARBALL_PATH }} + + - name: Test docker image + run: | + docker images - name: Setup Trivy Reports Directory run: | From e05638cb6fe28698e0b37c5f7da4e078f4e17d57 Mon Sep 17 00:00:00 2001 From: Maurice Atrops Date: Sat, 10 May 2025 15:14:49 +0200 Subject: [PATCH 05/11] debug --- .github/workflows/ci-pipeline.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-pipeline.yaml b/.github/workflows/ci-pipeline.yaml index fe408aa..1332dc8 100644 --- a/.github/workflows/ci-pipeline.yaml +++ b/.github/workflows/ci-pipeline.yaml @@ -62,7 +62,7 @@ jobs: contents: read env: PROJECT_IMAGE_NAME: shortwave6046/tmp-image:latest # TODO: Secret! - IMAGE_TARBALL_PATH: /tmp/built_image.tar' + IMAGE_TARBALL_PATH: /tmp/built_image.tar TRIVY_REPORT_DIR: /tmp/trivy_reports steps: - name: Checkout Repository @@ -87,14 +87,14 @@ jobs: - name: Build Project Docker Image uses: docker/build-push-action@v6 with: - push: false load: true # Image is available locally for Trivy tags: ${{ env.PROJECT_IMAGE_NAME }} - outputs: type=docker,dest=${{ env.IMAGE_TARBALL_PATH }} + # outputs: type=docker,dest=${{ env.IMAGE_TARBALL_PATH }} - name: Test docker image run: | docker images + ls -lisa /tmp/*.tar - name: Setup Trivy Reports Directory run: | From 1420732662db914cf2a29e47fce10454c3264cc9 Mon Sep 17 00:00:00 2001 From: Maurice Atrops Date: Sat, 10 May 2025 15:18:28 +0200 Subject: [PATCH 06/11] debug --- .github/workflows/ci-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yaml b/.github/workflows/ci-pipeline.yaml index 1332dc8..e7e7c63 100644 --- a/.github/workflows/ci-pipeline.yaml +++ b/.github/workflows/ci-pipeline.yaml @@ -94,7 +94,7 @@ jobs: - name: Test docker image run: | docker images - ls -lisa /tmp/*.tar + ls -lisa /tmp/ - name: Setup Trivy Reports Directory run: | From e054e9fbcc484244f9c376e872f66d07f5b0f2da Mon Sep 17 00:00:00 2001 From: Maurice Atrops Date: Sat, 10 May 2025 15:23:11 +0200 Subject: [PATCH 07/11] debug --- .github/workflows/ci-pipeline.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yaml b/.github/workflows/ci-pipeline.yaml index e7e7c63..9e45ae8 100644 --- a/.github/workflows/ci-pipeline.yaml +++ b/.github/workflows/ci-pipeline.yaml @@ -89,7 +89,9 @@ jobs: with: load: true # Image is available locally for Trivy tags: ${{ env.PROJECT_IMAGE_NAME }} - # outputs: type=docker,dest=${{ env.IMAGE_TARBALL_PATH }} + outputs: | + type=docker + type=docker,dest=${{ env.IMAGE_TARBALL_PATH }} - name: Test docker image run: | From 138eeaf37a007355680c3fb5d94a40ad6fa1acce Mon Sep 17 00:00:00 2001 From: Maurice Atrops Date: Sat, 10 May 2025 15:48:55 +0200 Subject: [PATCH 08/11] feat: consolidate into two steps; push to dockerhub --- .github/workflows/ci-pipeline.yaml | 53 +++++++++++------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci-pipeline.yaml b/.github/workflows/ci-pipeline.yaml index 9e45ae8..47f5dd8 100644 --- a/.github/workflows/ci-pipeline.yaml +++ b/.github/workflows/ci-pipeline.yaml @@ -13,6 +13,8 @@ on: env: PYTHON_IMAGE: 'python:3.12-slim' POETRY_VERSION: 2.1.2 + DOCKER_IMAGE_ARTIFACT_NAME: built_image + DOCKER_BUILD_SUMMARY: false # Deactivate build summary generation jobs: test-and-check: @@ -52,8 +54,8 @@ jobs: $POETRY_HOME/bin/poetry run bandit src/ -r -ll # Report medium vulnerabilities or higher # TODO (matrops): One could invest more effort here to generate SARIF files and integrate them in GitHub Security Monitoring - scan: - name: Scan Docker Resources + scan-and-push: + name: Scan and Push Docker Resources runs-on: ubuntu-latest permissions: # Needed for SARIF upload @@ -61,9 +63,10 @@ jobs: actions: read contents: read env: - PROJECT_IMAGE_NAME: shortwave6046/tmp-image:latest # TODO: Secret! - IMAGE_TARBALL_PATH: /tmp/built_image.tar + BUILD_IMAGE_NAME: tmp-image:latest + PUSH_IMAGE_NAME: python-cicd:latest TRIVY_REPORT_DIR: /tmp/trivy_reports + needs: test-and-check steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -87,16 +90,8 @@ jobs: - name: Build Project Docker Image uses: docker/build-push-action@v6 with: - load: true # Image is available locally for Trivy - tags: ${{ env.PROJECT_IMAGE_NAME }} - outputs: | - type=docker - type=docker,dest=${{ env.IMAGE_TARBALL_PATH }} - - - name: Test docker image - run: | - docker images - ls -lisa /tmp/ + load: true + tags: ${{ env.BUILD_IMAGE_NAME }} - name: Setup Trivy Reports Directory run: | @@ -105,7 +100,7 @@ jobs: - name: Run Trivy Image Scan run: | - trivy image --severity HIGH,CRITICAL --format sarif -o trivy-image-report.sarif $PROJECT_IMAGE_NAME + trivy image --severity HIGH,CRITICAL --format sarif -o trivy-image-report.sarif $BUILD_IMAGE_NAME - name: Upload Trivy Image Scan Result uses: github/codeql-action/upload-sarif@v3 @@ -123,24 +118,16 @@ jobs: sarif_file: trivy-dockerfile-report.sarif category: trivy-dockerfile-report - - name: Upload Docker Image IMAGE_TARBALL_NAME - uses: actions/upload-artifact@v4 + - name: Log in to Docker Hub + # if: github.ref_name == 'main' + uses: docker/login-action@v3 with: - name: built_image - path: ${{ env.IMAGE_TARBALL_PATH }} - - push-image: - name: Push Docker Image - runs-on: ubuntu-latest - needs: [test-and-check, scan] - if: github.ref_name == 'main' - steps: - - name: Checkout Repository - uses: actions/checkout@v4 + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Push Docker Image - run: | - echo "Hello World" - # Login to Docker Hub - # Push Image - # TODO: How to handle secrets? + # if: github.ref_name == 'main' + uses: docker/build-push-action@v6 + with: + push: true + tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.PUSH_IMAGE_NAME }} \ No newline at end of file From 44c54cb43dea5ac1d0f95549c86e8562a0ea8c90 Mon Sep 17 00:00:00 2001 From: Maurice Atrops Date: Sat, 10 May 2025 16:08:50 +0200 Subject: [PATCH 09/11] feat: add image versioning --- .github/workflows/ci-pipeline.yaml | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-pipeline.yaml b/.github/workflows/ci-pipeline.yaml index 47f5dd8..1b497b7 100644 --- a/.github/workflows/ci-pipeline.yaml +++ b/.github/workflows/ci-pipeline.yaml @@ -49,10 +49,10 @@ jobs: run: | $POETRY_HOME/bin/poetry run mypy src/ + # TODO (matrops): One could invest more effort here to generate SARIF files and integrate them in GitHub Security Monitoring - name: Run SAST run: | $POETRY_HOME/bin/poetry run bandit src/ -r -ll # Report medium vulnerabilities or higher - # TODO (matrops): One could invest more effort here to generate SARIF files and integrate them in GitHub Security Monitoring scan-and-push: name: Scan and Push Docker Resources @@ -64,17 +64,17 @@ jobs: contents: read env: BUILD_IMAGE_NAME: tmp-image:latest - PUSH_IMAGE_NAME: python-cicd:latest + PUSH_IMAGE_NAME: python-cicd TRIVY_REPORT_DIR: /tmp/trivy_reports needs: test-and-check steps: - name: Checkout Repository uses: actions/checkout@v4 + # I'm aware that Trivy has a GitHub Action, this is just for learning purposes - name: Install Trivy run: | # According to https://trivy.dev/latest/getting-started/installation/#debianubuntu-official - # Hint: I'm aware that Trivy has a GitHub Action, this is just for learning purposes sudo apt-get update sudo apt-get install wget gnupg wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | sudo tee /usr/share/keyrings/trivy.gpg > /dev/null @@ -118,16 +118,33 @@ jobs: sarif_file: trivy-dockerfile-report.sarif category: trivy-dockerfile-report - - name: Log in to Docker Hub + # === Push Docker Image to registry === + # My first approach was separating the Image Test and Push Stages, however this created too much hassle. + # Reason was that I wanted to avoid building the image twice. However, splitting the build and the push stages + # meant that I had to use artifacts to upload the built image in one stage and download & handle it in another + # I'm sure this would work eventually, however using the current approach was much faster and easier. Only + # downside is that image scanning and code testing is not parallelized anymore, resulting in a slightly + # slower CI/CD runtime. + + - name: Log into Docker Hub # if: github.ref_name == 'main' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Get Image Version + # if: github.ref_name == 'main' + id: get_image_version + run: | + IMAGE_VERSION=$(grep '^version = ' pyproject.toml | awk -F" " {'print $3'} | sed s/\"//g) + echo "image_version=$IMAGE_VERSION" >> $GITHUB_OUTPUT - name: Push Docker Image # if: github.ref_name == 'main' uses: docker/build-push-action@v6 with: push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.PUSH_IMAGE_NAME }} \ No newline at end of file + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.PUSH_IMAGE_NAME }}:${{ steps.get_image_version.outputs.image_version }} + ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.PUSH_IMAGE_NAME }}:latest \ No newline at end of file From 2e6a29be4b60655d2e328b08573aa392a115efe8 Mon Sep 17 00:00:00 2001 From: Maurice Atrops Date: Sat, 10 May 2025 16:14:34 +0200 Subject: [PATCH 10/11] feat: deactivate image pushing for non-main branches --- .github/workflows/ci-pipeline.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-pipeline.yaml b/.github/workflows/ci-pipeline.yaml index 1b497b7..286ac66 100644 --- a/.github/workflows/ci-pipeline.yaml +++ b/.github/workflows/ci-pipeline.yaml @@ -127,21 +127,22 @@ jobs: # slower CI/CD runtime. - name: Log into Docker Hub - # if: github.ref_name == 'main' + if: github.ref_name == 'main' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Get Image Version - # if: github.ref_name == 'main' + if: github.ref_name == 'main' id: get_image_version run: | IMAGE_VERSION=$(grep '^version = ' pyproject.toml | awk -F" " {'print $3'} | sed s/\"//g) + echo "Using image version '$IMAGE_VERSION'" echo "image_version=$IMAGE_VERSION" >> $GITHUB_OUTPUT - name: Push Docker Image - # if: github.ref_name == 'main' + if: github.ref_name == 'main' uses: docker/build-push-action@v6 with: push: true From 1f9a5a5c2779f305fc5cf6580fda39a6f32736f7 Mon Sep 17 00:00:00 2001 From: Maurice Atrops Date: Sat, 10 May 2025 16:20:03 +0200 Subject: [PATCH 11/11] fix: remove unused variables --- .github/workflows/ci-pipeline.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/ci-pipeline.yaml b/.github/workflows/ci-pipeline.yaml index 286ac66..c190b68 100644 --- a/.github/workflows/ci-pipeline.yaml +++ b/.github/workflows/ci-pipeline.yaml @@ -1,19 +1,14 @@ name: Continuous Integration -# Run only on main and feature branches if Python files were changed on: push: branches: - main - 'feature/**' - pull_request: - branches: - - main env: PYTHON_IMAGE: 'python:3.12-slim' POETRY_VERSION: 2.1.2 - DOCKER_IMAGE_ARTIFACT_NAME: built_image DOCKER_BUILD_SUMMARY: false # Deactivate build summary generation jobs: @@ -82,7 +77,6 @@ jobs: sudo apt-get update sudo apt-get install trivy - # Setup a DockerCLI plugin for Build Kit. Used to export the cache # Recommended in https://github.com/docker/build-push-action - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -148,4 +142,4 @@ jobs: push: true tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.PUSH_IMAGE_NAME }}:${{ steps.get_image_version.outputs.image_version }} - ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.PUSH_IMAGE_NAME }}:latest \ No newline at end of file + ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.PUSH_IMAGE_NAME }}:latest