From 97b0b3c81d650eced2b9175201d0d5036572d52f Mon Sep 17 00:00:00 2001 From: taddyb Date: Thu, 2 Oct 2025 13:50:41 -0400 Subject: [PATCH] initial commit: workflow update --- .github/workflows/cicd.yaml | 150 ++++++++++++++++++++++++++++++++++++ Dockerfile.github | 35 +++++++++ 2 files changed, 185 insertions(+) create mode 100644 .github/workflows/cicd.yaml create mode 100644 Dockerfile.github diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml new file mode 100644 index 0000000..639f44c --- /dev/null +++ b/.github/workflows/cicd.yaml @@ -0,0 +1,150 @@ +name: CI/CD Pipeline + +on: + pull_request: + branches: [main, development] + push: + branches: [main, development] + tags: ['v*.*.*'] + release: + types: [published, created] + +permissions: + contents: read + packages: write + security-events: write + +env: + REGISTRY: ghcr.io + PYTHON_VERSION: '3.13' + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + image_base: ${{ steps.vars.outputs.image_base }} + pr_tag: ${{ steps.vars.outputs.pr_tag }} + commit_sha: ${{ steps.vars.outputs.commit_sha }} + commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }} + test_image_tag: ${{ steps.vars.outputs.test_image_tag }} + steps: + - name: Compute image vars + id: vars + shell: bash + run: | + set -euo pipefail + ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" + REPO="$(basename "${GITHUB_REPOSITORY}")" + IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}" + echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT" + + if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then + PR_NUM="${{ github.event.pull_request.number }}" + PR_TAG="pr-${PR_NUM}-build" + echo "pr_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" + echo "test_image_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" + fi + + if [ "${GITHUB_EVENT_NAME}" = "push" ]; then + COMMIT_SHA="${GITHUB_SHA}" + SHORT_SHA="${COMMIT_SHA:0:12}" + echo "commit_sha=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" + echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + echo "test_image_tag=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + fi + + build-and-scan-hf-api: + name: Build Hydrofabric-API Container + runs-on: ubuntu-latest + needs: setup + steps: + - uses: actions/checkout@v4 + - name: Build hf-api image for scanning + id: build-hf-api + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile.github + # Load the image to the local Docker daemon, but do not push it + load: true + tags: ${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.test_image_tag }} + - name: Scan ingest container with Trivy + uses: aquasecurity/trivy-action@0.20.0 + with: + # Scan the locally available image + image-ref: ${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.test_image_tag }} + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results-ingest.sarif' + severity: 'CRITICAL,HIGH' + - name: Upload Ingest Trivy SARIF + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results-ingest.sarif' + category: 'ingest-container' + + codeql-scan: + name: CodeQL Scan + if: github.event_name == 'pull_request' || github.event_name == 'push' + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: python + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + + publish-hf-api: + name: Publish Hydrofabric API + if: > + github.event_name == 'push' && ( + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/development' || + startsWith(github.ref, 'refs/tags/v') + ) + runs-on: ubuntu-latest + needs: [setup, build-and-scan-hf-api, codeql-scan] + steps: + - uses: actions/checkout@v4 + - name: Prepare ingest image tags + id: prep_tags + run: | + # Always start with the unique commit SHA tag for traceability + TAGS="${{ needs.setup.outputs.image_base }}/ingest:${{ needs.setup.outputs.commit_sha_short }}" + + # If it's a push to the main branch, also add the 'latest' tag + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/ingest:latest" + fi + + # If the trigger was a version tag, add that version as a tag + if [[ "${{ github.ref }}" == refs/tags/v* ]]; then + # github.ref_name holds the tag name (e.g., "v1.0.0") + VERSION_TAG=${{ github.ref_name }} + TAGS="$TAGS,${{ needs.setup.outputs.image_base }}/ingest:${VERSION_TAG}" + fi + + echo "tags=${TAGS}" >> "$GITHUB_OUTPUT" + - name: Log in to registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build & push ingest image + uses: docker/build-push-action@v6 + with: + context: ./Source/Ingest + file: ./Source/Ingest/docker/Dockerfile.ingest + push: true + tags: ${{ steps.prep_tags.outputs.tags }} diff --git a/Dockerfile.github b/Dockerfile.github new file mode 100644 index 0000000..4061ad4 --- /dev/null +++ b/Dockerfile.github @@ -0,0 +1,35 @@ +FROM ghcr.io/ngwpc/hydrofabric_api/hydrofabric:base + +# Update packages +RUN dnf upgrade -y + +# Install Python, pip, and Git +RUN dnf install -y python311 python3.11-pip python3.11-devel + +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 \ + && dnf remove pip -y \ + && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3.11 1 + +# Copy application files +COPY ./djangoApps /app/djangoApps +COPY ./R /app/R + +COPY requirements.txt /app +COPY config.yml /app/config.yml +COPY VERSION /app/VERSION + +# We probably don't need these tests in the built image long term. +COPY ./tests /app/tests + +# Install Python dependencies +RUN pip install -r /app/requirements.txt + +# Set library path and expose port +ENV LD_LIBRARY_PATH=/usr/local/lib64 +EXPOSE 8000 + +# Set the working directory +WORKDIR /app/djangoApps/ + +# Start the application +CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]