ci.yml

name: CI

on:
  pull_request:
    branches: [main]

concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  packages: read
  pull-requests: read

env:
  JAVA_VERSION: '21'
  NODE_VERSION: '24'
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

jobs:
  detect-changes:
    name: Detect Changes
    runs-on: ubuntu-latest
    permissions:
      pull-requests: read
    outputs:
      any-kotlin: ${{ steps.compute.outputs.any-kotlin }}
      any-frontend: ${{ steps.compute.outputs.any-frontend }}
      any-python: ${{ steps.compute.outputs.any-python }}
      knowledge-ingest-worker: ${{ steps.compute.outputs.knowledge-ingest-worker }}
      agents-login: ${{ steps.compute.outputs.agents-login }}
      any-platform: ${{ steps.compute.outputs.any-platform }}
      run-system-tests: ${{ steps.compute.outputs.run-system-tests }}
      # Space-separated list of service names whose images need rebuilding
      # from the PR source. Anything else is pulled from ghcr :latest by
      # system-tests.yml. Shape is designed to be passed straight into
      # the reusable workflow's `changed-services` input.
      changed-services: ${{ steps.compute.outputs.changed-services }}
      lint-kotlin-matrix: ${{ steps.matrix.outputs.lint-kotlin }}
      test-kotlin-unit-matrix: ${{ steps.matrix.outputs.test-kotlin-unit }}
      test-kotlin-integration-matrix: ${{ steps.matrix.outputs.test-kotlin-integration }}
      verify-coverage-matrix: ${{ steps.matrix.outputs.verify-coverage }}
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: dorny/paths-filter@v4.0.1
        id: filter
        with:
          filters: |
            workflow:
              - '.github/workflows/full.yml'
              - '.github/workflows/system-tests.yml'
            auth-api:
              - 'services/auth-api/**'
            knowledge-api:
              - 'services/knowledge-api/**'
            knowledge-ingest-worker:
              - 'services/knowledge-ingest-worker/**'
            agents-login:
              - 'services/agents-login/**'
            gradle-config:
              - 'gradle/**'
              - 'gradle.properties'
              - 'build.gradle.kts'
              - 'settings.gradle.kts'
              - 'gradlew'
              - 'gradlew.bat'
            auth-ui:
              - 'services/auth-ui/**'
            app-ui:
              - 'services/app-ui/**'
            vue-common:
              - 'libs/vue-common/**'
            frontend-config:
              - 'package.json'
              - 'pnpm-lock.yaml'
              - 'pnpm-workspace.yaml'
              - 'eslint.config.ts'
            system-tests:
              - 'services/system-tests/**'
            infra:
              - 'infra/**'
              - 'docker-compose.yml'
            platform:
              - 'platform/**'
              - '.github/workflows/full.yml'
      - name: Compute change flags
        id: compute
        env:
          FILTER_WORKFLOW: ${{ steps.filter.outputs.workflow }}
          FILTER_GRADLE_CONFIG: ${{ steps.filter.outputs.gradle-config }}
          FILTER_AUTH_API: ${{ steps.filter.outputs.auth-api }}
          FILTER_KNOWLEDGE_API: ${{ steps.filter.outputs.knowledge-api }}
          FILTER_KNOWLEDGE_INGEST_WORKER: ${{ steps.filter.outputs.knowledge-ingest-worker }}
          FILTER_AGENTS_LOGIN: ${{ steps.filter.outputs.agents-login }}
          FILTER_VUE_COMMON: ${{ steps.filter.outputs.vue-common }}
          FILTER_FRONTEND_CONFIG: ${{ steps.filter.outputs.frontend-config }}
          FILTER_AUTH_UI: ${{ steps.filter.outputs.auth-ui }}
          FILTER_APP_UI: ${{ steps.filter.outputs.app-ui }}
          FILTER_SYSTEM_TESTS: ${{ steps.filter.outputs.system-tests }}
          FILTER_INFRA: ${{ steps.filter.outputs.infra }}
          FILTER_PLATFORM: ${{ steps.filter.outputs.platform }}
        run: |
          any() { for v in "$@"; do [[ "$v" == "true" ]] && echo "true" && return; done; echo "false"; }

          KOTLIN_SHARED=$(any "$FILTER_WORKFLOW" "$FILTER_GRADLE_CONFIG")
          FRONTEND_SHARED=$(any "$FILTER_WORKFLOW" "$FILTER_VUE_COMMON" "$FILTER_FRONTEND_CONFIG")

          AUTH_API_KOTLIN=$(any "$KOTLIN_SHARED" "$FILTER_AUTH_API")
          KNOWLEDGE_API_KOTLIN=$(any "$KOTLIN_SHARED" "$FILTER_KNOWLEDGE_API")
          ANY_KOTLIN=$(any "$KOTLIN_SHARED" "$FILTER_AUTH_API" "$FILTER_KNOWLEDGE_API")
          ANY_FRONTEND=$(any "$FRONTEND_SHARED" "$FILTER_AUTH_UI" "$FILTER_APP_UI")
          ANY_SERVICE=$(any "$ANY_KOTLIN" "$ANY_FRONTEND")
          RUN_SYSTEM_TESTS=$(any "$FILTER_WORKFLOW" "$FILTER_SYSTEM_TESTS" "$FILTER_INFRA" "$ANY_SERVICE")
          ANY_PLATFORM=$(any "$FILTER_PLATFORM")

          KNOWLEDGE_INGEST_WORKER=$(any "$FILTER_WORKFLOW" "$FILTER_KNOWLEDGE_INGEST_WORKER")
          ANY_PYTHON=$(any "$KNOWLEDGE_INGEST_WORKER")

          # agents-login is a standalone Node service — not part of the pnpm
          # workspace and not exercised by system-tests (operator-only tool;
          # no browser path to drive through Playwright).
          AGENTS_LOGIN=$(any "$FILTER_WORKFLOW" "$FILTER_AGENTS_LOGIN")

          # Per-service rebuild flags for the reusable system-tests
          # workflow. Backend images are rebuilt from the PR source
          # only when the service's own dir OR shared Gradle config
          # changed; otherwise
          # system-tests.yml pulls ghcr :latest and re-tags as :ci.
          #
          # UIs are always rebuilt: they bake VITE_AUTH_URL at build
          # time, and the ghcr :latest image is built with the prod
          # URL (auth.jorisjonkers.dev), whereas CI needs the .test
          # URL. Re-using the prod image in CI makes the SPA call
          # the wrong /me endpoint, fail auth, and render the
          # unauthenticated portfolio page.
          CHANGED_SERVICES=""
          [[ "$AUTH_API_KOTLIN"      == "true" ]] && CHANGED_SERVICES+=" auth-api"
          [[ "$KNOWLEDGE_API_KOTLIN" == "true" ]] && CHANGED_SERVICES+=" knowledge-api"
          CHANGED_SERVICES+=" auth-ui app-ui stalwart-tools"
          CHANGED_SERVICES="${CHANGED_SERVICES# }"

          echo "any-kotlin=$ANY_KOTLIN" >> "$GITHUB_OUTPUT"
          echo "any-frontend=$ANY_FRONTEND" >> "$GITHUB_OUTPUT"
          echo "any-python=$ANY_PYTHON" >> "$GITHUB_OUTPUT"
          echo "knowledge-ingest-worker=$KNOWLEDGE_INGEST_WORKER" >> "$GITHUB_OUTPUT"
          echo "agents-login=$AGENTS_LOGIN" >> "$GITHUB_OUTPUT"
          echo "any-platform=$ANY_PLATFORM" >> "$GITHUB_OUTPUT"
          echo "run-system-tests=$RUN_SYSTEM_TESTS" >> "$GITHUB_OUTPUT"
          echo "auth-api-kotlin=$AUTH_API_KOTLIN" >> "$GITHUB_OUTPUT"
          echo "knowledge-api-kotlin=$KNOWLEDGE_API_KOTLIN" >> "$GITHUB_OUTPUT"
          echo "changed-services=$CHANGED_SERVICES" >> "$GITHUB_OUTPUT"
      - name: Build dynamic matrices
        id: matrix
        env:
          AUTH_API_KOTLIN: ${{ steps.compute.outputs.auth-api-kotlin }}
          KNOWLEDGE_API_KOTLIN: ${{ steps.compute.outputs.knowledge-api-kotlin }}
        run: |
          KOTLIN_MATRIX="[]"
          if [[ "$AUTH_API_KOTLIN" == "true" ]]; then
            KOTLIN_MATRIX=$(echo "$KOTLIN_MATRIX" | jq -c '. + [{"module":"auth-api","gradle-path":":services:auth-api"}]')
          fi
          if [[ "$KNOWLEDGE_API_KOTLIN" == "true" ]]; then
            KOTLIN_MATRIX=$(echo "$KOTLIN_MATRIX" | jq -c '. + [{"module":"knowledge-api","gradle-path":":services:knowledge-api"}]')
          fi
          echo "lint-kotlin=$KOTLIN_MATRIX" >> "$GITHUB_OUTPUT"
          echo "test-kotlin-unit=$KOTLIN_MATRIX" >> "$GITHUB_OUTPUT"

          INT_MATRIX="[]"
          if [[ "$AUTH_API_KOTLIN" == "true" ]]; then
            INT_MATRIX=$(echo "$INT_MATRIX" | jq -c '. + [{"module":"auth-api","gradle-path":":services:auth-api"}]')
          fi
          if [[ "$KNOWLEDGE_API_KOTLIN" == "true" ]]; then
            INT_MATRIX=$(echo "$INT_MATRIX" | jq -c '. + [{"module":"knowledge-api","gradle-path":":services:knowledge-api"}]')
          fi
          echo "test-kotlin-integration=$INT_MATRIX" >> "$GITHUB_OUTPUT"

          COV_MATRIX="[]"
          if [[ "$AUTH_API_KOTLIN" == "true" ]]; then
            COV_MATRIX=$(echo "$COV_MATRIX" | jq -c '. + [{"module":"auth-api","gradle-path":":services:auth-api","has-integration":true}]')
          fi
          if [[ "$KNOWLEDGE_API_KOTLIN" == "true" ]]; then
            COV_MATRIX=$(echo "$COV_MATRIX" | jq -c '. + [{"module":"knowledge-api","gradle-path":":services:knowledge-api","has-integration":true}]')
          fi
          echo "verify-coverage=$COV_MATRIX" >> "$GITHUB_OUTPUT"

  lint-kotlin:
    name: Lint Kotlin (${{ matrix.module }})
    needs: detect-changes
    if: needs.detect-changes.outputs.any-kotlin == 'true'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include: ${{ fromJson(needs.detect-changes.outputs.lint-kotlin-matrix) }}
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: ./.github/actions/setup-java-gradle
        with:
          java-version: ${{ env.JAVA_VERSION }}
      - run: bash infra/scripts/run-strict-command.sh ./gradlew ${{ matrix.gradle-path }}:detekt ${{ matrix.gradle-path }}:ktlintCheck

  lint-frontend:
    name: Lint & Format
    needs: detect-changes
    if: needs.detect-changes.outputs.any-frontend == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: ./.github/actions/setup-node-pnpm
        with:
          node-version: ${{ env.NODE_VERSION }}
      - run: pnpm -r lint --max-warnings 0
      - run: pnpm format:check

  test-kotlin-unit:
    name: Kotlin Unit Tests (${{ matrix.module }})
    needs: detect-changes
    if: needs.detect-changes.outputs.any-kotlin == 'true'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include: ${{ fromJson(needs.detect-changes.outputs.test-kotlin-unit-matrix) }}
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: ./.github/actions/setup-java-gradle
        with:
          java-version: ${{ env.JAVA_VERSION }}
      - run: bash infra/scripts/run-strict-command.sh ./gradlew ${{ matrix.gradle-path }}:test

  test-frontend-unit:
    name: Frontend Unit Tests
    needs: detect-changes
    if: needs.detect-changes.outputs.any-frontend == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: ./.github/actions/setup-node-pnpm
        with:
          node-version: ${{ env.NODE_VERSION }}
      - run: pnpm -r test -- --coverage

  lint-python:
    name: Lint Python (knowledge-ingest-worker)
    needs: detect-changes
    if: needs.detect-changes.outputs.knowledge-ingest-worker == 'true'
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: services/knowledge-ingest-worker
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: astral-sh/setup-uv@v8.2.0
        with:
          version: '0.4.30'
      - run: uv sync --python 3.12 --frozen
      - run: uv run ruff check .
      - run: uv run mypy

  test-python-unit:
    name: Python Unit Tests (knowledge-ingest-worker)
    needs: detect-changes
    if: needs.detect-changes.outputs.knowledge-ingest-worker == 'true'
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: services/knowledge-ingest-worker
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: astral-sh/setup-uv@v8.2.0
        with:
          version: '0.4.30'
      - run: uv sync --python 3.12 --frozen
      - run: uv run pytest --cov

  test-python-integration:
    name: Python Integration Tests (knowledge-ingest-worker)
    needs: detect-changes
    if: needs.detect-changes.outputs.knowledge-ingest-worker == 'true'
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: services/knowledge-ingest-worker
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: astral-sh/setup-uv@v8.2.0
        with:
          version: '0.4.30'
      - run: uv sync --python 3.12 --frozen
      # Testcontainers spins up a real RabbitMQ — Docker on the runner
      # is sufficient (ubuntu-latest ships with Docker).
      - run: uv run pytest -m integration

  test-agents-login:
    name: agents-login (lint / build / test)
    needs: detect-changes
    if: needs.detect-changes.outputs.agents-login == 'true'
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: services/agents-login
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: actions/setup-node@v6
        with:
          node-version: ${{ env.NODE_VERSION }}
      - run: npm ci
      - run: npm run lint
      - run: npm run build
      - run: npm test

  test-kotlin-integration:
    name: Kotlin Integration Tests (${{ matrix.module }})
    needs: detect-changes
    if: needs.detect-changes.outputs.any-kotlin == 'true'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include: ${{ fromJson(needs.detect-changes.outputs.test-kotlin-integration-matrix) }}
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: ./.github/actions/setup-java-gradle
        with:
          java-version: ${{ env.JAVA_VERSION }}
      - run: bash infra/scripts/run-strict-command.sh ./gradlew ${{ matrix.gradle-path }}:integrationTest

  verify-coverage:
    name: Coverage Threshold (${{ matrix.module }})
    needs: [detect-changes, test-kotlin-unit, test-kotlin-integration]
    if: always() && needs.detect-changes.outputs.any-kotlin == 'true' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include: ${{ fromJson(needs.detect-changes.outputs.verify-coverage-matrix) }}
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: ./.github/actions/setup-java-gradle
        with:
          java-version: ${{ env.JAVA_VERSION }}
      - name: Run tests and verify coverage (80%)
        if: matrix.has-integration
        run: bash infra/scripts/run-strict-command.sh ./gradlew ${{ matrix.gradle-path }}:test ${{ matrix.gradle-path }}:integrationTest ${{ matrix.gradle-path }}:jacocoTestCoverageVerification
      - name: Run tests and verify coverage (80%) - unit only
        if: '!matrix.has-integration'
        run: bash infra/scripts/run-strict-command.sh ./gradlew ${{ matrix.gradle-path }}:test ${{ matrix.gradle-path }}:jacocoTestCoverageVerification

  test-architecture:
    name: Architecture Tests
    needs: detect-changes
    if: needs.detect-changes.outputs.any-kotlin == 'true' || needs.detect-changes.outputs.any-frontend == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: ./.github/actions/setup-java-gradle
        with:
          java-version: ${{ env.JAVA_VERSION }}
      - uses: ./.github/actions/setup-node-pnpm
        with:
          node-version: ${{ env.NODE_VERSION }}
      - run: bash infra/scripts/run-strict-command.sh ./gradlew :services:auth-api:test --tests "*ArchitectureTest*"
      - run: pnpm -r depcruise

  system-tests:
    name: System Tests
    needs: [detect-changes]
    if: needs.detect-changes.outputs.run-system-tests == 'true'
    uses: ./.github/workflows/system-tests.yml
    with:
      java-version: '21'
      # When a service's sources didn't change on this branch we let
      # the reusable workflow pull its ghcr :latest image and re-tag
      # it for CI instead of rebuilding from scratch. An empty string
      # here means "rebuild nothing, pull everything", which still
      # gives system tests a consistent set of images to exercise.
      changed-services: ${{ needs.detect-changes.outputs.changed-services }}

  platform-validate:
    # Platform manifests (fleet.yaml, Flux kustomizations, nix host
    # configs, helm values) are gated on every PR that touches
    # platform/**. Folded into the Full Pipeline so its success is a
    # required signal for merging — the separate platform-validate
    # workflow is gone.
    name: Platform Validate
    needs: [detect-changes]
    if: needs.detect-changes.outputs.any-platform == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: ./.github/actions/setup-java-gradle
        with:
          java-version: ${{ env.JAVA_VERSION }}
      - uses: ./.github/actions/setup-node-pnpm
        with:
          node-version: ${{ env.NODE_VERSION }}
      - name: Install deploy-config-schema toolkit
        env:
          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -euo pipefail
          npm install --no-save --prefix "$RUNNER_TEMP/dcs" @extratoast/deploy-config-schema@^0.7.0
          echo "DEPLOY_CONFIG_SCHEMA_BIN=$RUNNER_TEMP/dcs/node_modules/.bin/deploy-config-schema" >> "$GITHUB_ENV"
      - name: Install Nix
        uses: DeterminateSystems/nix-installer-action@main
      - name: Install kustomize
        uses: imranismail/setup-kustomize@v3
      - name: Install Helm
        uses: azure/setup-helm@v5
      - name: Install kubeconform
        run: |
          set -euo pipefail
          version="v0.6.7"
          curl -sSL -o /tmp/kubeconform.tar.gz \
            "https://github.com/yannh/kubeconform/releases/download/${version}/kubeconform-linux-amd64.tar.gz"
          tar -xzf /tmp/kubeconform.tar.gz -C /tmp
          sudo install /tmp/kubeconform /usr/local/bin/kubeconform
      - name: Install Flux CLI
        # flux-local shells out to `flux build ks` under the hood, so
        # the upstream CLI has to be on PATH.
        uses: fluxcd/flux2/action@main
      - name: Install flux-local
        # flux-local expands every HelmRelease CR against its real
        # chart + values, which is what exposes remote-chart template
        # errors (loki schema_config, nvidia plugin config shape,
        # SingleBinary + scalable replicas clash) before merge.
        run: pipx install flux-local
      - name: Validate platform render
        run: platform/scripts/validate/render-platform.sh

  deploy-config-schema-validate:
    name: Deploy Config Schema Validate
    needs: [detect-changes]
    if: needs.detect-changes.outputs.any-platform == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: ./.github/actions/setup-node-pnpm
        with:
          node-version: ${{ env.NODE_VERSION }}
      - name: Generate deploy config from fleet inventory
        env:
          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -euo pipefail
          npm install --no-save --prefix "$RUNNER_TEMP/dcs" @extratoast/deploy-config-schema@^0.7.0
          "$RUNNER_TEMP/dcs/node_modules/.bin/deploy-config-schema" fleet-to-deploy-config platform/inventory/fleet.yaml > "$RUNNER_TEMP/deploy-config.json"
      - name: Validate generated deploy config
        uses: ExtraToast/github-workflows/actions/platform-config-validate@v0.5.0
        with:
          config-paths: deploy-config.json
          schema-kind: deploy-config
          package-version: ^0.4.0
          working-directory: ${{ runner.temp }}
          node-auth-token: ${{ secrets.GITHUB_TOKEN }}

  platform-tests:
    # Platform validation suite (fleet.yaml inventory rules, nix host configs,
    # k3s labels/bootstrap, provisioning scripts, VSO/vault coverage, agent-kit
    # manifest). Node-based (no Kotlin/gradle); reads the committed tree.
    name: Platform Tests
    needs: [detect-changes]
    if: needs.detect-changes.outputs.any-platform == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6.0.3
      - uses: actions/setup-node@v6
        with:
          node-version: ${{ env.NODE_VERSION }}
      - name: Run platform tests
        working-directory: platform/tests
        run: |
          npm ci
          node --test

  security-scan:
    name: Security Scan
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: read
      pull-requests: write
    steps:
      - uses: actions/checkout@v6.0.3
        with:
          fetch-depth: 0
      - uses: aquasecurity/trivy-action@master
        with:
          scan-type: fs
          scan-ref: .
          severity: CRITICAL,HIGH
      - uses: ./.github/actions/setup-node-pnpm
        with:
          node-version: ${{ env.NODE_VERSION }}
      - run: pnpm audit --audit-level=high || true
      - name: Gitleaks
        uses: gitleaks/gitleaks-action@v3.0.0
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  migration-guard:
    name: Migration Guard
    needs: detect-changes
    uses: ./.github/workflows/migration-guard.yml
    secrets: inherit

  contract-validate:
    name: Contract Validate
    needs: detect-changes
    if: needs.detect-changes.outputs.any-kotlin == 'true' || needs.detect-changes.outputs.any-frontend == 'true'
    uses: ./.github/workflows/contract-validate.yml
    secrets: inherit

  vault-bootstrap-validate:
    name: Vault Bootstrap Validate
    needs: detect-changes
    if: needs.detect-changes.outputs.any-platform == 'true'
    uses: ./.github/workflows/vault-bootstrap-validate.yml
    secrets: inherit

  stalwart-tests:
    name: Stalwart Config Tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6.0.3
      - name: Run stalwart config tests
        run: sh tests/stalwart/run.sh

  pipeline-complete:
    name: Pipeline Complete
    if: always()
    needs:
      - detect-changes
      - lint-kotlin
      - lint-frontend
      - test-kotlin-unit
      - test-frontend-unit
      - test-kotlin-integration
      - verify-coverage
      - test-architecture
      - system-tests
      - platform-validate
      - deploy-config-schema-validate
      - platform-tests
      - security-scan
      - migration-guard
      - contract-validate
      - vault-bootstrap-validate
      - stalwart-tests
      - test-agents-login
    runs-on: ubuntu-latest
    steps:
      - name: Check pipeline result
        run: |
          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" || "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
            echo "Pipeline failed or was cancelled"
            exit 1
          fi
          echo "All checks passed (some may have been skipped due to no relevant changes)"