general.yml

name: General Checks
run-name: "General Checks for: ${{ github.event.pull_request.title || github.ref }}"

on:
  merge_group:
    paths-ignore:
      - "docs/**"
  pull_request:
  workflow_dispatch:

concurrency:
  group: ${{ github.ref == 'refs/heads/main' && github.run_id || format('general-{0}-{1}', github.workflow, github.ref) }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

env:
  FORCE_COLOR: 1
  TENSORZERO_CLICKHOUSE_URL: "http://chuser:chpassword@localhost:8123/tensorzero"
  SQLX_OFFLINE: 1
  R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
  R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
  RUST_MSRV: 1.88.0

jobs:
  check-version-consistency:
    permissions:
      contents: read
    runs-on: ubuntu-latest
    if: github.repository == 'RouterBase/tensorzero'
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
        with:
          fetch-depth: 0

      - name: Check version consistency
        run: ./ci/check-version-consistency.sh

  # This checks for specific markers in the codebase that require multiple files to be edited together.
  check-if-edited-then-edit:
    permissions:
      contents: read
    runs-on: ubuntu-latest
    # Should run on all PRs from forks as well, if the label is not present
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-if-edited-check') }}
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b
        with:
          python-version: "3.9"

      - name: Check coordinated edits
        run: python3 ci/check_coordinated_edits.py

  # Like `check-docker-compose-released`, but loads the freshly built `gateway` and `ui` container images first
  # This checks that all of our docker-compose files work with the changes in the PR
  check-docker-compose-commit:
    needs: [build-gateway-container, build-ui-container]
    permissions:
      # Permission to checkout the repository
      contents: read
      # Permission to fetch GitHub OIDC token authentication
      id-token: write
      # Permission to download artifacts
      actions: read

    runs-on: ubuntu-latest
    if: github.repository == 'RouterBase/tensorzero'
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      - name: Download container images
        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53
        with:
          pattern: build-{gateway,ui}-container
          merge-multiple: true

      - name: Load `gateway` and `ui` containers
        run: |
          docker load < gateway-container.tar
          docker load < ui-container.tar
          # Retag the loaded images to match what docker-compose expects
          docker tag tensorzero/gateway:sha-${{ github.sha }} tensorzero/gateway:latest
          docker tag tensorzero/ui:sha-${{ github.sha }} tensorzero/ui:latest

      - name: Create the `object_storage` directory for the multimodal-vision-finetuning example
        run: |
          mkdir -p examples/multimodal-vision-finetuning/object_storage
          chmod 777 examples/multimodal-vision-finetuning/object_storage

      - name: Apply Postgres migrations for the centralize-auth-rate-limits-and-more/auth example
        # Note: In isolation, applying migrations should not require `OPENAI_API_KEY`, but Docker Compose will complain if it's not set given our docker-compose.yml file.
        # Note: We must delete the Docker network because later we use `ci/internal-network.yml`
        # Note: This uses relay-gateway instead of gateway, so it requires explicit handling here
        run: |
          cd examples/docs/guides/operations/centralize-auth-rate-limits-and-more/auth
          OPENAI_API_KEY=dummy docker compose run --rm relay-gateway --run-postgres-migrations
          OPENAI_API_KEY=dummy docker compose down

      - name: Check latest docker-compose
        run: ./ci/check-all-docker-compose.sh

  # Checks our docker-compose files against the most recent published `gateway` and `ui` container images
  # (not the ones built for this PR)
  check-docker-compose-released:
    permissions:
      # Permission to checkout the repository
      contents: read
      # Permission to fetch GitHub OIDC token authentication
      id-token: write
    runs-on: ubuntu-latest
    timeout-minutes: 120
    if: github.repository == 'RouterBase/tensorzero'

    steps:
      - name: Set DNS
        run: echo "127.0.0.1 howdy.tensorzero.com" | sudo tee -a /etc/hosts
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      # We allow the namespace builder setup to fail on Dependabot PRs and PRs from forks
      # (where the oidc token is not available)

      - name: Install Namespace CLI
        uses: namespacelabs/nscloud-setup@d1c625762f7c926a54bd39252efff0705fd11c64
        continue-on-error: ${{ github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]' }}

      - name: Configure Namespace-powered Buildx
        uses: namespacelabs/nscloud-setup-buildx-action@91c2e6537780e3b092cb8476406be99a8f91bd5e
        with:
          wait-for-builder: true
        continue-on-error: ${{ github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]' }}

      - name: Create the `object_storage` directory for the multimodal-vision-finetuning example
        run: |
          mkdir -p examples/multimodal-vision-finetuning/object_storage
          chmod 777 examples/multimodal-vision-finetuning/object_storage

      - name: Apply Postgres migrations for the centralize-auth-rate-limits-and-more/auth example
        # Note: In isolation, applying migrations should not require `OPENAI_API_KEY`, but Docker Compose will complain if it's not set given our docker-compose.yml file.
        # Note: We must delete the Docker network because later we use `ci/internal-network.yml`
        # Note: This uses relay-gateway instead of gateway, so it requires explicit handling here
        run: |
          cd examples/docs/guides/operations/centralize-auth-rate-limits-and-more/auth
          OPENAI_API_KEY=dummy docker compose run --rm relay-gateway --run-postgres-migrations
          OPENAI_API_KEY=dummy docker compose down

      - name: Check all docker-compose.yml files
        run: ./ci/check-all-docker-compose.sh

  check-python-client-build:
    permissions:
      contents: read
    uses: ./.github/workflows/python-client-build.yml

  check-node-bindings:
    permissions:
      contents: read
      # Permission to upload artifacts
      actions: write
    runs-on: ubuntu-latest
    if: github.repository == 'RouterBase/tensorzero'
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      - name: Setup Node.js
        uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e
        with:
          node-version: "24.13.0"

      - name: Install pnpm
        run: |
          for attempt in 1 2 3; do
            if npm install -g pnpm@latest; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install pnpm after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash

      - name: Install dependencies
        working-directory: internal/tensorzero-node
        run: pnpm install --frozen-lockfile

      - name: Build Node.js bindings
        working-directory: internal/tensorzero-node
        run: pnpm build-bindings

      - name: Check for git diff after building bindings
        id: git_diff_check
        run: git diff --exit-code

      - name: Upload bindings artifact on diff failure
        if: failure()
        uses: namespace-actions/upload-artifact@9a78c62e083914789d908952f9773e42744b9f68
        with:
          name: node-bindings
          path: internal/tensorzero-node/lib/bindings
          retention-days: 7

      - name: Check tensorzero-node bindings are properly exported
        run: pnpm --filter=tensorzero-node run check-exports

  check-python-schemas:
    permissions:
      contents: read
    runs-on: ubuntu-latest
    if: github.repository == 'RouterBase/tensorzero'
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      - name: Setup Node.js
        uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e
        with:
          node-version: "24.13.0"

      - name: Install pnpm
        run: |
          for attempt in 1 2 3; do
            if npm install -g pnpm@latest; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install pnpm after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash

      - name: Install uv
        uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b
        with:
          version: "0.9.27"

      - name: Generate Python schemas
        run: pnpm generate-python-schemas

      - name: Check for git diff after generating schemas
        id: git_diff_check
        run: git diff --exit-code

  build-windows:
    permissions:
      contents: read
      # Permission for rust-cache to read/write cache
      actions: read
    runs-on: windows-latest
    if: github.repository == 'RouterBase/tensorzero'
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
      - name: Install Rust toolchain
        run: |
          for attempt in 1 2 3; do
            if rustup toolchain install stable && rustup default stable; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install Rust toolchain after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash
      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6
        with:
          cache-provider: "buildjet"
          save-if: ${{ github.event_name == 'merge_group' }}
      - name: Install uv
        uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b
        with:
          version: "0.9.27"
      - name: Install Python 3.10
        run: uv python install 3.10
      - name: Set PYO3_PYTHON
        run: echo "PYO3_PYTHON=$(uv python find python3.10)" >> $GITHUB_ENV
        shell: bash
      - name: Build Rust
        run: cargo build --workspace

  lint-rust:
    permissions:
      contents: read
      # Permission for rust-cache
      actions: read
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # Keep 'total_partitions' in sync with the number of entries in the 'partition' array
        partition: [1, 2, 3, 4]
        include:
          - total_partitions: 4
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
      # We install the latest rust when running lints, since we don't care if older rust versions lint successfully.
      # This avoids the need to deal with linter bugs in older rust versions.
      - name: Install Rust toolchain
        run: |
          for attempt in 1 2 3; do
            if rustup toolchain install stable --component clippy && rustup default stable; then
              break
            fi
          done
        shell: bash
      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6
        with:
          cache-provider: "buildjet"
          shared-key: "lint-rust-cache"
          save-if: ${{ github.event_name == 'merge_group' }}

      - name: Install cargo-hack
        uses: taiki-e/install-action@60581cd7025e0e855cebd745379013e286d9c787
        with:
          tool: cargo-hack

      - name: Lint (Rust)
        run: |
          cargo hack --partition '${{ matrix.partition }}/${{ matrix.total_partitions }}' clippy --all-targets --each-feature -- -D warnings

  validate-python:
    permissions:
      contents: read
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      # We deliberately install our MSRV here (rather than 'stable') to ensure that everything compiles with that version
      - name: Install Rust ${{ env.RUST_MSRV }}
        run: |
          for attempt in 1 2 3; do
            if rustup install ${{ env.RUST_MSRV }} --component rustfmt && rustup default ${{ env.RUST_MSRV }}; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install Rust ${{ env.RUST_MSRV }} after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash

      - name: Print Rust version
        run: rustc --version

      - name: Install uv
        uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b
        with:
          version: "0.9.27"

      - name: Install Python for python async client tests
        run: uv python install 3.9

      - name: Lint (Python:ruff)
        run: |
          uvx ruff@0.14.0 check --output-format=github --extend-select I .
          uvx ruff@0.14.0 format --check .

      - name: "Python: Pyo3 Client: Build and install dependencies"
        working-directory: clients/python
        run: uv sync

      - name: "Python: PyO3 Client: pyright"
        working-directory: clients/python
        run: |
          uv pip install pyright==1.1.394
          uv run pyright

      - name: "Python: PyO3 Client: stubtest"
        working-directory: clients/python
        run: |
          uv run stubtest tensorzero.tensorzero

      - name: "Python: Recipes: Install dependencies"
        working-directory: recipes
        run: |
          uv venv
          uv sync

      - name: "Python: Recipes: pyright"
        working-directory: recipes
        run: |
          uv run pyright

      - name: Compile / validate notebooks
        run: ci/compile-check-notebooks.sh

  validate-node:
    permissions:
      contents: read
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      # We deliberately install our MSRV here (rather than 'stable') to ensure that everything compiles with that version
      - name: Install Rust ${{ env.RUST_MSRV }}
        run: |
          for attempt in 1 2 3; do
            if rustup install ${{ env.RUST_MSRV }} --component rustfmt && rustup default ${{ env.RUST_MSRV }}; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install Rust ${{ env.RUST_MSRV }} after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash

      - name: Print Rust version
        run: rustc --version

      - name: Setup Node.js
        uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e
        with:
          node-version: "24.13.0"

      - name: Install pnpm
        run: |
          for attempt in 1 2 3; do
            if npm install -g pnpm@latest; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install pnpm after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash

      - name: Install JS dependencies
        run: pnpm install --frozen-lockfile

      - name: Build npm workspace
        run: pnpm -r build

      - name: Format tensorzero-node package
        run: pnpm --filter=tensorzero-node run format:check

      - name: Lint tensorzero-node package
        run: pnpm --filter=tensorzero-node run lint:check

      - name: Typecheck tensorzero-node package
        run: pnpm --filter=tensorzero-node run typecheck

      - name: Run ESLint
        run: pnpm --filter=tensorzero-ui run lint:check

      - name: Run Prettier
        run: pnpm --filter=tensorzero-ui run format:check

      - name: pnpm TypeScript type checking
        run: pnpm --filter=tensorzero-ui run typecheck

  rust-build:
    permissions:
      contents: read
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      # We deliberately install our MSRV here (rather than 'stable') to ensure that everything compiles with that version
      - name: Install Rust ${{ env.RUST_MSRV }}
        run: |
          for attempt in 1 2 3; do
            if rustup install ${{ env.RUST_MSRV }} --component rustfmt && rustup default ${{ env.RUST_MSRV }}; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install Rust ${{ env.RUST_MSRV }} after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash

      - name: Install cargo-nextest, cargo-deny, and cargo-hack
        uses: taiki-e/install-action@60581cd7025e0e855cebd745379013e286d9c787
        with:
          tool: cargo-nextest,cargo-deny,cargo-hack

      - name: Build (Rust)
        run: cargo build --workspace --verbose

      - name: Validate config
        run: |
          unset TENSORZERO_CLICKHOUSE_URL
          cargo run-e2e --validate-and-exit

  rust-test:
    permissions:
      contents: read
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      # We deliberately install our MSRV here (rather than 'stable') to ensure that everything compiles with that version
      - name: Install Rust ${{ env.RUST_MSRV }}
        run: |
          for attempt in 1 2 3; do
            if rustup install ${{ env.RUST_MSRV }} --component rustfmt && rustup default ${{ env.RUST_MSRV }}; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install Rust ${{ env.RUST_MSRV }} after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash

      - name: Install cargo-nextest, cargo-deny, and cargo-hack
        uses: taiki-e/install-action@60581cd7025e0e855cebd745379013e286d9c787
        with:
          tool: cargo-nextest,cargo-deny,cargo-hack

      - name: Test (Rust)
        env:
          # Write the exported bindings to somewhere we don't care about
          TS_RS_EXPORT_DIR: /tmp/tensorzero-bindings
        run: |
          cargo test-unit ${{ vars.CARGO_NEXTEST_EXTRA_ARGS }}

  validate:
    permissions:
      contents: read
    runs-on: ubuntu-latest

    timeout-minutes: 30
    if: github.repository == 'RouterBase/tensorzero'

    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      # We deliberately install our MSRV here (rather than 'stable') to ensure that everything compiles with that version
      - name: Install Rust ${{ env.RUST_MSRV }}
        run: |
          for attempt in 1 2 3; do
            if rustup install ${{ env.RUST_MSRV }} --component rustfmt && rustup default ${{ env.RUST_MSRV }}; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install Rust ${{ env.RUST_MSRV }} after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash

      - name: Print Rust version
        run: rustc --version

      - name: Setup Node.js
        uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e
        with:
          node-version: "24.13.0"

      - name: Install pnpm
        run: |
          for attempt in 1 2 3; do
            if npm install -g pnpm@latest; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install pnpm after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash

      - name: Install uv
        uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b
        with:
          version: "0.9.27"

      - name: Install JS dependencies
        run: pnpm install --frozen-lockfile

      - name: Set up Helm
        uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0
        with:
          version: "3.18.6" # Conservatively match the `Helm 3.2.0+` version specified in examples/production-deployment-k8s-helm/README.md

      - name: Install helm-values-schema-json plugin
        run: helm plugin install https://github.com/losisin/helm-values-schema-json.git

      - name: check‑case‑conflict
        run: uv run --with pre-commit pre-commit run check-case-conflict --all-files

      - name: end-of-file-fixer
        run: uv run --with pre-commit pre-commit run end-of-file-fixer --all-files

      - name: check‑executables‑have‑shebangs
        run: uv run --with pre-commit pre-commit run check-executables-have-shebangs --all-files

      - name: check‑json
        run: uv run --with pre-commit pre-commit run check-json --all-files

      - name: check‑yaml
        run: uv run --with pre-commit pre-commit run check-yaml --all-files

      - name: check‑toml
        run: uv run --with pre-commit pre-commit run check-toml --all-files

      - name: check‑xml
        run: uv run --with pre-commit pre-commit run check-xml --all-files

      - name: check‑merge‑conflict
        run: uv run --with pre-commit pre-commit run check-merge-conflict --all-files

      - name: check‑symlinks
        run: uv run --with pre-commit pre-commit run check-symlinks --all-files

      - name: check‑vcs‑permalinks
        run: uv run --with pre-commit pre-commit run check-vcs-permalinks --all-files

      - name: detect‑private‑key
        run: uv run --with pre-commit pre-commit run detect-private-key --all-files

      - name: helm-values-schema
        run: uv run --with pre-commit pre-commit run helm-schema --all-files

      - name: Check helm schema sync
        run: ./ci/check-helm-schema-sync.sh

      - name: prettier-root
        run: "uv run --with pre-commit pre-commit run 'prettier: root' --all-files"

      # We don't run these two because we want to allow template files to have trailing whitespace
      # TODO: how do we exclude minijinja files using pre-commit in GHA?
      # - name: end‑of‑file‑fixer
      #   run: uv run --with pre-commit pre-commit run end-of-file-fixer

      # - name: trailing‑whitespace
      #   run: uv run --with pre-commit pre-commit run trailing-whitespace

      # TODO: Enable this if we can figure out the invocation
      # - name: Run nb-clean
      #   run: uv run --with nb-clean nb-clean check --remove-empty-cells

      - name: Install cargo-nextest and cargo-hack
        uses: taiki-e/install-action@60581cd7025e0e855cebd745379013e286d9c787
        with:
          tool: cargo-nextest,cargo-hack

      - name: Check durable dependencies
        run: ./ci/check-durable-deps.sh

      - name: Run cargo fmt
        run: cargo fmt --all --check

      - name: "Node.js: Run prettier"
        run: pnpm --filter=openai-node run format

      - name: "Node.js: OpenAI Client: typecheck"
        run: pnpm --filter=openai-node run typecheck

      - name: "Node.js: OpenAI Client: lint"
        working-directory: clients/openai-node
        run: pnpm --filter=openai-node run lint

      - name: Lint Helm charts
        run: find . -name "Chart.yaml" -exec dirname {} \; | xargs -I {} helm lint {}

  # This job is intentionally NOT included in check-all-general-jobs-passed
  # It runs as an optional check that shows up in the PR but doesn't block merging
  cargo-deny:
    permissions:
      contents: read
    runs-on: ubuntu-latest
    if: github.repository == 'RouterBase/tensorzero'
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      - name: Install cargo-deny
        uses: taiki-e/install-action@60581cd7025e0e855cebd745379013e286d9c787
        with:
          tool: cargo-deny

      - name: Run cargo-deny
        run: cargo deny check

  clickhouse-tests:
    name: "ClickHouse tests (replicated: ${{ matrix.replicated }}) (version: ${{ matrix.clickhouse_version.tag }})"
    permissions:
      contents: read
      # Permission to download artifacts and for rust-cache
      actions: read
      # Permission to fetch GitHub OIDC token authentication for Namespace login
      id-token: write
    needs:
      [
        build-gateway-container,
        build-mock-provider-api-container,
        build-fixtures-container,
      ]

    # We don't run many tests here, so use a normal runner with Github Actions caching
    # to avoid unnecessarily using Namespace credits (it should still always finish before
    # the main 'validate' job)
    runs-on: ${{ matrix.replicated && 'namespace-profile-tensorzero-8x16;ephemeral-storage.size-multiplier=2' || 'namespace-profile-tensorzero-8x16' }}
    continue-on-error: ${{ matrix.clickhouse_version.allow_failure }}
    # This needs to pull from Docker Hub, so only run in the merge queue, or when running on a PR from the main repository
    # TODO - switch back to 'latest': https://github.com/tensorzero/tensorzero/issues/6080
    if: ${{ github.event_name == 'merge_group' || (github.event.pull_request.head.repo.full_name == github.repository) }}
    strategy:
      matrix:
        # Only include replicated: true when running in merge queue
        replicated: ${{ github.event_name == 'merge_group' && fromJSON('[true, false]') || fromJSON('[false]') }}
        clickhouse_version: ${{ github.event_name == 'merge_group' && fromJSON('[{"tag":"lts","prefix":"lts","allow_failure":false},{"tag":"25.11","prefix":"25.11","allow_failure":false}]') || fromJSON('[{"tag":"lts","prefix":"lts","allow_failure":false}]') }}
        exclude:
          - replicated: true
            clickhouse_version:
              { "tag": "25.11", "prefix": "25.11", "allow_failure": false }
        include: ${{ github.event_name == 'merge_group' && fromJSON('[{"clickhouse_version":{"tag":"25.11","prefix":"25.11","allow_failure":false},"replicated":true}]') || fromJson('[]') }}

    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
      - name: Install Rust toolchain
        run: |
          for attempt in 1 2 3; do
            if rustup toolchain install stable && rustup default stable; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install Rust toolchain after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash
      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6
        with:
          cache-provider: "buildjet"
          shared-key: "build-gateway-cache"
          save-if: false
      - name: "Check disk space before cleanup"
        run: df -h
      # ClickHouse intermittently runs out of disk space on the `ubuntu-latest` runner
      # Use extreme disk cleanup script that frees up ~26GB of space (11-phase cleanup)
      - name: "Free up disk space"
        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
      - name: Install cargo-nextest
        uses: taiki-e/install-action@60581cd7025e0e855cebd745379013e286d9c787
        with:
          tool: cargo-nextest

      - name: Set ClickHouse replicated cluster name
        if: matrix.replicated == true
        run: echo "TENSORZERO_CLICKHOUSE_CLUSTER_NAME=tensorzero_e2e_tests_cluster" >> $GITHUB_ENV

      - name: Setup Namespace caching for Clickhouse fixtures
        uses: namespacelabs/nscloud-cache-action@446d8f390563cd54ca27e8de5bdb816f63c0b706
        with:
          path: |
            ./ui/fixtures/large-fixtures/

      - name: Install uv
        uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b
        with:
          version: "0.9.27"

      - name: Download container images
        run: |
          docker pull nscr.io/igvf4asmf8kri/gateway:sha-${{ github.sha }}
          docker pull nscr.io/igvf4asmf8kri/mock-provider-api:sha-${{ github.sha }}
          docker pull nscr.io/igvf4asmf8kri/fixtures:sha-${{ github.sha }}
          # Retag the images to what we expect the names to be
          docker tag nscr.io/igvf4asmf8kri/gateway:sha-${{ github.sha }} tensorzero/gateway:sha-${{ github.sha }}
          docker tag nscr.io/igvf4asmf8kri/mock-provider-api:sha-${{ github.sha }} tensorzero/mock-provider-api:sha-${{ github.sha }}
          docker tag nscr.io/igvf4asmf8kri/fixtures:sha-${{ github.sha }} tensorzero/fixtures:sha-${{ github.sha }}
        continue-on-error: ${{ github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]' }}

      - name: Download ClickHouse fixtures
        run: |
          uv run ./ui/fixtures/download-large-fixtures.py
          uv run ./ui/fixtures/download-small-fixtures.py

      - name: Set up TENSORZERO_CLICKHOUSE_URL for E2E tests (non-replicated)
        if: matrix.replicated == false
        run: |
          echo "TENSORZERO_CLICKHOUSE_URL=http://chuser:chpassword@localhost:8123/tensorzero_e2e_tests" >> $GITHUB_ENV

      - name: Set up TENSORZERO_CLICKHOUSE_URL for E2E tests (replicated)
        if: matrix.replicated == true
        run: |
          # Note: In replicated mode, we use the HTTP port (8124) of the second replica so we can ensure this works
          echo "TENSORZERO_CLICKHOUSE_URL=http://chuser:chpassword@localhost:8124/tensorzero_e2e_tests" >> $GITHUB_ENV

      - name: Set container image tags
        run: |
          echo "TENSORZERO_GATEWAY_TAG=sha-${{ github.sha }}" >> $GITHUB_ENV
          echo "TENSORZERO_MOCK_PROVIDER_API_TAG=sha-${{ github.sha }}" >> $GITHUB_ENV
          echo "TENSORZERO_COMMIT_TAG=sha-${{ github.sha }}" >> $GITHUB_ENV

      - name: Launch dependency services with non-replicated ClickHouse container for E2E tests
        if: matrix.replicated == false
        run: TENSORZERO_CLICKHOUSE_VERSION=${{ matrix.clickhouse_version.tag }} docker compose -f tensorzero-core/tests/e2e/docker-compose.yml up --no-build --wait

      - name: Launch replicated ClickHouse container for E2E tests
        if: matrix.replicated == true
        run: TENSORZERO_CLICKHOUSE_VERSION=${{ matrix.clickhouse_version.tag }} docker compose -f tensorzero-core/tests/e2e/docker-compose.replicated.yml up --no-build --wait

      # Make an HTTP request to ClickHouse and check that the version matches '${{ matrix.clickhouse_version }}'
      - name: Check ClickHouse version
        run: |
          CLICKHOUSE_VERSION=$(curl -s "http://localhost:8123/query?user=chuser&password=chpassword" --data-binary "SELECT version()")
          echo "Detected ClickHouse version: $CLICKHOUSE_VERSION"
          echo "$CLICKHOUSE_VERSION" | grep -q "${{ matrix.clickhouse_version.prefix }}" || echo "WARNING: ClickHouse version does not match expected ${{ matrix.clickhouse_version.prefix }}"

      # We run this as a separate step so that we can see live build logs
      # (and fail the job immediately if the build fails)
      - name: Build the gateway for E2E tests
        run: cargo build-e2e

      - name: Launch the gateway for E2E tests (not configured for replication)
        if: matrix.replicated == false
        run: |
          cargo run-e2e > e2e_logs.txt 2>&1 &
            count=0
            max_attempts=20
            while ! curl http://localhost:3000/health; do
              echo "Waiting for gateway to be healthy..."
              sleep 1
              count=$((count + 1))
              if [ $count -ge $max_attempts ]; then
                echo "Gateway failed to become healthy after $max_attempts attempts"
                exit 1
              fi
            done
          echo "GATEWAY_PID=$!" >> $GITHUB_ENV

      - name: Launch the gateway for E2E tests (configured for replication)
        if: matrix.replicated == true
        env:
          TENSORZERO_CLICKHOUSE_URL: http://chuser:chpassword@localhost:8123/tensorzero_e2e_tests
        run: |
          cargo run-e2e --run-clickhouse-migrations &&
          cargo run-e2e > e2e_logs.txt 2>&1 &
            count=0
            max_attempts=40
            while ! curl http://localhost:3000/health; do
              echo "Waiting for gateway to be healthy..."
              sleep 1
              count=$((count + 1))
              if [ $count -ge $max_attempts ]; then
                echo "Gateway failed to become healthy after $max_attempts attempts"
                exit 1
              fi
            done
          echo "GATEWAY_PID=$!" >> $GITHUB_ENV

      - name: Test (Rust)
        env:
          DATABASE_URL: postgres://postgres:postgres@localhost:5432/tensorzero-e2e-tests
          TENSORZERO_POSTGRES_URL: postgres://postgres:postgres@localhost:5432/tensorzero-e2e-tests
        run: cargo test-clickhouse

      - name: Print docker compose logs (replicated)
        if: always() && matrix.replicated == true
        run: |
          TENSORZERO_CLICKHOUSE_VERSION=${{ matrix.clickhouse_version.tag }} docker compose -f tensorzero-core/tests/e2e/docker-compose.replicated.yml logs -t

      - name: Print ClickHouse error logs (replicated)
        if: always() && matrix.replicated == true
        run: |
          echo "Error logs for ClickHouse 01:"
          docker cp e2e-clickhouse-01-1:/var/log/clickhouse-server/clickhouse-server.err.log -
          echo "Error logs for ClickHouse 02:"
          docker cp e2e-clickhouse-02-1:/var/log/clickhouse-server/clickhouse-server.err.log -
          echo "Error logs for ClickHouse 03:"
          docker cp e2e-clickhouse-03-1:/var/log/clickhouse-server/clickhouse-server.err.log -

      - name: Print ClickHouse trace logs (replicated)
        if: always() && matrix.replicated == true
        run: |
          echo "Trace logs for ClickHouse 01:"
          docker cp e2e-clickhouse-01-1:/var/log/clickhouse-server/clickhouse-server.log -
          echo "Trace logs for ClickHouse 02:"
          docker cp e2e-clickhouse-02-1:/var/log/clickhouse-server/clickhouse-server.log -
          echo "Trace logs for ClickHouse 03:"
          docker cp e2e-clickhouse-03-1:/var/log/clickhouse-server/clickhouse-server.log -

      - name: Print container health checks (replicated)
        if: always() && matrix.replicated == true
        run: |
          echo "Health check for ClickHouse 01:"
          docker inspect --format "{{json .State.Health }}" e2e-clickhouse-01-1 | jq
          echo "Health check for ClickHouse 02:"
          docker inspect --format "{{json .State.Health }}" e2e-clickhouse-02-1 | jq
          echo "Health check for ClickHouse 03:"
          docker inspect --format "{{json .State.Health }}" e2e-clickhouse-03-1 | jq

      - name: Print docker compose logs (non-replicated)
        if: always() && matrix.replicated == false
        run: |
          TENSORZERO_CLICKHOUSE_VERSION=${{ matrix.clickhouse_version.tag }} docker compose -f tensorzero-core/tests/e2e/docker-compose.yml logs -t

      - name: Print ClickHouse error logs (non-replicated)
        if: always() && matrix.replicated == false
        run: docker cp e2e-clickhouse-1:/var/log/clickhouse-server/clickhouse-server.err.log -

      - name: Print ClickHouse trace logs (non-replicated)
        if: always() && matrix.replicated == false
        run: docker cp e2e-clickhouse-1:/var/log/clickhouse-server/clickhouse-server.log -

      - name: Print container health checks (non-replicated)
        if: always() && matrix.replicated == false
        run: |
          echo "Health check for ClickHouse:"
          docker inspect --format "{{json .State.Health }}" e2e-clickhouse-1 | jq || echo "Could not read health check"

      - name: Print e2e logs
        if: always()
        run: cat e2e_logs.txt

  # Run Postgres tests
  # Postgres does not have unit test coverage, because sqlx doesn't expose mocks. We include it here to get early CI signals.
  postgres-tests:
    name: "Postgres tests"
    permissions:
      contents: read
      # Permission to download artifacts and for rust-cache
      actions: read
      # Permission to fetch GitHub OIDC token authentication for Namespace login
      id-token: write
    runs-on: ubuntu-latest
    needs: [build-gateway-container, build-fixtures-container]
    if: ${{ github.event_name == 'merge_group' || (github.event.pull_request.head.repo.full_name == github.repository) }}

    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      - name: Install Rust toolchain
        run: |
          for attempt in 1 2 3; do
            if rustup toolchain install stable && rustup default stable; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install Rust toolchain after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash

      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6
        with:
          cache-provider: "buildjet"
          shared-key: "build-gateway-cache"
          save-if: false

      - name: Install cargo-nextest
        uses: taiki-e/install-action@60581cd7025e0e855cebd745379013e286d9c787
        with:
          tool: cargo-nextest

      - name: Install Namespace CLI
        uses: namespacelabs/nscloud-setup@d1c625762f7c926a54bd39252efff0705fd11c64
        continue-on-error: ${{ github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]' }}

      - name: Login to Namespace registry
        run: nsc docker login
        continue-on-error: ${{ github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]' }}

      - name: Download container images
        run: |
          docker pull nscr.io/igvf4asmf8kri/gateway:sha-${{ github.sha }}
          docker pull nscr.io/igvf4asmf8kri/fixtures:sha-${{ github.sha }}
          docker tag nscr.io/igvf4asmf8kri/gateway:sha-${{ github.sha }} tensorzero/gateway:sha-${{ github.sha }}
          docker tag nscr.io/igvf4asmf8kri/fixtures:sha-${{ github.sha }} tensorzero/fixtures:sha-${{ github.sha }}
        continue-on-error: ${{ github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]' }}

      - name: Set container image tags
        run: |
          echo "TENSORZERO_GATEWAY_TAG=sha-${{ github.sha }}" >> $GITHUB_ENV
          echo "TENSORZERO_COMMIT_TAG=sha-${{ github.sha }}" >> $GITHUB_ENV

      - name: Set environment variables for Postgres
        run: |
          echo "DATABASE_URL=postgres://postgres:postgres@localhost:5432/tensorzero-e2e-tests" >> $GITHUB_ENV
          echo "TENSORZERO_POSTGRES_URL=postgres://postgres:postgres@localhost:5432/tensorzero-e2e-tests" >> $GITHUB_ENV

      - name: Launch Postgres
        run: docker compose -f tensorzero-core/tests/e2e/docker-compose.yml up --no-build postgres gateway-postgres-migrations fixtures-postgres -d --wait

      - name: Test (Rust)
        run: cargo nextest run --filterset 'test(db::) & test(postgres)'

      - name: Print docker compose logs
        if: always()
        run: docker compose -f tensorzero-core/tests/e2e/docker-compose.yml logs -t

  # Run endpoint tests against both ClickHouse and Postgres backends
  endpoint-tests:
    name: "Endpoint tests (database: ${{ matrix.database }})"
    permissions:
      contents: read
      actions: read
      # Permission to fetch GitHub OIDC token authentication for Namespace login
      id-token: write
    needs: [build-gateway-container, build-mock-provider-api-container]
    runs-on: ubuntu-latest
    if: ${{ github.event_name == 'merge_group' || (github.event.pull_request.head.repo.full_name == github.repository) }}
    strategy:
      fail-fast: false
      matrix:
        database: [clickhouse, postgres]
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      - name: Cleanup disk space
        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be

      - name: Install Rust toolchain
        run: |
          for attempt in 1 2 3; do
            if rustup toolchain install stable && rustup default stable; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install Rust toolchain after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash

      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6
        with:
          cache-provider: "buildjet"
          shared-key: "build-gateway-cache"
          save-if: false

      - name: Install cargo-nextest
        uses: taiki-e/install-action@60581cd7025e0e855cebd745379013e286d9c787
        with:
          tool: cargo-nextest

      - name: Install Namespace CLI
        uses: namespacelabs/nscloud-setup@d1c625762f7c926a54bd39252efff0705fd11c64
        continue-on-error: ${{ github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]' }}

      - name: Install uv
        uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b
        with:
          version: "0.9.27"

      - name: Download ClickHouse fixtures
        run: |
          uv run ./ui/fixtures/download-small-fixtures.py

      - name: Login to Namespace registry
        run: nsc docker login
        continue-on-error: ${{ github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]' }}

      - name: Download container images
        run: |
          docker pull nscr.io/igvf4asmf8kri/gateway:sha-${{ github.sha }}
          docker pull nscr.io/igvf4asmf8kri/mock-provider-api:sha-${{ github.sha }}
          docker tag nscr.io/igvf4asmf8kri/gateway:sha-${{ github.sha }} tensorzero/gateway:sha-${{ github.sha }}
          docker tag nscr.io/igvf4asmf8kri/mock-provider-api:sha-${{ github.sha }} tensorzero/mock-provider-api:sha-${{ github.sha }}
        continue-on-error: ${{ github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]' }}

      - name: Set environment variables
        run: |
          echo "TENSORZERO_CLICKHOUSE_URL=http://chuser:chpassword@localhost:8123/tensorzero_e2e_tests" >> $GITHUB_ENV
          echo "TENSORZERO_POSTGRES_URL=postgres://postgres:postgres@localhost:5432/tensorzero-e2e-tests" >> $GITHUB_ENV
          echo "TENSORZERO_GATEWAY_TAG=sha-${{ github.sha }}" >> $GITHUB_ENV
          echo "TENSORZERO_MOCK_PROVIDER_API_TAG=sha-${{ github.sha }}" >> $GITHUB_ENV
          echo "TENSORZERO_SKIP_LARGE_FIXTURES=1" >> $GITHUB_ENV

      - name: Set Postgres read and write feature flags
        if: matrix.database == 'postgres'
        run: |
          echo "TENSORZERO_INTERNAL_FLAG_ENABLE_POSTGRES_READ=1" >> $GITHUB_ENV
          echo "TENSORZERO_INTERNAL_FLAG_ENABLE_POSTGRES_WRITE=1" >> $GITHUB_ENV

      - name: Launch dependency services
        run: docker compose -f tensorzero-core/tests/e2e/docker-compose.yml up --wait

      - name: Build the gateway for E2E tests
        run: cargo build-e2e

      - name: Launch the gateway for E2E tests
        run: |
          cargo run-e2e > e2e_logs.txt 2>&1 &
          count=0
          max_attempts=20
          while ! curl http://localhost:3000/health; do
            echo "Waiting for gateway to be healthy..."
            sleep 1
            count=$((count + 1))
            if [ $count -ge $max_attempts ]; then
              echo "Gateway failed to become healthy after $max_attempts attempts"
              exit 1
            fi
          done

      - name: Run endpoint tests
        run: cargo test-endpoints

      - name: Print docker compose logs
        if: always()
        run: docker compose -f tensorzero-core/tests/e2e/docker-compose.yml logs -t

      - name: Print e2e logs
        if: always()
        run: cat e2e_logs.txt

  # Run 'cargo test-optimization' against mock-provider-api
  mock-optimization-tests:
    permissions:
      contents: read
      # Permission for rust-cache
      actions: read
    runs-on: ubuntu-latest
    if: github.repository == 'RouterBase/tensorzero'
    env:
      OPENAI_API_KEY: not_used
      FIREWORKS_API_KEY: not_used
      TOGETHER_API_KEY: not_used
      TENSORZERO_INTERNAL_MOCK_PROVIDER_API: http://localhost:3030
      TENSORZERO_SKIP_LARGE_FIXTURES: 1
      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}

    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
      - name: Install Rust toolchain
        run: |
          for attempt in 1 2 3; do
            if rustup toolchain install stable && rustup default stable; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install Rust toolchain after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash
      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6
        with:
          cache-provider: "buildjet"
          shared-key: "build-gateway-cache"
          save-if: false
      - name: Install cargo-nextest
        uses: taiki-e/install-action@60581cd7025e0e855cebd745379013e286d9c787
        with:
          tool: cargo-nextest

      - name: Install uv
        uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b
        with:
          version: "0.9.27"
          # Without this, the post-install hook tries to prune and save the cache, but jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be already nukes /opt/hostedtoolcache/*, so it fails.
          enable-cache: "false"

      - name: Download ClickHouse fixtures
        run: |
          uv run ./ui/fixtures/download-large-fixtures.py
          uv run ./ui/fixtures/download-small-fixtures.py

      - name: Cleanup disk space
        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be

      - name: Set up TENSORZERO_CLICKHOUSE_URL for E2E tests
        run: |
          echo "TENSORZERO_CLICKHOUSE_URL=http://chuser:chpassword@localhost:8123/tensorzero_e2e_tests" >> $GITHUB_ENV

      - name: Make sure dependency services are up
        run: docker compose -f tensorzero-core/tests/e2e/docker-compose.yml up --wait

      - name: Launch the gateway for E2E tests
        run: |
          cargo run-e2e > e2e_logs.txt 2>&1 &
          GATEWAY_PID=$!
          while ! curl -s http://localhost:3000/health; do
            if ! kill -0 $GATEWAY_PID 2>/dev/null; then
              echo "Gateway process died!"
              exit 1
            fi
            echo "Waiting for gateway to be healthy..."
            sleep 1
          done

      - name: Test (Rust)
        run: cargo test-optimization-mock

      - name: Print docker compose logs
        if: always()
        run: docker compose -f tensorzero-core/tests/e2e/docker-compose.yml logs -t

      - name: Print e2e logs
        if: always()
        run: cat e2e_logs.txt

  build-gateway-e2e-container:
    uses: ./.github/workflows/build-gateway-e2e-container.yml
    if: (github.repository == 'RouterBase/tensorzero' && github.event_name == 'merge_group')
    permissions:
      # Permission to checkout the repository
      contents: read
      # Permission to fetch GitHub OIDC token authentication
      id-token: write
    secrets:
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_LIMITED_TOKEN: ${{ secrets.DOCKERHUB_LIMITED_TOKEN }}

  build-provider-proxy-container:
    uses: ./.github/workflows/build-provider-proxy-container.yml
    if: (github.repository == 'tensorzero/tensorzero' && github.event_name == 'merge_group')
    permissions:
      # Permission to checkout the repository
      contents: read
      # Permission to fetch GitHub OIDC token authentication
      id-token: write
    secrets:
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_LIMITED_TOKEN: ${{ secrets.DOCKERHUB_LIMITED_TOKEN }}

  build-live-tests-container:
    uses: ./.github/workflows/build-live-tests-container.yml
    if: (github.repository == 'tensorzero/tensorzero' && github.event_name == 'merge_group')
    permissions:
      # Permission to checkout the repository
      contents: read
      # Permission to fetch GitHub OIDC token authentication
      id-token: write
    secrets:
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_LIMITED_TOKEN: ${{ secrets.DOCKERHUB_LIMITED_TOKEN }}

  build-gateway-container:
    uses: ./.github/workflows/build-gateway-container.yml
    permissions:
      # Permission to checkout the repository
      contents: read
      # Permission to fetch GitHub OIDC token authentication
      id-token: write

  build-ui-container:
    uses: ./.github/workflows/build-ui-container.yml
    permissions:
      # Permission to checkout the repository
      contents: read
      # Permission to fetch GitHub OIDC token authentication
      id-token: write

  build-mock-provider-api-container:
    uses: ./.github/workflows/build-mock-provider-api-container.yml
    permissions:
      # Permission to checkout the repository
      contents: read
      # Permission to fetch GitHub OIDC token authentication
      id-token: write

  build-fixtures-container:
    uses: ./.github/workflows/build-fixtures-container.yml
    permissions:
      # Permission to checkout the repository
      contents: read
      # Permission to fetch GitHub OIDC token authentication
      id-token: write
    secrets:
      DOCKERHUB_LIMITED_TOKEN: ${{ secrets.DOCKERHUB_LIMITED_TOKEN }}

  ui-tests:
    permissions:
      contents: read
      actions: read
    uses: ./.github/workflows/ui-tests.yml
    with:
      is_merge_group: ${{ github.event_name == 'merge_group' }}
    secrets:
      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
    needs: [build-gateway-container, build-mock-provider-api-container]

  ui-tests-e2e:
    permissions:
      contents: read
      actions: write
    uses: ./.github/workflows/ui-tests-e2e.yml
    with:
      is_merge_group: ${{ github.event_name == 'merge_group' }}
    needs:
      [
        build-gateway-container,
        build-ui-container,
        build-mock-provider-api-container,
      ]
    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      FIREWORKS_ACCOUNT_ID: ${{ secrets.FIREWORKS_ACCOUNT_ID }}
      FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
      S3_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      S3_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      GCP_JWT_KEY: ${{ secrets.GCP_JWT_KEY }}
      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}

  check-production-deployment-docker-compose:
    needs: [build-gateway-container]
    permissions:
      contents: read
      # Permission to download artifacts
      actions: read
    runs-on: ubuntu-latest
    if: github.repository == 'RouterBase/tensorzero' && github.event_name == 'merge_group'
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      - name: Download gateway container image
        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53
        with:
          name: build-gateway-container

      - name: Load gateway container
        run: |
          docker load < gateway-container.tar
          docker tag tensorzero/gateway:sha-${{ github.sha }} tensorzero/gateway:latest

      - name: Launch ClickHouse container for E2E tests
        run: |
          docker compose -f tensorzero-core/tests/e2e/docker-compose.yml up clickhouse -d --wait

      - name: Set up .env file for production deployment tests
        run: |
          echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" > examples/production-deployment/.env
          echo "TENSORZERO_CLICKHOUSE_URL=http://chuser:chpassword@host.docker.internal:8123/tensorzero" >> examples/production-deployment/.env
          echo "TENSORZERO_DISABLE_PSEUDONYMOUS_USAGE_ANALYTICS=1" >> examples/production-deployment/.env

      - name: Run docker compose for production deployment tests
        run: docker compose -f examples/production-deployment/docker-compose.yml up -d --wait

      - name: Run inference for production deployment tests
        run: examples/production-deployment/run.sh

      - name: Print Docker compose logs
        if: always()
        run: |
          docker compose -f examples/production-deployment/docker-compose.yml logs -t

      - name: Take down docker compose for production deployment tests
        run: |
          docker compose -f examples/production-deployment/docker-compose.yml down
          docker compose -f tensorzero-core/tests/e2e/docker-compose.yml down

  # Test that the ui e2e tests still pass after we regenerate the model inference cache
  ui-tests-e2e-regen-model-inference-cache:
    permissions:
      contents: read
      actions: write
    if: github.repository == 'RouterBase/tensorzero' && github.event_name == 'merge_group'
    uses: ./.github/workflows/ui-tests-e2e-model-inference-cache.yml
    with:
      regen_cache: true
      is_merge_group: true
      force_no_auth: true
    needs:
      [
        build-gateway-container,
        build-ui-container,
        build-mock-provider-api-container,
      ]
    secrets:
      S3_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      S3_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      FIREWORKS_ACCOUNT_ID: ${{ secrets.FIREWORKS_ACCOUNT_ID }}
      FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

  run-merge-queue-checks:
    needs:
      [
        build-gateway-container,
        build-gateway-e2e-container,
        build-mock-provider-api-container,
        build-fixtures-container,
        build-provider-proxy-container,
        build-live-tests-container,
      ]
    if: (github.repository == 'RouterBase/tensorzero' && github.event_name == 'merge_group')
    permissions:
      # Permission to checkout the repository
      contents: read
      # Permission to fetch GitHub OIDC token authentication
      id-token: write
      actions: write
    uses: ./.github/workflows/merge-queue.yml
    secrets: inherit

  minikube:
    needs: [build-gateway-container, build-ui-container]
    permissions:
      contents: read
    uses: ./.github/workflows/minikube.yml

  mocked-batch-tests:
    permissions:
      contents: read
      actions: read
    if: github.repository == 'RouterBase/tensorzero' && github.event_name == 'merge_group'
    uses: ./.github/workflows/mocked-batch-test.yml
    needs: [build-mock-provider-api-container]
    secrets: inherit

  autopilot-e2e:
    permissions:
      contents: read
    if: github.repository == 'RouterBase/tensorzero' && github.event_name == 'merge_group'
    needs: [build-gateway-e2e-container, publish-dev-docker-hub-images]
    uses: ./.github/workflows/dispatch-autopilot-e2e.yml
    secrets:
      AUTOPILOT_DISPATCH_APP_ID: ${{ secrets.AUTOPILOT_DISPATCH_APP_ID }}
      AUTOPILOT_DISPATCH_PRIVATE_KEY: ${{ secrets.AUTOPILOT_DISPATCH_PRIVATE_KEY }}

  publish-dev-docker-hub-images:
    permissions:
      contents: read
      actions: read
      attestations: write
      id-token: write
      packages: write
    if: github.repository == 'RouterBase/tensorzero' && github.event_name == 'merge_group'
    uses: ./.github/workflows/docker-hub-publish.yml
    with:
      is_dev: true
    secrets:
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

  # See 'ci/README.md' at the repository root for more details.
  check-all-general-jobs-passed:
    permissions: {}
    if: always() && github.repository == 'RouterBase/tensorzero'
    needs:
      [
        check-version-consistency,
        check-production-deployment-docker-compose,
        check-if-edited-then-edit,
        check-docker-compose-released,
        check-docker-compose-commit,
        check-python-client-build,
        check-node-bindings,
        check-python-schemas,
        build-windows,
        build-ui-container,
        build-gateway-container,
        build-gateway-e2e-container,
        build-provider-proxy-container,
        build-live-tests-container,
        build-mock-provider-api-container,
        mocked-batch-tests,
        publish-dev-docker-hub-images,
        minikube,
        rust-build,
        rust-test,
        validate,
        validate-node,
        validate-python,
        lint-rust,
        clickhouse-tests,
        postgres-tests,
        endpoint-tests,
        ui-tests,
        ui-tests-e2e,
        ui-tests-e2e-regen-model-inference-cache,
        mock-optimization-tests,
        run-merge-queue-checks,
        autopilot-e2e,
      ]
    runs-on: ubuntu-latest
    steps:
      - name: Print all job results
        run: |
          echo "'needs': ${{ toJson(needs) }}"
          echo "github.event_name: ${{ github.event_name }}"
      # When running in the merge queue, jobs should never be skipped.
      # In PR CI, some jobs may be intentionally skipped (e.g. due to running from a fork, or to save money)
      - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || (github.event_name == 'merge_group' && contains(needs.*.result, 'skipped')) }}
        run: exit 1