push.yml

name: Build

on:
  push:
    paths:
      - '.github/actions/smoke.sh'
      - '.github/actions/integration/**'
      - '.github/workflows/push.yml'
      - '.github/workflows/master.yml'
      - 'packages/**'
      - 'rust/cubestore/**'
      - 'rust/cubesql/**'
      - 'rust/cubenativeutils/**'
      - 'rust/cubeorchestrator/**'
      - 'rust/cubeshared/**'
      - 'rust/cubesqlplanner/**'
      - '.eslintrc.js'
      - '.prettierrc'
      - 'package.json'
      - 'lerna.json'
      - 'rollup.config.js'
      - 'jest.base.config.js'
      - 'yarn.lock'
    branches:
      - 'master'
  pull_request:
    paths:
      - '.github/actions/smoke.sh'
      - '.github/actions/integration/**'
      - '.github/workflows/push.yml'
      - '.github/workflows/master.yml'
      - 'packages/**'
      - 'rust/cubestore/**'
      - 'rust/cubesql/**'
      - 'rust/cubenativeutils/**'
      - 'rust/cubeorchestrator/**'
      - 'rust/cubeshared/**'
      - 'rust/cubesqlplanner/**'
      - '.eslintrc.js'
      - '.prettierrc'
      - 'package.json'
      - 'lerna.json'
      - 'rollup.config.js'
      - 'jest.base.config.js'
      - 'yarn.lock'

env:
  CUBEJS_TESSERACT_ORCHESTRATOR: true
  # Current user version for Python. Should be aligned between build-native & docker-dev
  PYTHON_VERSION_CURRENT: 3.11

jobs:
  unit:
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    needs: [latest-tag-sha, build-native-linux]
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)

    strategy:
      matrix:
        # Current docker version + next LTS
        # TODO: Add 24 after it's been released
        node-version: [22.x]
        # Don't forget to update build-native-release
        python-version: [3.11]
        transpile-worker-threads: [false, true]
      fail-fast: false

    env:
      CUBEJS_TRANSPILATION_WORKER_THREADS: ${{ matrix.transpile-worker-threads }}
    steps:
      - id: get-tag-out
        run: echo "$OUT"
        env:
          OUT: ${{ needs['latest-tag-sha'].outputs.sha }}
      - name: Checkout
        uses: actions/checkout@v4
        with:
          # pulls all commits (needed for codecov)
          fetch-depth: 2
      - name: Download backend-native artifact
        uses: actions/download-artifact@v4
        with:
          name: "native-linux-x64-glibc-${{ matrix.python-version }}.node"
          path: ./packages/cubejs-backend-native/
      - name: Install Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Set Yarn version
        run: yarn policies set-version v1.22.22
      - name: Yarn install
        uses: nick-fields/retry@v3
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Lerna tsc
        run: yarn tsc
      - name: Build client
        run: yarn build
      - name: Lerna test
        run: yarn lerna run --concurrency 1 --stream --no-prefix unit
      - name: Fix lcov paths
        if: (matrix.node-version == '22.x' && matrix.transpile-worker-threads == true)
        run: |
          ./.github/actions/codecov-fix.sh
      - name: Combine all fixed LCOV files
        if: (matrix.node-version == '22.x' && matrix.transpile-worker-threads == true)
        run: |
          echo "" > ./combined-unit.lcov
          find ./packages -type f -name lcov.fixed.info -exec cat {} + >> ./combined-unit.lcov || true
      - name: Upload coverage artifact
        if: (matrix.node-version == '22.x' && matrix.transpile-worker-threads == true)
        uses: actions/upload-artifact@v4
        with:
          name: coverage-unit
          path: ./combined-unit.lcov

  lint:
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    needs: latest-tag-sha
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)

    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: 1.84.1
          # override: true # this is by default on
          rustflags: ""
          components: rustfmt
      - name: Install Node.js 22.x
        uses: actions/setup-node@v4
        with:
          node-version: 22.x
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Set Yarn version
        run: yarn policies set-version v1.22.22
      - name: Yarn install
        uses: nick-fields/retry@v3
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Check Yarn lock wasn't modified
        run: if [ "$(git status | grep nothing)x" = "x" ]; then echo "Non empty changeset after lerna bootstrap"; git status; exit 1; else echo "Nothing to commit. Proceeding"; fi;
      - name: NPM lint
        run: yarn lint:npm
      - name: Lerna lint
        run: yarn lerna run --concurrency 1 lint
      - name: Cargo fmt cubeorchestrator
        run: |
          cargo fmt --manifest-path rust/cubeorchestrator/Cargo.toml -- --check
      - name: Cargo fmt cubenativeutils
        run: |
          cargo fmt --manifest-path rust/cubenativeutils/Cargo.toml -- --check
      - name: Cargo fmt cubeshared
        run: |
          cargo fmt --manifest-path rust/cubeshared/Cargo.toml -- --check
#      - name: Cargo fmt cubesql
#        run: |
#          cargo fmt --manifest-path rust/cubesql/Cargo.toml -- --check
      - name: Cargo fmt cubesqlplanner
        run: |
          cargo fmt --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -- --check

  unit-core:
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    needs: latest-tag-sha
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)

    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: 1.84.1
          components: rustfmt
      - name: Cargo test cubeorchestrator
        run: cargo test --manifest-path rust/cubeorchestrator/Cargo.toml -j 4
      - name: Cargo test cubenativeutils
        run: cargo test --manifest-path rust/cubenativeutils/Cargo.toml -j 4
      - name: Cargo test cubeshared
        run: cargo test --manifest-path rust/cubeshared/Cargo.toml -j 4
      - name: Cargo test cubesqlplanner
        run: cargo test --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -j 4

  build-cubestore:
    needs: [latest-tag-sha]
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)
    container:
      image: cubejs/rust-cross:x86_64-unknown-linux-gnu-15082024
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: nightly-2024-01-29
          # override: true # this is by default on
          rustflags: ""
          components: rustfmt
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: ./rust/cubestore -> target
          # Separate path for release key to protect cache bloating
          shared-key: cubestore-release
          key: ubuntu-24.04
      - name: Build Cube Store
        run: |
          cd rust/cubestore
          cargo build --release -j 4 -p cubestore
      - name: 'Upload cubestored-x86_64-unknown-linux-gnu-release artifact'
        uses: actions/upload-artifact@v4
        with:
          name: cubestored-x86_64-unknown-linux-gnu-release
          path: ./rust/cubestore/target/release/cubestored
          retention-days: 5

  build-native-linux:
    runs-on: ubuntu-24.04
    needs: [latest-tag-sha]
    timeout-minutes: 60
    name: "Build native linux (Python: ${{ matrix.python-version }})"
    strategy:
      matrix:
        # Don't forget to align with PYTHON_VERSION_CURRENT
        python-version: [ 3.11 ]
      fail-fast: false
    container:
      image: cubejs/rust-cross:x86_64-unknown-linux-gnu-15082024-python-${{ matrix.python-version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: 1.84.1
          # override: true # this is by default on
          rustflags: ""
          components: rustfmt
          target: x86_64-unknown-linux-gnu
          cache: false
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: ./packages/cubejs-backend-native
          key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu
          shared-key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu
      - name: Install Node.js 22
        uses: actions/setup-node@v4
        with:
          node-version: 22
      - name: Install Yarn
        run: npm install -g yarn
      - name: Set Yarn version
        run: yarn policies set-version v1.22.22
        # We don't need to install all yarn deps to build native
      - name: Install cargo-cp-artifact
        run: npm install -g cargo-cp-artifact@0.1
      - name: Build native (with Python)
        env:
          PYO3_PYTHON: python${{ env.PYTHON_VERSION_CURRENT }}
          CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu
        working-directory: ./packages/cubejs-backend-native
        run: yarn run native:build-release-python
      - name: Store build artifact for dev image
        uses: actions/upload-artifact@v4
        with:
          name: "native-linux-x64-glibc-${{ env.PYTHON_VERSION_CURRENT }}.node"
          path: ./packages/cubejs-backend-native/index.node
          overwrite: true

  integration-cubestore:
    needs: [latest-tag-sha, build-cubestore, build-native-linux]
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)

    strategy:
      matrix:
        node-version: [22.x]
      fail-fast: false

    steps:
      - name: Maximize build space (disk space limitations)
        run: |
          echo "Before"
          df -h
          sudo apt-get remove -y 'php.*'
          sudo apt-get remove -y '^mongodb-.*'
          sudo apt-get remove -y '^mysql-.*'
          sudo apt-get autoremove -y
          sudo apt-get clean

          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          echo "After"
          df -h
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION_CURRENT }}
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Set Yarn version
        run: yarn policies set-version v1.22.22
      - name: Yarn install
        uses: nick-fields/retry@v3
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Lerna tsc
        run: yarn tsc
      - name: Download backend-native artifact
        uses: actions/download-artifact@v4
        with:
          name: "native-linux-x64-glibc-${{ env.PYTHON_VERSION_CURRENT }}.node"
          path: ./packages/cubejs-backend-native/
      - name: Download cubestored artifact
        uses: actions/download-artifact@v4
        with:
          path: ./rust/cubestore/target/release/
          name: cubestored-x86_64-unknown-linux-gnu-release
      - name: Run Cube Store in background
        run: |
          chmod +x ./rust/cubestore/target/release/cubestored
          ./rust/cubestore/target/release/cubestored &
      - name: Run Cubestore Integration
        timeout-minutes: 10
        run: |
          yarn lerna run --concurrency 1 --stream --no-prefix integration:cubestore

  integration:
    needs: [unit, lint, latest-tag-sha, build-native-linux]
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)
    env:
      CLOUD_DATABASES: >
        athena
        bigquery
        snowflake
        firebolt
        dremio
      # Athena (just to check for secrets availability)
      DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY }}

    strategy:
      matrix:
        node-version: [22.x]
        db: [
            'athena', 'bigquery', 'snowflake',
            'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb',
            'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt', 'dremio', 'vertica'
        ]
      fail-fast: false

    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download backend-native artifact
        uses: actions/download-artifact@v4
        with:
          name: "native-linux-x64-glibc-${{ env.PYTHON_VERSION_CURRENT }}.node"
          path: ./packages/cubejs-backend-native/
      - name: Install Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: 1.84.1
          # override: true # this is by default on
          rustflags: ""
          components: rustfmt
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION_CURRENT }}
      - name: Install Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Set Yarn version
        run: yarn policies set-version v1.22.22
      - name: Yarn install
        uses: nick-fields/retry@v3
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Build Core Client libraries
        run: yarn build
      - name: Lerna tsc
        run: yarn tsc
      - name: Run Integration tests for ${{ matrix.db }} matrix
        uses: nick-fields/retry@v3
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 30
          command: ./.github/actions/integration/${{ matrix.db }}.sh
        # It's enough to test for any one secret because they are set all at once or not set all
        if: |
          (contains(env.CLOUD_DATABASES, matrix.db) && env.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY != '') ||
          (!contains(env.CLOUD_DATABASES, matrix.db))
        env:
          # Firebolt Integration
          DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME }}
          DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME }}
          DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT }}
          DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER }}
          DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS }}
          # Dremio Integration
          DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL }}
          DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME }}
          DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN }}

          # BigQuery
          DRIVERS_TESTS_SNOWFLAKE_CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.CUBEJS_DB_BQ_CREDENTIALS }}

          # AWS Athena
          DRIVERS_TESTS_SNOWFLAKE_CUBEJS_AWS_KEY: ${{ secrets.CUBEJS_AWS_KEY }}
          DRIVERS_TESTS_SNOWFLAKE_CUBEJS_AWS_SECRET: ${{ secrets.CUBEJS_AWS_SECRET }}

          # Snowflake
          DRIVERS_TESTS_SNOWFLAKE_CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER }}
          DRIVERS_TESTS_SNOWFLAKE_CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS }}
      - name: Fix lcov paths
        run: |
          ./.github/actions/codecov-fix.sh
      - name: Combine all fixed LCOV files
        run: |
          echo "" > ./combined-integration-${{ matrix.db }}.lcov
          find ./packages -type f -name lcov.fixed.info -exec cat {} + >> ./combined-integration-${{ matrix.db }}.lcov || true
      - name: Upload coverage artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-integration-${{ matrix.db }}
          path: ./combined-integration-${{ matrix.db }}.lcov

  integration-smoke:
    needs: [latest-tag-sha, build-cubestore, build-native-linux]
    runs-on: ubuntu-24.04
    timeout-minutes: 90
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)

    strategy:
      matrix:
        node-version: [ 22.x ]
        python-version: [ 3.11 ]
      fail-fast: false

    steps:
      - name: Maximize build space (disk space limitations)
        run: |
          echo "Before"
          df -h
          sudo apt-get remove -y 'php.*'
          sudo apt-get remove -y '^mongodb-.*'
          sudo apt-get remove -y '^mysql-.*'
          sudo apt-get autoremove -y
          sudo apt-get clean

          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          echo "After"
          df -h
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download backend-native artifact
        uses: actions/download-artifact@v4
        with:
          name: "native-linux-x64-glibc-${{ matrix.python-version }}.node"
          path: ./packages/cubejs-backend-native/
      - name: Install Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Set Yarn version
        run: yarn policies set-version v1.22.22
      - name: Yarn install
        uses: nick-fields/retry@v3
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Install instant client for Oracle
        uses: GoodManWEN/oracle-client-action@main
      - name: Build client
        run: yarn build
      - name: Lerna tsc
        run: yarn tsc
      - name: Download cubestored-x86_64-unknown-linux-gnu-release artifact
        uses: actions/download-artifact@v4
        with:
          path: rust/cubestore/downloaded/latest/bin/
          name: cubestored-x86_64-unknown-linux-gnu-release
      - name: Chmod +x for cubestored
        run: |
          chmod +x ./rust/cubestore/downloaded/latest/bin/cubestored
      - name: Run Integration smoke tests
        timeout-minutes: 30
        run: ./.github/actions/smoke.sh

  docker-image-latest-set-tag:
    # At least git should be completed pushed up until this moment
    needs: [lint, latest-tag-sha]
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)
    runs-on: ubuntu-24.04
    outputs:
      tag: ${{ steps.get-tag.outputs.tag }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - id: get-tag
        run: echo "tag=$(git tag --contains "$GITHUB_SHA")" >> "$GITHUB_OUTPUT"
        env:
          GITHUB_SHA: ${{ github.sha }}

  latest-tag-sha:
    runs-on: ubuntu-24.04
    outputs:
      sha: ${{ steps.get-tag.outputs.sha }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - id: git-log
        run: git log HEAD~30..HEAD
      - id: get-tag-test
        run: echo "$SHA $(git rev-list -n 1 "$(git tag --contains "$SHA")")"
        env:
          SHA: ${{ github.sha }}
      - id: get-tag
        run: echo "sha=$(git rev-list -n 1 "$(git tag --contains "$SHA")")" >> "$GITHUB_OUTPUT"
        env:
          SHA: ${{ github.sha }}
      - id: get-tag-out
        run: echo "$OUT"
        env:
          OUT: ${{ steps.get-tag.outputs.sha }}

  docker-dev:
    needs: [latest-tag-sha, build-native-linux]
    if: (needs['latest-tag-sha'].outputs.sha != github.sha)
    name: Build & Test :dev for ${{ matrix.name }} without pushing
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    services:
      registry:
        image: registry:2
        ports:
          - 5000:5000
    strategy:
      matrix:
        node-version: [ 22 ]
        target: [ "x86_64-unknown-linux-gnu" ]
        dockerfile:
          - dev.Dockerfile
        include:
          - dockerfile: dev.Dockerfile
            name: Debian
            tag: tmp-dev
      fail-fast: false
    steps:
      - name: Maximize build space (disk space limitations)
        run: |
          echo "Before"
          df -h
          sudo apt-get remove -y 'php.*'
          sudo apt-get remove -y '^mongodb-.*'
          sudo apt-get remove -y '^mysql-.*'
          sudo apt-get autoremove -y
          sudo apt-get clean

          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          echo "After"
          df -h
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download backend-native artifact
        uses: actions/download-artifact@v4
        with:
          name: "native-linux-x64-glibc-${{ env.PYTHON_VERSION_CURRENT }}.node"
          path: ./packages/cubejs-backend-native/
      - name: Install Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      - name: Install Yarn
        run: npm install -g yarn
      - name: Set Yarn version
        run: yarn policies set-version v1.22.22
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Yarn install
        uses: nick-fields/retry@v3
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Build client
        run: yarn build
      - name: Lerna tsc
        run: yarn tsc
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      # current .dockerignore prevents use of native build
      - name: Unignore native from .dockerignore
        run: |
          grep -v -E "packages/cubejs-backend-native/((native)|(index.node))" .dockerignore > .dockerignore.tmp
          mv .dockerignore.tmp .dockerignore
      - name: Build image
        uses: docker/build-push-action@v6
        timeout-minutes: 30
        with:
          context: .
          file: ./packages/cubejs-docker/${{ matrix.dockerfile }}
          platforms: linux/amd64
          push: true
          tags: localhost:5000/cubejs/cube:${{ matrix.tag }}
      - name: Testing CubeJS (container mode) via BirdBox
        run: |
          cd packages/cubejs-testing/
          export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
          export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
          export DEBUG=testcontainers
          yarn run dataset:minimal
          yarn run birdbox:postgresql
          yarn run birdbox:postgresql-pre-aggregations
      # - name: Testing Athena driver (container mode) via BirdBox
      #   env:
      #     CUBEJS_AWS_KEY: ${{ secrets.CUBEJS_AWS_KEY }}
      #     CUBEJS_AWS_SECRET: ${{ secrets.CUBEJS_AWS_SECRET }}
      #     CUBEJS_AWS_REGION: us-east-1
      #     CUBEJS_AWS_S3_OUTPUT_LOCATION: s3://cubejs-opensource/testing/output
      #     CUBEJS_DB_EXPORT_BUCKET: s3://cubejs-opensource/testing/export
      #   run: |
      #     cd packages/cubejs-testing/
      #     export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
      #     export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
      #     export DEBUG=testcontainers
      #     yarn run driver:athena --log=ignore --mode=docker
      # - name: Testing BigQuery driver (container mode) via BirdBox
      #   env:
      #     CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.CUBEJS_DB_BQ_CREDENTIALS }}
      #     CUBEJS_DB_BQ_PROJECT_ID: cube-open-source
      #     CUBEJS_DB_EXPORT_BUCKET: cube-open-source-export-bucket
      #   run: |
      #     cd packages/cubejs-testing/
      #     export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
      #     export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
      #     export DEBUG=testcontainers
      #     yarn run driver:bigquery --log=ignore --mode=docker
      - name: Testing PostgreSQL driver (container mode) via BirdBox
        env:
          CUBEJS_DB_TYPE: postgres
          CUBEJS_DB_USER: postgres
          CUBEJS_DB_PASS: postgres
        run: |
          cd packages/cubejs-testing/
          export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
          export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
          export DEBUG=testcontainers
          yarn run driver:postgres --log=ignore --mode=docker
      - name: Testing Docker image via Cypress (Chrome)
        env:
          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
          BIRDBOX_CYPRESS_UPDATE_SCREENSHOTS: ${{ contains(github.event.head_commit.message, '[update screenshots]') }}
        run: |
          cd packages/cubejs-testing/
          export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
          export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
          export BIRDBOX_CYPRESS_BROWSER=chrome
          export BIRDBOX_CYPRESS_TARGET=postgresql
          export DEBUG=testcontainers
          yarn run cypress:install
          yarn run cypress:birdbox
      - name: Upload screenshots on failure
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: cypress-screenshots-docker-dev-${{ matrix.name }}
          path: packages/cubejs-testing/cypress/screenshots

  upload-coverage:
    name: Upload merged coverage to Codecov
    needs: [unit, integration]
    runs-on: ubuntu-latest
    permissions:
      contents: read
      actions: read
    steps:
      - name: Install Codecov CLI
        run: |
          curl -Os https://uploader.codecov.io/latest/linux/codecov
          chmod +x codecov
      - name: Download all coverage artifacts
        uses: actions/download-artifact@v4
        with:
          path: all-coverage
      - name: Merge all coverage files
        run: |
          find all-coverage -name '*.lcov' -exec cat {} + > merged-coverage.info
      - name: Upload merged coverage to Codecov
        run: |
          ./codecov --token "${{ secrets.CODECOV_TOKEN }}" \
              --file ./merged-coverage.info \
              --flags cube-backend \
              --name "cube backend coverage"