performance-test.yml

name: Performance Tests

on:
  push:
    branches:
    - "staging"
    paths:
    - '**/*.go'
    - '**/*.java'

permissions:
  actions: read
  id-token: write
  contents: read

jobs:
  performance-tests:
    environment: Performance Testing
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
        - driver: mysql
          driver_upper: MYSQL
        - driver: postgres
          driver_upper: POSTGRES
        # TODO: add benchmark tests for the below databases
        # - driver: mongodb
        #   driver_upper: MONGODB
        # - driver: oracle
        #   driver_upper: ORACLE
      fail-fast: false

    steps:
    - name: Checkout code
      uses: actions/checkout@v3

    - name: Set up Go
      uses: actions/setup-go@v4
      with:
        go-version: '1.24.2'

    - name: Set up Java for Maven
      uses: actions/setup-java@v3
      with:
        distribution: 'temurin'
        java-version: '17'

    - name: Setup VPN Client
      if: matrix.driver != 'oracle'
      run: |
        sudo apt-get update -qq && sudo apt-get install -y openvpn iproute2 iputils-ping netcat-openbsd telnet
        sudo mkdir -p /etc/openvpn/client && sudo chmod 700 /etc/openvpn/client
        echo "${{ secrets[format('{0}_OPENVPN_CONFIG', matrix.driver_upper)] }}" | base64 --decode | sudo tee /etc/openvpn/client/client.ovpn > /dev/null
        echo "${{ secrets.OPENVPN_USERNAME }}" | sudo tee /etc/openvpn/client/auth.txt > /dev/null
        sudo chmod 600 /etc/openvpn/client/client.ovpn /etc/openvpn/client/auth.txt
        sudo chown root:root /etc/openvpn/client/client.ovpn /etc/openvpn/client/auth.txt
        sudo openvpn --config /etc/openvpn/client/client.ovpn --daemon ovpn-client --log /var/log/openvpn-client.log --verb 3
        echo "Establishing VPN connection..."
        for i in {1..30}; do
          if ip addr show | grep -q "tun0\|tap0"; then echo "✅ VPN connected"; break; fi
          [ $i -eq 30 ] && { echo "❌ VPN timeout"; sudo cat /var/log/openvpn-client.log; exit 1; }
          sleep 2
        done
        sudo resolvectl dns tun0 ${{ secrets.VPN_DNS_SERVER }}
        sudo resolvectl domain tun0 ~private.postgres.database.azure.com

    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-session-name: performance_test_gh
        role-to-assume: ${{ secrets.AWS_GITHUB_ROLE }}
        aws-region: ${{ secrets.AWS_REGION }}

    - name: Install Go Dependencies
      run: go mod download

    - name: Build Project
      run: go build -v ./...

    - name: Build Iceberg Sink
      working-directory: ./destination/iceberg/olake-iceberg-java-writer
      run: mvn clean package -DskipTests

    - name: Create source
      run: |
        echo '${{ secrets[format('{0}_SOURCE_JSON', matrix.driver_upper)] }}' | base64 --decode > ./drivers/${{ matrix.driver }}/internal/testdata/source.json

    - name: Create destination
      run: |
        echo '${{ secrets[format('{0}_DESTINATION_JSON', matrix.driver_upper)] }}' | base64 --decode > ./drivers/${{ matrix.driver }}/internal/testdata/destination.json
    
    - name: Get Last Successful Run ID
      id: last_run
      env:
        GH_TOKEN: ${{ github.token }}
      run: |
        PREVIOUS_ID=$(gh run list --workflow ".github/workflows/performance-test.yml" --branch "staging" --status success --limit 1 --json databaseId --jq '.[0].databaseId')
        
        if [ -n "$PREVIOUS_ID" ]; then
          echo "id=$PREVIOUS_ID" >> $GITHUB_OUTPUT
        fi

    - name: Download Benchmarks History
      if: steps.last_run.outputs.id != ''
      uses: actions/download-artifact@v4
      continue-on-error: true
      with:
        name: benchmarks_${{ matrix.driver }}
        path: ./drivers/${{ matrix.driver }}/internal/testdata/
        run-id: ${{ steps.last_run.outputs.id }}
        github-token: ${{ github.token }}
    
    - name: Run Performance Tests
      run: go test -v ./drivers/${{ matrix.driver }}/internal/... -timeout 0 -run 'Performance'
    
    - name: Upload Benchmarks History
      uses: actions/upload-artifact@v4
      if: success()
      continue-on-error: true
      with:
        name: benchmarks_${{ matrix.driver }}
        path: ./drivers/${{ matrix.driver }}/internal/testdata/benchmarks.json
        retention-days: 90
   
    - name: Cleanup
      if: always()
      run: |
        # Delete all Glue databases matching prefix
        for db in $(aws glue get-databases \
          --query "DatabaseList[?starts_with(Name, 'performance_${{ matrix.driver }}')].Name" \
          --output text); do
          echo "Deleting Glue database: $db"
          aws glue delete-database --name "$db" \
          || echo "Failed to delete Glue database: $db"
        done

        # Delete corresponding S3 path
        aws s3 rm s3://dz-stag-github-actions/performance_${{ matrix.driver }}/ --recursive \
        || { echo "Failed to delete S3 bucket: performance_${{ matrix.driver }}"; \
        aws s3 ls s3://dz-stag-github-actions/performance_${{ matrix.driver }} || true; }

        echo "Catalog cleanup completed"

        if [[ "${{ matrix.driver }}" != "oracle" ]]; then
            sudo pkill -f "openvpn.*client.ovpn" || true
            sudo rm -rf /etc/openvpn/client/ /var/log/openvpn-client.log || true
            echo "VPN cleanup completed"
        fi