Feat/graphql delta fix #131
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generic Benchmark CI | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| packages: | |
| description: 'Packages to benchmark (e.g. ./packages/server/...)' | |
| required: true | |
| default: './packages/server/pkg/flow/simulation' | |
| type: string | |
| count: | |
| description: 'Number of benchmark runs' | |
| required: false | |
| default: '5' | |
| type: string | |
| force_comparison: | |
| description: 'Force comparison even if no previous artifacts' | |
| required: false | |
| default: false | |
| type: boolean | |
| pull_request: | |
| paths: | |
| - 'packages/server/**' | |
| - 'tools/benchmark/**' | |
| - '.github/workflows/benchmark-generic.yml' | |
| push: | |
| paths: | |
| - 'packages/server/**' | |
| - 'tools/benchmark/**' | |
| - '.github/workflows/benchmark-generic.yml' | |
| branches: [main] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| benchmark: | |
| name: Run Benchmarks | |
| runs-on: ubuntu-latest | |
| outputs: | |
| comparison: ${{ steps.compare-results.outputs.comparison }} | |
| has-regressions: ${{ steps.compare-results.outputs.has-regressions }} | |
| has-changes: ${{ steps.check-changes.outputs.changed }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup runner environment | |
| uses: ./.github/actions/setup | |
| - name: Check if relevant files changed | |
| id: check-changes | |
| run: | | |
| if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| echo "changed=true" >> $GITHUB_OUTPUT | |
| else | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| BASE="${{ github.event.pull_request.base.sha }}" | |
| else | |
| BASE="${{ github.event.before }}" | |
| fi | |
| CHANGED=$(git diff --name-only $BASE...HEAD | grep -E "^packages/server/|^tools/benchmark/|^.github/workflows/benchmark-generic.yml" || true) | |
| if [ -n "$CHANGED" ]; then | |
| echo "changed=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "changed=false" >> $GITHUB_OUTPUT | |
| fi | |
| fi | |
| - name: Determine Target Packages | |
| id: target | |
| if: steps.check-changes.outputs.changed == 'true' || github.event_name == 'workflow_dispatch' | |
| run: | | |
| if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| echo "packages=${{ github.event.inputs.packages }}" >> $GITHUB_OUTPUT | |
| echo "count=${{ github.event.inputs.count }}" >> $GITHUB_OUTPUT | |
| else | |
| # Default for PR/Push: simulation package which is most critical | |
| echo "packages=./packages/server/pkg/flow/simulation" >> $GITHUB_OUTPUT | |
| echo "count=5" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Run Current Benchmarks | |
| id: run-current | |
| if: steps.check-changes.outputs.changed == 'true' || github.event_name == 'workflow_dispatch' | |
| run: | | |
| mkdir -p bench_data | |
| # Run benchmarks using standard go test | |
| go test -bench=. -benchmem -run=^$ \ | |
| -count=${{ steps.target.outputs.count }} \ | |
| -timeout=30m \ | |
| ${{ steps.target.outputs.packages }} | tee bench_data/current.txt | |
| # Parse to JSON for artifact storage and comparison | |
| go run tools/benchmark/*.go parse \ | |
| --input=bench_data/current.txt \ | |
| --output=bench_data/current.json | |
| - name: Download Baseline Artifacts | |
| if: (steps.check-changes.outputs.changed == 'true' || github.event_name == 'workflow_dispatch') | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| echo "Looking for latest successful run on main..." | |
| LATEST_RUN_ID=$(gh run list --workflow="benchmark-generic.yml" --branch=main --status=success --event=push --limit=1 --json databaseId --jq '.[0].databaseId') | |
| if [ -n "$LATEST_RUN_ID" ]; then | |
| echo "Found run ID: $LATEST_RUN_ID. Downloading artifact..." | |
| mkdir -p bench_data/previous | |
| gh run download $LATEST_RUN_ID -n benchmark-results-main -D bench_data/previous || echo "⚠️ Failed to download artifact (it might not exist or expired)." | |
| else | |
| echo "⚠️ No successful previous runs found on main." | |
| fi | |
| - name: Run Baseline (If Artifact Missing) | |
| if: (steps.check-changes.outputs.changed == 'true' || github.event_name == 'workflow_dispatch') | |
| run: | | |
| if [ ! -f "bench_data/previous/current.json" ]; then | |
| echo "⚠️ No baseline artifact found. Running baseline on current checkout (approximate)..." | |
| # Ideally we would checkout 'main' here, but for simplicity in this V1 we might skip or warn. | |
| # A robust implementation would fetch the base commit. | |
| # For now, let's skip comparison if missing, UNLESS forced. | |
| if [ "${{ github.event.inputs.force_comparison }}" == "true" ]; then | |
| echo "Running baseline benchmarks..." | |
| go test -bench=. -benchmem -run=^$ \ | |
| -count=${{ steps.target.outputs.count }} \ | |
| -timeout=30m \ | |
| ${{ steps.target.outputs.packages }} | tee bench_data/baseline.txt | |
| go run tools/benchmark/*.go parse \ | |
| --input=bench_data/baseline.txt \ | |
| --output=bench_data/baseline.json | |
| else | |
| echo "Skipping baseline generation." | |
| fi | |
| else | |
| mv bench_data/previous/current.json bench_data/baseline.json | |
| fi | |
| - name: Compare Results | |
| id: compare-results | |
| if: steps.check-changes.outputs.changed == 'true' || github.event_name == 'workflow_dispatch' | |
| run: | | |
| if [ -f "bench_data/baseline.json" ] && [ -f "bench_data/current.json" ]; then | |
| go run tools/benchmark/*.go compare \ | |
| --baseline=bench_data/baseline.json \ | |
| --current=bench_data/current.json \ | |
| --output-md=bench_data/comparison.md \ | |
| --output-json=bench_data/comparison.json || { | |
| echo "has-regressions=true" >> $GITHUB_OUTPUT | |
| } | |
| # Output to Job Summary | |
| cat bench_data/comparison.md >> $GITHUB_STEP_SUMMARY | |
| # Read markdown for PR comment | |
| COMPARISON=$(cat bench_data/comparison.md | sed 's/`/\\`/g' | sed 's/$/\\n/' | tr -d '\n') | |
| echo "comparison<<EOF" >> $GITHUB_OUTPUT | |
| echo "$COMPARISON" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| else | |
| echo "Skipping comparison (missing files)." | |
| echo "has-regressions=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Upload Artifacts | |
| if: steps.check-changes.outputs.changed == 'true' || github.event_name == 'workflow_dispatch' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results-${{ github.sha }} | |
| path: bench_data/ | |
| retention-days: 14 | |
| - name: Upload Main Baseline (Push Only) | |
| if: github.ref == 'refs/heads/main' && github.event_name == 'push' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results-main | |
| path: bench_data/current.json | |
| retention-days: 30 | |
| comment: | |
| name: Post Results Comment | |
| runs-on: ubuntu-latest | |
| needs: benchmark | |
| if: | | |
| always() && | |
| needs.benchmark.result == 'success' && | |
| needs.benchmark.outputs.has-changes == 'true' && | |
| github.com.event_name == 'pull_request' | |
| steps: | |
| - name: Find and update PR comment | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const comparison = `${{ needs.benchmark.outputs.comparison }}`; | |
| const hasRegressions = `${{ needs.benchmark.outputs.has-regressions }}` === 'true'; | |
| if (!comparison) return; | |
| // Find existing performance comment | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| const existingComment = comments.find(comment => | |
| comment.user.type === 'Bot' && | |
| (comment.body.includes('📊 Performance Comparison')) | |
| ); | |
| let commentBody = comparison; | |
| if (hasRegressions) { | |
| commentBody = '\n⚠️ **Performance regressions detected!**\n\n' + commentBody; | |
| } | |
| if (existingComment) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: existingComment.id, | |
| body: commentBody, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: commentBody, | |
| }); | |
| } |