eval-coverage.yml

# Eval Coverage Gap Check
# Runs on PRs that touch hook, tool, or prompt files.
# Detects when code changes affect eval-covered paths without adding new evals.
#
# This workflow is a prototype for GSoC integration into the gemini-cli CI.
# In production, it would run the scanner against the real evals/ directory.

name: Eval Coverage Check
on:
  pull_request:
    paths:
      - 'packages/core/src/hooks/**'
      - 'packages/core/src/tools/**'
      - 'packages/core/src/prompt/**'
      - 'evals/**'

jobs:
  eval-coverage:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Install eval toolkit
        run: |
          cd eval-toolkit
          npm ci

      - name: Run eval gap analysis
        id: gaps
        run: |
          cd eval-toolkit
          npx tsx src/cli.ts gaps ../evals --json > /tmp/gap-report.json
          CRITICAL=$(jq '.gaps | map(select(.severity == "critical")) | length' /tmp/gap-report.json)
          echo "critical_gaps=$CRITICAL" >> "$GITHUB_OUTPUT"

      - name: Run simulator-based eval tests
        run: |
          cd eval-toolkit
          npm test

      - name: Comment on PR if new critical gaps
        if: steps.gaps.outputs.critical_gaps > 0
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const report = JSON.parse(fs.readFileSync('/tmp/gap-report.json', 'utf8'));
            const critical = report.gaps.filter(g => g.severity === 'critical');
            const body = [
              '## ⚠️ Eval Coverage Gaps Detected',
              '',
              `This PR touches hook/tool/prompt paths with **${critical.length} critical** eval coverage gaps:`,
              '',
              ...critical.map(g => `- **${g.description}**${g.relatedChanges?.length ? ` (${g.relatedChanges.join(', ')})` : ''}`),
              '',
              'Consider adding behavioral evals for these paths. See [CONTRIBUTING.md](./eval-toolkit/CONTRIBUTING.md) for the eval authoring workflow.',
            ].join('\n');

            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body,
            });