|
| 1 | +# Purpose: Run the first-party eval golden dataset via the n8n delegator. |
| 2 | +# Docs: docs/RFC-test-automation.md, cmd/sin-code/internal/eval/eval.doc.md |
| 3 | +# n8n-delegated per mandate M1 (see AGENTS.md §M1). |
| 4 | +name: eval-n8n |
| 5 | + |
| 6 | +on: |
| 7 | + push: |
| 8 | + branches: [main] |
| 9 | + paths: |
| 10 | + - 'evals/**' |
| 11 | + - 'cmd/sin-code/internal/eval/**' |
| 12 | + - 'cmd/sin-code/internal/evalharness/**' |
| 13 | + - 'cmd/sin-code/internal/dataset/**' |
| 14 | + - 'cmd/sin-code/eval_cmd.go' |
| 15 | + - '.github/workflows/eval-n8n.yml' |
| 16 | + pull_request: |
| 17 | + paths: |
| 18 | + - 'evals/**' |
| 19 | + - 'cmd/sin-code/internal/eval/**' |
| 20 | + - 'cmd/sin-code/internal/evalharness/**' |
| 21 | + - 'cmd/sin-code/internal/dataset/**' |
| 22 | + - 'cmd/sin-code/eval_cmd.go' |
| 23 | + workflow_dispatch: |
| 24 | + inputs: |
| 25 | + dataset: |
| 26 | + description: 'Dataset path to run (default: evals/test-generation.json)' |
| 27 | + required: false |
| 28 | + default: 'evals/test-generation.json' |
| 29 | + min_pass_rate: |
| 30 | + description: 'Minimum pass rate (0.0..1.0)' |
| 31 | + required: false |
| 32 | + default: '0.8' |
| 33 | + |
| 34 | +jobs: |
| 35 | + delegate: |
| 36 | + name: Delegate eval run to n8n |
| 37 | + runs-on: ubuntu-latest |
| 38 | + timeout-minutes: 2 |
| 39 | + steps: |
| 40 | + - name: Trigger n8n webhook |
| 41 | + env: |
| 42 | + N8N_CI_WEBHOOK_URL: ${{ secrets.N8N_CI_WEBHOOK_URL }} |
| 43 | + run: | |
| 44 | + if [[ -z "$N8N_CI_WEBHOOK_URL" ]]; then |
| 45 | + echo "::error::N8N_CI_WEBHOOK_URL secret is not configured" |
| 46 | + exit 1 |
| 47 | + fi |
| 48 | + curl -fsS -X POST \ |
| 49 | + -H "Content-Type: application/json" \ |
| 50 | + -d '{"workflow":"eval-n8n","ref":"${{ github.ref }}","sha":"${{ github.sha }}","repo":"${{ github.repository }}","actor":"${{ github.actor }}","dataset":"${{ inputs.dataset || 'evals/test-generation.json' }}","min_pass_rate":"${{ inputs.min_pass_rate || '0.8' }}"}' \ |
| 51 | + "$N8N_CI_WEBHOOK_URL" |
| 52 | + - name: Echo local pre-checks |
| 53 | + run: | |
| 54 | + go version |
| 55 | + go vet ./cmd/sin-code/internal/eval/... |
| 56 | + go vet ./cmd/sin-code/internal/evalharness/... |
| 57 | + go vet ./cmd/sin-code/internal/dataset/... |
| 58 | + echo "Heavy eval work delegated to n8n." |
0 commit comments