Skip to content

Commit 1bcc634

Browse files
author
SIN CI
committed
ci(eval): add n8n-delegated eval workflow for test-generation dataset
1 parent 718ff25 commit 1bcc634

1 file changed

Lines changed: 58 additions & 0 deletions

File tree

.github/workflows/eval-n8n.yml

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Purpose: Run the first-party eval golden dataset via the n8n delegator.
2+
# Docs: docs/RFC-test-automation.md, cmd/sin-code/internal/eval/eval.doc.md
3+
# n8n-delegated per mandate M1 (see AGENTS.md §M1).
4+
name: eval-n8n
5+
6+
on:
7+
push:
8+
branches: [main]
9+
paths:
10+
- 'evals/**'
11+
- 'cmd/sin-code/internal/eval/**'
12+
- 'cmd/sin-code/internal/evalharness/**'
13+
- 'cmd/sin-code/internal/dataset/**'
14+
- 'cmd/sin-code/eval_cmd.go'
15+
- '.github/workflows/eval-n8n.yml'
16+
pull_request:
17+
paths:
18+
- 'evals/**'
19+
- 'cmd/sin-code/internal/eval/**'
20+
- 'cmd/sin-code/internal/evalharness/**'
21+
- 'cmd/sin-code/internal/dataset/**'
22+
- 'cmd/sin-code/eval_cmd.go'
23+
workflow_dispatch:
24+
inputs:
25+
dataset:
26+
description: 'Dataset path to run (default: evals/test-generation.json)'
27+
required: false
28+
default: 'evals/test-generation.json'
29+
min_pass_rate:
30+
description: 'Minimum pass rate (0.0..1.0)'
31+
required: false
32+
default: '0.8'
33+
34+
jobs:
35+
delegate:
36+
name: Delegate eval run to n8n
37+
runs-on: ubuntu-latest
38+
timeout-minutes: 2
39+
steps:
40+
- name: Trigger n8n webhook
41+
env:
42+
N8N_CI_WEBHOOK_URL: ${{ secrets.N8N_CI_WEBHOOK_URL }}
43+
run: |
44+
if [[ -z "$N8N_CI_WEBHOOK_URL" ]]; then
45+
echo "::error::N8N_CI_WEBHOOK_URL secret is not configured"
46+
exit 1
47+
fi
48+
curl -fsS -X POST \
49+
-H "Content-Type: application/json" \
50+
-d '{"workflow":"eval-n8n","ref":"${{ github.ref }}","sha":"${{ github.sha }}","repo":"${{ github.repository }}","actor":"${{ github.actor }}","dataset":"${{ inputs.dataset || 'evals/test-generation.json' }}","min_pass_rate":"${{ inputs.min_pass_rate || '0.8' }}"}' \
51+
"$N8N_CI_WEBHOOK_URL"
52+
- name: Echo local pre-checks
53+
run: |
54+
go version
55+
go vet ./cmd/sin-code/internal/eval/...
56+
go vet ./cmd/sin-code/internal/evalharness/...
57+
go vet ./cmd/sin-code/internal/dataset/...
58+
echo "Heavy eval work delegated to n8n."

0 commit comments

Comments
 (0)