swebench-nightly #27
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Purpose: Nightly SWE-bench evaluation via n8n delegator (mandate M1). | |
| # Docs: docs/swebench.md | |
| name: swebench-nightly | |
| on: | |
| schedule: | |
| - cron: '0 6 * * *' # daily at 06:00 UTC | |
| workflow_dispatch: | |
| inputs: | |
| dataset: | |
| description: 'SWE-bench dataset path' | |
| required: false | |
| default: 'evals/swe-bench-example.json' | |
| max_instances: | |
| description: 'Max instances to evaluate (0 = all)' | |
| required: false | |
| default: '10' | |
| jobs: | |
| delegate: | |
| name: Delegate SWE-bench run to n8n | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-go@v6 | |
| with: | |
| go-version: '1.26' | |
| - name: Local validation | |
| run: | | |
| go version | |
| go vet ./cmd/sin-code/internal/eval/... | |
| go vet ./cmd/sin-code/internal/swebench/... | |
| echo "Heavy SWE-bench work delegated to n8n." | |
| - name: Trigger n8n webhook | |
| env: | |
| N8N_CI_WEBHOOK_URL: ${{ secrets.N8N_CI_WEBHOOK_URL }} | |
| run: | | |
| if [[ -z "$N8N_CI_WEBHOOK_URL" ]]; then | |
| echo "::warning::N8N_CI_WEBHOOK_URL secret is not configured — skipping n8n delegation. Local validation passed." | |
| exit 0 | |
| fi | |
| DATASET="${{ inputs.dataset || 'evals/swe-bench-example.json' }}" | |
| MAX_INSTANCES="${{ inputs.max_instances || '10' }}" | |
| curl -fsS -X POST \ | |
| -H "Content-Type: application/json" \ | |
| -d "{\"workflow\":\"swebench-nightly\",\"ref\":\"${{ github.ref }}\",\"sha\":\"${{ github.sha }}\",\"repo\":\"${{ github.repository }}\",\"actor\":\"${{ github.actor }}\",\"dataset\":\"$DATASET\",\"max_instances\":\"$MAX_INSTANCES\"}" \ | |
| "$N8N_CI_WEBHOOK_URL" |