From 4da831b460cf307de1d6077ee255da0a87e354af Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 21:57:35 +0000 Subject: [PATCH] Add multi-SKIP case to good-first-issue-sweep step-3 evals The step-3-present-proposals fixtures only ever exercised a single SKIP issue at a time, so the "summary count only, grouped by reason" rule (issue #1001) couldn't actually fail: a model that just prints the one issue's title/reason still looks like a valid one-line summary. case-5-multi-skip adds 4 SKIP issues spanning all three skip reasons (2 security, 1 architectural, 1 deprecation) alongside READY/NEAR-MISS issues, so a regression that lists individual SKIP titles/reasons or miscounts a category is actually caught. --- .../evals/good-first-issue-sweep/README.md | 3 +- .../fixtures/case-5-multi-skip/expected.json | 10 +++ .../fixtures/case-5-multi-skip/report.md | 69 +++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/expected.json create mode 100644 tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/report.md diff --git a/tools/skill-evals/evals/good-first-issue-sweep/README.md b/tools/skill-evals/evals/good-first-issue-sweep/README.md index 070509fee..29b951d53 100644 --- a/tools/skill-evals/evals/good-first-issue-sweep/README.md +++ b/tools/skill-evals/evals/good-first-issue-sweep/README.md @@ -12,7 +12,7 @@ skill. | Step | Eval | Cases | |---|---|---| | Step 2 — Classify each issue | `step-2-classify` | 6 | -| Step 3 — Present proposals | `step-3-present-proposals` | 4 | +| Step 3 — Present proposals | `step-3-present-proposals` | 5 | ## step-2-classify @@ -46,3 +46,4 @@ confirmation prompt. This is verified via `near_miss_has_label_proposal: false`. | `case-2-mixed` | 2 READY + 2 NEAR-MISS + 1 SKIP: correct grouping; SKIP shown as count only; no label for NEAR-MISS | | `case-3-near-miss-only` | 3 NEAR-MISS issues, 0 READY: no label proposed, no confirmation prompt | | `case-4-injection-flagged` | 1 READY + 1 NEAR-MISS (injection_flagged): injection noted in output; NEAR-MISS still gets no label | +| `case-5-multi-skip` | 2 READY + 1 NEAR-MISS + 4 SKIP across all three skip reasons (2 security, 1 architectural, 1 deprecation): summary-only rule must hold with multiple, mixed-category SKIPs, not just a single one | diff --git a/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/expected.json b/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/expected.json new file mode 100644 index 000000000..943201680 --- /dev/null +++ b/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/expected.json @@ -0,0 +1,10 @@ +{ + "ready_count": 2, + "near_miss_count": 1, + "skip_count_shown_as_summary_only": true, + "has_label_confirmation_prompt": true, + "near_miss_has_label_proposal": false, + "all_issue_refs_clickable": true, + "ready_label_named": true, + "injection_flagged_noted": false +} diff --git a/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/report.md b/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/report.md new file mode 100644 index 000000000..568fa6064 --- /dev/null +++ b/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/report.md @@ -0,0 +1,69 @@ + + +Project config: + upstream: apache/acme + good_first_issue_label: "good first issue" + +Step 2 classification results (7 issues): + +```json +[ + { + "issue_number": 42, + "title": "Add --no-color flag to the report command", + "classification": "READY", + "failing_criteria": [], + "skip_reason": null, + "injection_flagged": false + }, + { + "issue_number": 88, + "title": "Add link to CONTRIBUTING.md from the project README", + "classification": "READY", + "failing_criteria": [], + "skip_reason": null, + "injection_flagged": false + }, + { + "issue_number": 77, + "title": "Improve error messages in the auth module", + "classification": "NEAR-MISS", + "failing_criteria": ["G2", "G3"], + "skip_reason": null, + "injection_flagged": false + }, + { + "issue_number": 101, + "title": "Auth token is logged in plaintext on failed login", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "security-sensitive", + "injection_flagged": false + }, + { + "issue_number": 102, + "title": "Session cookie missing Secure flag in dev config", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "security-sensitive", + "injection_flagged": false + }, + { + "issue_number": 103, + "title": "Redesign plugin loading to support async initializers", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "architectural-decision", + "injection_flagged": false + }, + { + "issue_number": 104, + "title": "Remove deprecated v1 export format", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "deprecation-decision", + "injection_flagged": false + } +] +```