From 783816f969bc9c78b3f65adfdc4f998e411bce5b Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 26 Jul 2026 05:02:15 -0700 Subject: [PATCH] =?UTF-8?q?feat(orb):=20golden=20gate=20corpus=20=E2=80=94?= =?UTF-8?q?=20production=20failure=20archetypes=20pinned=20as=20replayable?= =?UTF-8?q?=20verdicts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ORB's decision policy shipped without an evaluation gate, in a repo that enforces 99% patch coverage on everything else. This closes that asymmetry: a versioned corpus of decision-input snapshots drawn from the real failure archetypes (the inconclusive-AI incident class, the reversal classes, mode-flip hardening, the not-synced neutral contract), each replayed through the PURE evaluateGateCheck and pinned to its expected conclusion + blocker set. Closes #8832. Part of epic #8828 (Phase 2 -- regression gate). - knownBad entries carry a standing invariant: they may NEVER evaluate to success, under their own policy OR the permissive default -- the exact regression class that motivated the epic - append-only workflow: a production failure becomes a labeled corpus entry in the same PR that fixes it - runs in validate-tests on every PR (the issue's fast tier -- the corpus is pure and millisecond-cheap), so a dedicated path-filtered CI job would add machinery without adding protection; the full AI-inclusive tier lands with #8834/#8838's model-input capture, as noted on the issue - two corpus expectations were corrected against reality during construction (guardrail holds are policy-driven, the slop blocker code is slop_risk_above_threshold) -- exactly the drift-finding this corpus exists to do --- test/golden-corpus/gate-corpus.json | 326 +++++++++++++++++++++++++++ test/unit/gate-golden-corpus.test.ts | 65 ++++++ 2 files changed, 391 insertions(+) create mode 100644 test/golden-corpus/gate-corpus.json create mode 100644 test/unit/gate-golden-corpus.test.ts diff --git a/test/golden-corpus/gate-corpus.json b/test/golden-corpus/gate-corpus.json new file mode 100644 index 0000000000..38ba172007 --- /dev/null +++ b/test/golden-corpus/gate-corpus.json @@ -0,0 +1,326 @@ +{ + "$comment": "Golden gate corpus (#8832, epic #8828). Each entry replays the PURE gate evaluation (evaluateGateCheck) against a decision-input snapshot drawn from a REAL production failure archetype, pinning the expected conclusion and blocker set. knownBad entries additionally assert the invariant that they may NEVER evaluate to success \u2014 the exact regression class that motivated the epic. Version bumps when entry semantics change; additions are append-only.", + "version": 1, + "entries": [ + { + "id": "clean-pr-succeeds", + "source": "baseline", + "description": "A clean PR with no findings passes the gate.", + "knownBad": false, + "findings": [], + "policy": {}, + "expected": { + "conclusion": "success", + "blockerCodes": [] + } + }, + { + "id": "secret-leak-always-fails", + "source": "incident", + "description": "A detected secret is a hard blocker under every configuration.", + "knownBad": true, + "findings": [ + { + "code": "secret_leak", + "severity": "critical", + "title": "Secret detected", + "detail": "A credential-shaped string was added." + } + ], + "policy": {}, + "expected": { + "conclusion": "failure", + "blockerCodes": [ + "secret_leak" + ] + } + }, + { + "id": "inconclusive-ai-holds-not-passes", + "source": "incident-8794", + "description": "PRs 8735 et al: a block-mode AI review that returned no usable verdict must HOLD (neutral), never pass automatically and never fail on the model's own hiccup.", + "knownBad": false, + "findings": [ + { + "code": "ai_review_inconclusive", + "severity": "warning", + "title": "AI review inconclusive", + "detail": "The dual-model AI review did not return a usable verdict." + } + ], + "policy": { + "aiReviewGateMode": "block" + }, + "expected": { + "conclusion": "neutral", + "blockerCodes": [] + } + }, + { + "id": "inconclusive-ai-cannot-bury-a-blocker", + "source": "incident-8794", + "description": "An inconclusive AI review alongside a real deterministic blocker must still FAIL \u2014 the hold path must never soften a genuine violation.", + "knownBad": true, + "findings": [ + { + "code": "ai_review_inconclusive", + "severity": "warning", + "title": "AI review inconclusive", + "detail": "No usable verdict." + }, + { + "code": "secret_leak", + "severity": "critical", + "title": "Secret detected", + "detail": "A credential-shaped string was added." + } + ], + "policy": { + "aiReviewGateMode": "block" + }, + "expected": { + "conclusion": "failure", + "blockerCodes": [ + "secret_leak" + ] + } + }, + { + "id": "guardrail-path-holds", + "source": "baseline", + "description": "A guardrail-protected path (policy.guardrailHit) yields a manual-review HOLD (neutral), never a failure and never an auto-merge.", + "knownBad": false, + "findings": [], + "policy": { + "guardrailHit": true, + "guardrailMatches": [ + { + "path": "migrations/0001_init.sql", + "glob": "migrations/**" + } + ] + }, + "expected": { + "conclusion": "neutral", + "blockerCodes": [] + } + }, + { + "id": "linked-issue-block-mode-fails", + "source": "reversal-class", + "description": "Missing linked issue under block mode is a hard failure.", + "knownBad": false, + "findings": [ + { + "code": "missing_linked_issue", + "severity": "warning", + "title": "No linked issue", + "detail": "The PR body links no eligible open issue." + } + ], + "policy": { + "linkedIssueGateMode": "block" + }, + "expected": { + "conclusion": "failure", + "blockerCodes": [ + "missing_linked_issue" + ] + } + }, + { + "id": "linked-issue-advisory-mode-passes", + "source": "reversal-class", + "description": "The SAME missing-linked-issue finding under advisory mode passes with a warning \u2014 mode flips must never silently harden.", + "knownBad": false, + "findings": [ + { + "code": "missing_linked_issue", + "severity": "warning", + "title": "No linked issue", + "detail": "The PR body links no eligible open issue." + } + ], + "policy": { + "linkedIssueGateMode": "advisory" + }, + "expected": { + "conclusion": "success", + "blockerCodes": [] + } + }, + { + "id": "duplicate-block-mode-fails", + "source": "reversal-class", + "description": "Duplicate-PR risk under block mode fails the gate.", + "knownBad": false, + "findings": [ + { + "code": "duplicate_pr_risk", + "severity": "warning", + "title": "Duplicate PR", + "detail": "Substantially overlaps an open PR." + } + ], + "policy": { + "duplicatePrGateMode": "block" + }, + "expected": { + "conclusion": "failure", + "blockerCodes": [ + "duplicate_pr_risk" + ] + } + }, + { + "id": "slop-below-floor-blocks-when-opted-in", + "source": "reversal-class", + "description": "Deterministic slop at/over the configured floor is a hard blocker only under slopGateMode: block.", + "knownBad": false, + "findings": [], + "policy": { + "slopGateMode": "block", + "slopGateMinScore": 60, + "slopRisk": 72 + }, + "expected": { + "conclusion": "failure", + "blockerCodes": [ + "slop_risk_above_threshold" + ] + } + }, + { + "id": "slop-under-threshold-passes", + "source": "baseline", + "description": "Slop risk under the floor passes even in block mode.", + "knownBad": false, + "findings": [], + "policy": { + "slopGateMode": "block", + "slopGateMinScore": 60, + "slopRisk": 35 + }, + "expected": { + "conclusion": "success", + "blockerCodes": [] + } + }, + { + "id": "not-synced-is-neutral-never-blocks", + "source": "baseline", + "description": "The app's own unsynced state (pr_not_cached) must never block a contributor.", + "knownBad": false, + "findings": [ + { + "code": "pr_not_cached", + "severity": "info", + "title": "Not synced", + "detail": "LoopOver has not finished syncing this repo." + } + ], + "policy": {}, + "expected": { + "conclusion": "neutral", + "blockerCodes": [] + } + }, + { + "id": "ai-consensus-defect-block-mode-fails", + "source": "disagreement-class", + "description": "A confident dual-model consensus defect under aiReviewGateMode: block is a hard blocker.", + "knownBad": false, + "findings": [ + { + "code": "ai_consensus_defect", + "severity": "critical", + "title": "Consensus defect", + "detail": "Both reviewers found the same defect.", + "confidence": 0.97 + } + ], + "policy": { + "aiReviewGateMode": "block", + "aiReviewCloseConfidence": 0.93 + }, + "expected": { + "conclusion": "failure", + "blockerCodes": [ + "ai_consensus_defect" + ] + } + }, + { + "id": "ai-defect-advisory-only-disposition-passes", + "source": "disagreement-class", + "description": "advisory_only low-confidence disposition: a SUB-FLOOR consensus defect does not block (#4603's advisory_only arm).", + "knownBad": false, + "findings": [ + { + "code": "ai_consensus_defect", + "severity": "critical", + "title": "Consensus defect", + "detail": "Same defect, low confidence.", + "confidence": 0.4 + } + ], + "policy": { + "aiReviewGateMode": "block", + "aiReviewCloseConfidence": 0.93, + "aiReviewLowConfidenceDisposition": "advisory_only" + }, + "expected": { + "conclusion": "success", + "blockerCodes": [] + } + }, + { + "id": "dry-run-previews-without-enforcing", + "source": "baseline", + "description": "Dry-run: an advisory-mode AI defect posts SUCCESS as the real conclusion while displayConclusion previews the would-be failure.", + "knownBad": false, + "findings": [ + { + "code": "ai_consensus_defect", + "severity": "critical", + "title": "Consensus defect", + "detail": "Both reviewers found the same defect.", + "confidence": 0.97 + } + ], + "policy": { + "aiReviewGateMode": "advisory", + "aiReviewCloseConfidence": 0.93, + "dryRun": true + }, + "expected": { + "conclusion": "success", + "blockerCodes": [], + "displayConclusion": "failure" + } + }, + { + "id": "merge-readiness-composite-escalates-subgates", + "source": "baseline", + "description": "The merge-readiness composite escalates linked-issue/duplicate/slop to its mode with one switch.", + "knownBad": false, + "findings": [ + { + "code": "missing_linked_issue", + "severity": "warning", + "title": "No linked issue", + "detail": "The PR body links no eligible open issue." + } + ], + "policy": { + "mergeReadinessGateMode": "block" + }, + "expected": { + "conclusion": "failure", + "blockerCodes": [ + "missing_linked_issue" + ] + } + } + ] +} \ No newline at end of file diff --git a/test/unit/gate-golden-corpus.test.ts b/test/unit/gate-golden-corpus.test.ts new file mode 100644 index 0000000000..85daee6fd3 --- /dev/null +++ b/test/unit/gate-golden-corpus.test.ts @@ -0,0 +1,65 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { evaluateGateCheck, type GateCheckPolicy } from "../../src/rules/advisory"; +import type { Advisory, AdvisoryFinding } from "../../src/types"; + +// #8832 (epic #8828): the golden gate corpus. Each entry replays the PURE gate evaluation against a +// decision-input snapshot drawn from a real production failure archetype and pins the expected conclusion + +// blocker set. This suite runs on every PR (the fast tier the issue requires — the corpus is pure and +// millisecond-cheap); entries marked knownBad additionally pin the invariant that they may NEVER evaluate to +// success, the exact regression class that motivated the epic. Adding to the corpus is append-only: a +// production failure becomes a labeled entry here in the same PR that fixes it. +type CorpusEntry = { + id: string; + source: string; + description: string; + knownBad: boolean; + findings: Array & { confidence?: number }>; + policy: GateCheckPolicy; + expected: { conclusion: string; blockerCodes: string[]; displayConclusion?: string }; +}; + +const corpus = JSON.parse(readFileSync("test/golden-corpus/gate-corpus.json", "utf8")) as { version: number; entries: CorpusEntry[] }; + +function advisoryOf(entry: CorpusEntry): Advisory { + return { + id: `golden-${entry.id}`, + targetType: "pull_request", + targetKey: "golden/corpus#1", + repoFullName: "golden/corpus", + pullNumber: 1, + conclusion: "neutral", + severity: "info", + title: "golden", + summary: "golden corpus entry", + findings: entry.findings as AdvisoryFinding[], + generatedAt: "2026-07-26T00:00:00.000Z", + }; +} + +describe("golden gate corpus (#8832)", () => { + it("corpus file is well-formed: version, unique ids, non-empty archetype coverage", () => { + expect(corpus.version).toBe(1); + expect(corpus.entries.length).toBeGreaterThanOrEqual(14); + expect(new Set(corpus.entries.map((entry) => entry.id)).size).toBe(corpus.entries.length); + // The corpus must always carry at least one knownBad guard — the never-flips-to-merge invariant is its point. + expect(corpus.entries.some((entry) => entry.knownBad)).toBe(true); + }); + + it.each(corpus.entries.map((entry) => [entry.id, entry] as const))("replays %s to its pinned verdict", (_id, entry) => { + const evaluation = evaluateGateCheck(advisoryOf(entry), entry.policy); + expect(evaluation.conclusion).toBe(entry.expected.conclusion); + expect(evaluation.blockers.map((blocker) => blocker.code).sort()).toEqual([...entry.expected.blockerCodes].sort()); + if (entry.expected.displayConclusion !== undefined) { + expect(evaluation.displayConclusion).toBe(entry.expected.displayConclusion); + } + }); + + it("INVARIANT: no knownBad entry may EVER evaluate to success — under its own policy or the permissive default", () => { + for (const entry of corpus.entries.filter((candidate) => candidate.knownBad)) { + expect(evaluateGateCheck(advisoryOf(entry), entry.policy).conclusion).not.toBe("success"); + // A knownBad violation must also survive a maintainer relaxing every optional mode to default. + expect(evaluateGateCheck(advisoryOf(entry), {}).conclusion).not.toBe("success"); + } + }); +});