From 36ec0aaa2a1864100822ce416e8f36df4157189b Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 26 Jul 2026 06:58:09 -0700 Subject: [PATCH 1/3] feat(scripts): backfill the risk-control calibration set from decision history ai_review_cache has retained per-finding confidence since 2026-06-28 and review_audit holds every acted gate decision, so the risk-control label floor can be cleared from existing evidence instead of weeks of live accrual. Pure core stages acted closes (close_arm; closed-then-merged is the definitive incorrect class) and AI-judgment-only holds (holdout_close, the below-floor strata a lower certified threshold needs); the wrapper reuses buildDecisionRecord/canonicalJson so backfilled records are byte-compatible, with configDigest 'backfill:unavailable' as the honest provenance sentinel and no ledger writes (the chain attests live finalizations only). --- scripts/backfill-decision-labels-core.ts | 149 ++++++++++++++++++ scripts/backfill-decision-labels.ts | 131 +++++++++++++++ .../backfill-decision-labels-core.test.ts | 135 ++++++++++++++++ 3 files changed, 415 insertions(+) create mode 100644 scripts/backfill-decision-labels-core.ts create mode 100644 scripts/backfill-decision-labels.ts create mode 100644 test/unit/backfill-decision-labels-core.test.ts diff --git a/scripts/backfill-decision-labels-core.ts b/scripts/backfill-decision-labels-core.ts new file mode 100644 index 0000000000..e4e20ac12c --- /dev/null +++ b/scripts/backfill-decision-labels-core.ts @@ -0,0 +1,149 @@ +// Backfill the risk-control calibration set from decision history (#8828 epic follow-through) — PURE core. +// +// The risk-control guarantee (#8835) needs adjudicated (confidence, correct?) pairs, and until 2026-07-26 +// decision records did not persist confidence — but ai_review_cache has retained per-finding confidence +// since 2026-06-28, and review_audit holds every acted gate decision. This core reconstructs the calibration +// candidates from those two histories so the label floor (199 clean close labels) can be cleared from +// EXISTING evidence instead of waiting weeks for live accrual. +// +// Population rules, both load-bearing: +// • ACTED CLOSES (stratum close_arm): the decision the guarantee governs. A realized outcome of `merged` +// on a target the gate CLOSED means the PR was later reopened and merged — the definitive incorrect-close +// class — so those candidates carry `definitiveAdjudication: "incorrect"` (no judgment needed). +// • HOLDS whose ONLY blockers were AI-judgment findings (stratum holdout_close): the would-have-closed-at- +// a-lower-threshold population. These are the pairs that let the sweep certify a λ̂ BELOW today's static +// floor — pairs at/above the floor alone can only re-bless the status quo. Holds with any non-AI blocker +// (CI red, conflicts, policy) are EXCLUDED: "would closing have been right" is unanswerable when the +// hold was about something other than the finding. +// +// Confidence reconstruction mirrors the live writer (processors.ts finalize site): the FIRST finding whose +// code is in AI_JUDGMENT_BLOCKER_CODES supplies the confidence (`gate.blockers.find(...)`), and findings_json +// preserves the derivation order the blocker list was built from. Candidates without a confidence-bearing +// AI-judgment finding are skipped (rule-only decisions cannot join a confidence-thresholded guarantee). +// +// PURE: decides WHAT to stage from rows the caller supplies. The thin IO wrapper owns stdin/stdout. +import { AI_JUDGMENT_BLOCKER_CODES } from "../src/rules/advisory"; + +/** One gate decision joined to its review's cached findings, as extracted from review_audit + ai_review_cache. */ +export type CandidateRow = { + targetId: string; // "owner/repo#123" + project: string; // "owner/repo" + pullNumber: number; + decision: "close" | "hold"; + headSha: string | null; + decidedAt: string; // review_audit.created_at of the gate_decision row + findingsJson: string; // ai_review_cache.findings_json for the target + /** gate_outcomes.blocker_codes_json for holds (the codes that held it); null/absent for closes. */ + blockerCodesJson?: string | null | undefined; + /** Latest realized pr_outcome for the target, when one exists. */ + realizedOutcome?: "merged" | "closed" | null | undefined; +}; + +export type StagedCalibrationTarget = { + targetId: string; + project: string; + pullNumber: number; + headSha: string; + stratum: "close_arm" | "holdout_close"; + verdict: "close"; + outcome: "merged" | "closed" | null; + aiConfidence: number; + reasonCode: string; // the shaping finding's code — the record's clause + findingTitle: string; // for the adjudication worklist, never persisted to the record + decidedAt: string; + /** Set when the realized outcome already decides the label (close then merged = reopened+merged). */ + definitiveAdjudication: "incorrect" | null; +}; + +export type BackfillLabelPlan = { + staged: StagedCalibrationTarget[]; + skipped: { + noShapingFinding: number; + unparseableFindings: number; + mixedBlockerHold: number; + missingHeadSha: number; + duplicateTarget: number; + }; +}; + +type Finding = { code?: unknown; confidence?: unknown; title?: unknown }; + +/** The live selection contract: first AI-judgment finding, its confidence required numeric in [0, 1]. */ +function shapingFinding(findingsJson: string): { code: string; confidence: number; title: string } | "unparseable" | null { + let findings: unknown; + try { + findings = JSON.parse(findingsJson); + } catch { + return "unparseable"; + } + if (!Array.isArray(findings)) return "unparseable"; + for (const raw of findings as Finding[]) { + if (typeof raw?.code !== "string" || !AI_JUDGMENT_BLOCKER_CODES.has(raw.code)) continue; + if (typeof raw.confidence !== "number" || !Number.isFinite(raw.confidence) || raw.confidence < 0 || raw.confidence > 1) continue; + return { code: raw.code, confidence: raw.confidence, title: typeof raw.title === "string" ? raw.title : "" }; + } + return null; +} + +/** True when every code that held the PR is an AI-judgment code — the pure would-close population. */ +function holdWasAiJudgmentOnly(blockerCodesJson: string | null | undefined): boolean { + if (typeof blockerCodesJson !== "string" || blockerCodesJson.trim() === "") return false; + try { + const codes = JSON.parse(blockerCodesJson); + return Array.isArray(codes) && codes.length > 0 && codes.every((code) => typeof code === "string" && AI_JUDGMENT_BLOCKER_CODES.has(code)); + } catch { + return false; + } +} + +/** + * Stage calibration targets from candidate rows. One target contributes at most ONE pair (the UNIQUE + * target_id contract of decision_audit_labels): when a target appears as both an acted close and an earlier + * hold, the ACTED decision wins — it is the one the guarantee is about. + */ +export function planDecisionLabelBackfill(rows: CandidateRow[]): BackfillLabelPlan { + const skipped = { noShapingFinding: 0, unparseableFindings: 0, mixedBlockerHold: 0, missingHeadSha: 0, duplicateTarget: 0 }; + const staged: StagedCalibrationTarget[] = []; + const seen = new Set(); + const ordered = [...rows].sort((a, b) => (a.decision === b.decision ? 0 : a.decision === "close" ? -1 : 1)); + for (const row of ordered) { + if (seen.has(row.targetId)) { + skipped.duplicateTarget += 1; + continue; + } + if (row.decision === "hold" && !holdWasAiJudgmentOnly(row.blockerCodesJson)) { + skipped.mixedBlockerHold += 1; + continue; + } + const finding = shapingFinding(row.findingsJson); + if (finding === "unparseable") { + skipped.unparseableFindings += 1; + continue; + } + if (finding === null) { + skipped.noShapingFinding += 1; + continue; + } + if (typeof row.headSha !== "string" || row.headSha.trim() === "") { + skipped.missingHeadSha += 1; + continue; + } + seen.add(row.targetId); + const outcome = row.decision === "close" ? (row.realizedOutcome ?? null) : null; + staged.push({ + targetId: row.targetId, + project: row.project, + pullNumber: row.pullNumber, + headSha: row.headSha, + stratum: row.decision === "close" ? "close_arm" : "holdout_close", + verdict: "close", + outcome, + aiConfidence: finding.confidence, + reasonCode: finding.code, + findingTitle: finding.title, + decidedAt: row.decidedAt, + definitiveAdjudication: row.decision === "close" && outcome === "merged" ? "incorrect" : null, + }); + } + return { staged, skipped }; +} diff --git a/scripts/backfill-decision-labels.ts b/scripts/backfill-decision-labels.ts new file mode 100644 index 0000000000..2b16fcbb9d --- /dev/null +++ b/scripts/backfill-decision-labels.ts @@ -0,0 +1,131 @@ +#!/usr/bin/env node +// Backfill the risk-control calibration set (#8828 follow-through) — thin IO around +// backfill-decision-labels-core.ts. Reads candidate rows as JSON on stdin, emits a staging bundle on +// stdout. Deliberately infrastructure-free: the operator owns extraction and application. +// +// EXTRACT (run against the instance's Postgres; produces the stdin payload): +// SELECT json_agg(t) FROM ( +// SELECT ra.target_id AS "targetId", +// split_part(ra.target_id, '#', 1) AS "project", +// c.pull_number AS "pullNumber", +// ra.decision AS "decision", +// ra.head_sha AS "headSha", +// ra.created_at AS "decidedAt", +// c.findings_json AS "findingsJson", +// go.blocker_codes_json AS "blockerCodesJson", +// (SELECT ra2.decision FROM review_audit ra2 +// WHERE ra2.event_type = 'pr_outcome' AND ra2.target_id = ra.target_id +// ORDER BY ra2.created_at DESC LIMIT 1) AS "realizedOutcome" +// -- the LAST pr_outcome is the definitive story: merged after a close = reopened + merged +// FROM review_audit ra +// JOIN LATERAL ( +// SELECT * FROM ai_review_cache c +// WHERE c.repo_full_name || '#' || c.pull_number = ra.target_id +// AND c.findings_json LIKE '%confidence%' +// ORDER BY (c.head_sha = ra.head_sha) DESC, c.created_at DESC LIMIT 1 +// ) c ON true +// LEFT JOIN gate_outcomes go +// ON go.repo_full_name || '#' || go.pull_number = ra.target_id AND go.head_sha = ra.head_sha +// WHERE ra.event_type = 'gate_decision' AND ra.decision IN ('close', 'hold') +// AND ra.created_at >= '2026-06-28' +// ) t; +// +// STAGE: +// node --experimental-strip-types scripts/backfill-decision-labels.ts < candidates.json > bundle.json +// +// APPLY (after adjudication sign-off; both idempotent via ON CONFLICT DO NOTHING): +// \set records `jq -c .records bundle.json` +// INSERT INTO decision_records SELECT * FROM jsonb_populate_recordset(NULL::decision_records, :'records'::jsonb) +// ON CONFLICT (id) DO NOTHING; +// -- likewise .labels -> decision_audit_labels (staged status='pending'; adjudications are a later UPDATE) +// +// Backfilled records carry configDigest "backfill:unavailable" — the historical resolved config is +// unknowable and a fabricated sha256 would corrupt the commitment semantics; the sentinel is also the +// record's provenance marker. They are NOT appended to the decision ledger: the ledger attests decisions +// as they were finalized live, and reconstructed history has no place in that chain. +import { planDecisionLabelBackfill, type CandidateRow } from "./backfill-decision-labels-core.js"; +import { buildDecisionRecord, canonicalJson } from "../src/review/decision-record.js"; +import { DECISION_AUDIT_RUBRIC_VERSION } from "../src/review/decision-audit.js"; + +async function readStdin(): Promise { + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) chunks.push(chunk as Buffer); + return Buffer.concat(chunks).toString("utf8"); +} + +export async function buildBundle(rows: CandidateRow[], stagedAt: string): Promise> { + const plan = planDecisionLabelBackfill(rows); + const records = []; + const labels = []; + const worklist = []; + for (const target of plan.staged) { + const { record, recordDigest } = await buildDecisionRecord({ + repoFullName: target.project, + pullNumber: target.pullNumber, + headSha: target.headSha, + action: target.stratum === "close_arm" ? "close" : "hold", + reasonCode: target.reasonCode, + configDigest: "backfill:unavailable", + modelId: null, + promptDigest: null, + aiConfidence: target.aiConfidence, + decidedAt: target.decidedAt, + }); + records.push({ + id: `record:${target.targetId}@${target.headSha}`.slice(0, 250), + repo_full_name: target.project.slice(0, 200), + pull_number: target.pullNumber, + head_sha: target.headSha, + action: record.action, + reason_code: record.reasonCode.slice(0, 200), + record_digest: recordDigest, + record_json: canonicalJson(record), + created_at: target.decidedAt, + }); + labels.push({ + id: `audit:${target.targetId}`.slice(0, 190), + project: target.project.slice(0, 200), + target_id: target.targetId, + verdict: target.verdict, + outcome: target.outcome, + stratum: target.stratum, + rubric_version: DECISION_AUDIT_RUBRIC_VERSION, + sampled_at: stagedAt, + status: "pending", + adjudication: null, + reason_category: null, + adjudicated_at: null, + }); + worklist.push({ + targetId: target.targetId, + stratum: target.stratum, + confidence: target.aiConfidence, + reasonCode: target.reasonCode, + findingTitle: target.findingTitle.slice(0, 300), + decidedAt: target.decidedAt, + outcome: target.outcome, + definitiveAdjudication: target.definitiveAdjudication, + }); + } + return { stagedAt, skipped: plan.skipped, records, labels, worklist }; +} + +const invokedDirectly = process.argv[1]?.endsWith("backfill-decision-labels.ts") === true; +if (invokedDirectly) { + readStdin() + .then(async (raw) => { + const rows = JSON.parse(raw) as CandidateRow[]; + const bundle = await buildBundle(rows, new Date().toISOString()); + process.stdout.write(`${JSON.stringify(bundle, null, 1)}\n`); + const skipped = bundle.skipped as Record; + console.error( + `backfill-decision-labels: ${rows.length} candidate(s) -> ${(bundle.records as unknown[]).length} staged; skipped: ${Object.entries(skipped) + .map(([key, count]) => `${key}=${count}`) + .join(" ")}`, + ); + }) + .catch((error: unknown) => { + console.error(`backfill-decision-labels: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + }); +} diff --git a/test/unit/backfill-decision-labels-core.test.ts b/test/unit/backfill-decision-labels-core.test.ts new file mode 100644 index 0000000000..8aa566dea7 --- /dev/null +++ b/test/unit/backfill-decision-labels-core.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; +import { planDecisionLabelBackfill, type CandidateRow } from "../../scripts/backfill-decision-labels-core"; +import { buildBundle } from "../../scripts/backfill-decision-labels"; +import { canonicalJson, contentDigest } from "../../src/review/decision-record"; + +// The backfill's contracts: acted closes stage as close_arm (merged outcome = definitive incorrect), +// AI-judgment-only holds stage as holdout_close, everything else is skipped with its reason counted, +// and one target contributes at most one pair with the acted decision winning. + +function candidate(overrides: Partial): CandidateRow { + return { + targetId: "o/r#1", + project: "o/r", + pullNumber: 1, + decision: "close", + headSha: "sha1", + decidedAt: "2026-07-01T00:00:00.000Z", + findingsJson: JSON.stringify([{ code: "ai_consensus_defect", confidence: 0.9, title: "defect" }]), + realizedOutcome: "closed", + ...overrides, + }; +} + +describe("planDecisionLabelBackfill", () => { + it("stages an acted close as close_arm carrying the shaping finding's confidence", () => { + const plan = planDecisionLabelBackfill([candidate({})]); + expect(plan.staged).toHaveLength(1); + expect(plan.staged[0]).toMatchObject({ + stratum: "close_arm", + verdict: "close", + aiConfidence: 0.9, + reasonCode: "ai_consensus_defect", + outcome: "closed", + definitiveAdjudication: null, + }); + }); + + it("a closed-then-MERGED target is the definitive incorrect-close class", () => { + const plan = planDecisionLabelBackfill([candidate({ realizedOutcome: "merged" })]); + expect(plan.staged[0]!.definitiveAdjudication).toBe("incorrect"); + expect(plan.staged[0]!.outcome).toBe("merged"); + }); + + it("stages a hold as holdout_close ONLY when every blocker code was an AI judgment; outcome stays null", () => { + const pure = candidate({ decision: "hold", blockerCodesJson: JSON.stringify(["ai_consensus_defect"]) }); + const mixed = candidate({ + targetId: "o/r#2", + pullNumber: 2, + decision: "hold", + blockerCodesJson: JSON.stringify(["ai_consensus_defect", "ci_failing"]), + }); + const absent = candidate({ targetId: "o/r#3", pullNumber: 3, decision: "hold", blockerCodesJson: null }); + const empty = candidate({ targetId: "o/r#4", pullNumber: 4, decision: "hold", blockerCodesJson: "[]" }); + const garbled = candidate({ targetId: "o/r#5", pullNumber: 5, decision: "hold", blockerCodesJson: "{nope" }); + const plan = planDecisionLabelBackfill([pure, mixed, absent, empty, garbled]); + expect(plan.staged).toHaveLength(1); + expect(plan.staged[0]).toMatchObject({ stratum: "holdout_close", outcome: null, definitiveAdjudication: null }); + expect(plan.skipped.mixedBlockerHold).toBe(4); + }); + + it("selects the FIRST AI-judgment finding (the live gate.blockers.find contract), skipping non-judgment codes", () => { + const plan = planDecisionLabelBackfill([ + candidate({ + findingsJson: JSON.stringify([ + { code: "slop_signal", confidence: 0.99, title: "not a judgment" }, + { code: "ai_review_split", confidence: 0.62, title: "split" }, + { code: "ai_consensus_defect", confidence: 0.97, title: "later" }, + ]), + }), + ]); + expect(plan.staged[0]).toMatchObject({ aiConfidence: 0.62, reasonCode: "ai_review_split" }); + }); + + it("skips candidates with no usable confidence: missing, non-numeric, out of range, or no judgment finding", () => { + const plan = planDecisionLabelBackfill([ + candidate({ findingsJson: JSON.stringify([{ code: "ai_consensus_defect", title: "no confidence" }]) }), + candidate({ targetId: "o/r#2", pullNumber: 2, findingsJson: JSON.stringify([{ code: "ai_consensus_defect", confidence: "high" }]) }), + candidate({ targetId: "o/r#3", pullNumber: 3, findingsJson: JSON.stringify([{ code: "ai_consensus_defect", confidence: 1.7 }]) }), + candidate({ targetId: "o/r#4", pullNumber: 4, findingsJson: JSON.stringify([{ code: "readiness_low" }]) }), + ]); + expect(plan.staged).toHaveLength(0); + expect(plan.skipped.noShapingFinding).toBe(4); + }); + + it("counts unparseable findings and missing head shas separately", () => { + const plan = planDecisionLabelBackfill([ + candidate({ findingsJson: "{broken" }), + candidate({ targetId: "o/r#2", pullNumber: 2, findingsJson: JSON.stringify({ not: "an array" }) }), + candidate({ targetId: "o/r#3", pullNumber: 3, headSha: null }), + candidate({ targetId: "o/r#4", pullNumber: 4, headSha: " " }), + ]); + expect(plan.staged).toHaveLength(0); + expect(plan.skipped.unparseableFindings).toBe(2); + expect(plan.skipped.missingHeadSha).toBe(2); + }); + + it("one pair per target: the ACTED close wins over the same target's hold regardless of input order", () => { + const hold = candidate({ decision: "hold", blockerCodesJson: JSON.stringify(["ai_consensus_defect"]) }); + const close = candidate({ findingsJson: JSON.stringify([{ code: "ai_consensus_defect", confidence: 0.8, title: "t" }]) }); + const plan = planDecisionLabelBackfill([hold, close]); + expect(plan.staged).toHaveLength(1); + expect(plan.staged[0]).toMatchObject({ stratum: "close_arm", aiConfidence: 0.8 }); + expect(plan.skipped.duplicateTarget).toBe(1); + }); +}); + +describe("buildBundle", () => { + it("emits records whose digest matches their canonical JSON, backfill-sentinel config, and pending labels", async () => { + const bundle = await buildBundle([candidate({}), candidate({ targetId: "o/r#7", pullNumber: 7, decision: "hold", blockerCodesJson: '["ai_consensus_defect"]' })], "2026-07-26T12:00:00.000Z"); + const records = bundle.records as Array>; + const labels = bundle.labels as Array>; + expect(records).toHaveLength(2); + for (const row of records) { + const record = JSON.parse(String(row.record_json)) as Record; + expect(await contentDigest(record)).toBe(row.record_digest); + expect(canonicalJson(record)).toBe(row.record_json); + expect(record.configDigest).toBe("backfill:unavailable"); + expect(record.schemaVersion).toBe("2"); + } + expect(records[0]).toMatchObject({ id: "record:o/r#1@sha1", action: "close", created_at: "2026-07-01T00:00:00.000Z" }); + expect(records[1]).toMatchObject({ action: "hold" }); + const parsed = JSON.parse(String(records[0]!.record_json)) as { aiConfidence: number }; + expect(parsed.aiConfidence).toBe(0.9); + expect(labels[0]).toMatchObject({ + id: "audit:o/r#1", + status: "pending", + stratum: "close_arm", + rubric_version: "1", + sampled_at: "2026-07-26T12:00:00.000Z", + adjudication: null, + }); + expect(labels[1]).toMatchObject({ stratum: "holdout_close", outcome: null }); + expect((bundle.worklist as unknown[]).length).toBe(2); + }); +}); From 6768568679aac1a312dd2904186016ef9d878321 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 26 Jul 2026 07:24:37 -0700 Subject: [PATCH 2/3] feat(orb): pre-registered alpha(n) tightening schedule for the close arm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When no env alpha pins an arm, the close arm's error budget now follows a deterministic schedule of usable-label count: 0.05 under 350 pairs, 0.025 from 350, 0.015 from 700. Sample size is information, not the test statistic, so walking the schedule as labels accrue is not outcome snooping and each daily recalibration still runs exactly one level-delta test; the published guarantee self-tightens 95% -> 97.5% -> 98.5% with no human step. Also documents the backfill population finding: staged holdout_close rows are analysis-only — surviving holds are not valid close counterfactuals; only acted closes enter calibration. --- scripts/backfill-decision-labels-core.ts | 7 ++-- scripts/backfill-decision-labels.ts | 3 +- src/review/risk-control-wire.ts | 42 ++++++++++++++++++++---- test/unit/risk-control-wire.test.ts | 31 +++++++++++++---- 4 files changed, 67 insertions(+), 16 deletions(-) diff --git a/scripts/backfill-decision-labels-core.ts b/scripts/backfill-decision-labels-core.ts index e4e20ac12c..4121db63e8 100644 --- a/scripts/backfill-decision-labels-core.ts +++ b/scripts/backfill-decision-labels-core.ts @@ -10,8 +10,11 @@ // • ACTED CLOSES (stratum close_arm): the decision the guarantee governs. A realized outcome of `merged` // on a target the gate CLOSED means the PR was later reopened and merged — the definitive incorrect-close // class — so those candidates carry `definitiveAdjudication: "incorrect"` (no judgment needed). -// • HOLDS whose ONLY blockers were AI-judgment findings (stratum holdout_close): the would-have-closed-at- -// a-lower-threshold population. These are the pairs that let the sweep certify a λ̂ BELOW today's static +// • HOLDS whose ONLY blockers were AI-judgment findings (stratum holdout_close): staged for ANALYSIS, not +// for calibration — apply must skip them. Adjudicating the 2026-07 backfill proved they are NOT valid +// close counterfactuals: a hold that survived means some non-confidence criterion (split vs consensus, +// author tier) blocked the close, so "would close have been right" samples a different population than +// the acted-close guarantee governs. The LIVE ε-holdout (#8831) samples would-close PRs and stays valid. These are the pairs that let the sweep certify a λ̂ BELOW today's static // floor — pairs at/above the floor alone can only re-bless the status quo. Holds with any non-AI blocker // (CI red, conflicts, policy) are EXCLUDED: "would closing have been right" is unanswerable when the // hold was about something other than the finding. diff --git a/scripts/backfill-decision-labels.ts b/scripts/backfill-decision-labels.ts index 2b16fcbb9d..b7a4d335e3 100644 --- a/scripts/backfill-decision-labels.ts +++ b/scripts/backfill-decision-labels.ts @@ -33,7 +33,8 @@ // STAGE: // node --experimental-strip-types scripts/backfill-decision-labels.ts < candidates.json > bundle.json // -// APPLY (after adjudication sign-off; both idempotent via ON CONFLICT DO NOTHING): +// APPLY (after adjudication sign-off; both idempotent via ON CONFLICT DO NOTHING). CLOSE_ARM LABELS ONLY — +// holdout_close rows are analysis output, not calibration rows (see the core header's population note): // \set records `jq -c .records bundle.json` // INSERT INTO decision_records SELECT * FROM jsonb_populate_recordset(NULL::decision_records, :'records'::jsonb) // ON CONFLICT (id) DO NOTHING; diff --git a/src/review/risk-control-wire.ts b/src/review/risk-control-wire.ts index 4af1aaccea..8243397687 100644 --- a/src/review/risk-control-wire.ts +++ b/src/review/risk-control-wire.ts @@ -31,15 +31,42 @@ export function parseBudget(raw: string | undefined, fallback: number, max: numb } /** Per-arm error budgets (#8835's Neyman–Pearson requirement) and the calibration confidence level. - * Defaults: close α=0.015 (~199-label floor), merge α=0.005 (~598 — stricter than close by 3x; the earlier - * 0.002 draft needed ~1,497 labels, a year of adjudication for the last 3x of strictness, which contradicts - * the minimal-human-involvement objective this instrument serves). */ -export function riskControlArms(env: Env): Array<{ arm: "close" | "merge"; verdict: "close" | "merge"; alpha: number }> { + * An explicit env α pins the arm; when unset, the CLOSE arm follows the pre-registered α(n) schedule below + * and the merge arm stays at 0.005 (~598-label floor — stricter than close by design; the earlier 0.002 + * draft needed ~1,497 labels, a year of adjudication for the last 3x of strictness, which contradicts the + * minimal-human-involvement objective this instrument serves). */ +export function riskControlArms(env: Env): Array<{ arm: "close" | "merge"; verdict: "close" | "merge"; alpha: number | null }> { return [ - { arm: "close", verdict: "close", alpha: parseBudget(env.LOOPOVER_RISK_CONTROL_CLOSE_ALPHA, 0.015, 0.05) }, - { arm: "merge", verdict: "merge", alpha: parseBudget(env.LOOPOVER_RISK_CONTROL_MERGE_ALPHA, 0.005, 0.05) }, + { arm: "close", verdict: "close", alpha: parseBudgetOrNull(env.LOOPOVER_RISK_CONTROL_CLOSE_ALPHA, 0.05) }, + { arm: "merge", verdict: "merge", alpha: parseBudgetOrNull(env.LOOPOVER_RISK_CONTROL_MERGE_ALPHA, 0.05) }, ]; } + +/** Like parseBudget but with no fallback: null when absent/garbage/outside (0, max] — "use the schedule". */ +export function parseBudgetOrNull(raw: string | undefined, max: number): number | null { + const value = Number((raw ?? "").trim()); + if (!Number.isFinite(value) || value <= 0 || value > max) return null; + return value; +} + +/** + * The PRE-REGISTERED α(n) tightening schedule: the strongest error budget the arm's usable label count can + * power, chosen as a deterministic function of SAMPLE SIZE alone. Because n is information size, not the + * test statistic, walking this schedule as labels accrue is not outcome-snooping and needs no multiplicity + * correction — each daily recalibration still runs exactly ONE level-δ test. The alternative (testing an + * α-grid each day and publishing the strongest pass) would need a Bonferroni δ-split that raises every + * floor by ~35% and, at today's label volume, certifies nothing at all. + * + * Tiers (close): α=0.05 under 350 usable pairs (guarantee arrives at the honest floor of ~59 clean at-λ + * labels), 0.025 from 350, 0.015 from 700 — the homepage claim self-tightens 95% → 97.5% → 98.5% with zero + * human steps. Merge: fixed 0.005 until its own volume justifies a schedule (tracked on #8828's epic). + */ +export function scheduledAlpha(arm: "close" | "merge", usablePairs: number): number { + if (arm === "merge") return 0.005; + if (usablePairs >= 700) return 0.015; + if (usablePairs >= 350) return 0.025; + return 0.05; +} export function riskControlDelta(env: Env): number { return parseBudget(env.LOOPOVER_RISK_CONTROL_DELTA, 0.05, 0.2); } @@ -86,9 +113,10 @@ export async function loadCalibrationPairs(env: Env, verdict: "close" | "merge", /** One arm's recalibration (global when `project` is null, else that repo's own labels): calibrate → * publish or retract. Best-effort per arm. */ -async function recalibrateArm(env: Env, arm: string, verdict: "close" | "merge", alpha: number, project: string | null): Promise { +async function recalibrateArm(env: Env, arm: "close" | "merge", verdict: "close" | "merge", envAlpha: number | null, project: string | null): Promise { const delta = riskControlDelta(env); const pairs = await loadCalibrationPairs(env, verdict, project); + const alpha = envAlpha ?? scheduledAlpha(arm, pairs.length); const result = calibrateActThreshold(pairs, alpha, delta); const scope = project === null ? arm : `${arm}:${project}`; if (result.status === "calibrated") { diff --git a/test/unit/risk-control-wire.test.ts b/test/unit/risk-control-wire.test.ts index 74a5ef5f63..438e426598 100644 --- a/test/unit/risk-control-wire.test.ts +++ b/test/unit/risk-control-wire.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { isRiskControlEnabled, loadCalibrationPairs, parseBudget, readCalibratedThreshold, resolveAutomaticCloseConfidence, riskControlArms, riskControlFlagKey, runRiskControlRecalibration } from "../../src/review/risk-control-wire"; +import { isRiskControlEnabled, loadCalibrationPairs, parseBudget, parseBudgetOrNull, readCalibratedThreshold, resolveAutomaticCloseConfidence, riskControlArms, riskControlFlagKey, runRiskControlRecalibration, scheduledAlpha } from "../../src/review/risk-control-wire"; import { processJob } from "../../src/queue/processors"; import { createTestEnv } from "../helpers/d1"; @@ -69,7 +69,7 @@ describe("runRiskControlRecalibration", () => { const flag = await env.DB.prepare(`SELECT value FROM system_flags WHERE key = ?`).bind(riskControlFlagKey("close")).first(); expect(flag).toBeFalsy(); // a stale guarantee is a lie — retracted const audit = await env.DB.prepare(`SELECT detail FROM audit_events WHERE event_type = 'risk_control_insufficient' AND target_key = 'riskcontrol:close'`).first<{ detail: string }>(); - expect(audit!.detail).toContain("of 199 needed"); // close-arm alpha 0.015 floor + expect(audit!.detail).toContain("of 59 needed"); // scheduled close alpha 0.05 under 350 pairs → 59-label floor }); it("CALIBRATED: publishes lambda + the certified statement once the close arm clears its floor", async () => { @@ -77,13 +77,14 @@ describe("runRiskControlRecalibration", () => { for (let i = 1; i <= 210; i += 1) await seedLabeledDecision(env, i, "close", "correct", 0.9 + (i % 5) / 100); const summary = await runRiskControlRecalibration(env); expect(summary.close).toBe("calibrated"); - expect(summary.merge).toBe("insufficient_labels"); // merge alpha 0.002 needs 1497 — genuinely separate arms + expect(summary.merge).toBe("insufficient_labels"); // merge alpha 0.005 needs 598 — genuinely separate arms const flag = await env.DB.prepare(`SELECT value FROM system_flags WHERE key = ?`).bind(riskControlFlagKey("close")).first<{ value: string }>(); - const stored = JSON.parse(flag!.value) as { lambda: number; coverageAtLambda: number }; + const stored = JSON.parse(flag!.value) as { lambda: number; coverageAtLambda: number; alpha: number }; expect(stored.lambda).toBe(0.9); expect(stored.coverageAtLambda).toBe(1); + expect(stored.alpha).toBe(0.05); // 210 pairs < 350 → the schedule's first tier const audit = await env.DB.prepare(`SELECT detail FROM audit_events WHERE event_type = 'risk_control_calibrated'`).first<{ detail: string }>(); - expect(audit!.detail).toContain("P(wrong | acted) ≤ 0.015 guaranteed at 100% coverage"); + expect(audit!.detail).toContain("P(wrong | acted) ≤ 0.05 guaranteed at 100% coverage"); }); }); @@ -169,7 +170,25 @@ describe("env-configurable budgets", () => { expect(parseBudget("0.01", 0.015, 0.05)).toBe(0.01); const arms = riskControlArms(createTestEnv({ LOOPOVER_RISK_CONTROL_CLOSE_ALPHA: "0.02" })); expect(arms.find((a) => a.arm === "close")!.alpha).toBe(0.02); - expect(arms.find((a) => a.arm === "merge")!.alpha).toBe(0.005); + expect(arms.find((a) => a.arm === "merge")!.alpha).toBeNull(); // unset env = follow the schedule + }); + + it("parseBudgetOrNull: null on absent/garbage/out-of-range means 'use the schedule'", () => { + expect(parseBudgetOrNull(undefined, 0.05)).toBeNull(); + expect(parseBudgetOrNull("nope", 0.05)).toBeNull(); + expect(parseBudgetOrNull("0.2", 0.05)).toBeNull(); + expect(parseBudgetOrNull("0", 0.05)).toBeNull(); + expect(parseBudgetOrNull("0.03", 0.05)).toBe(0.03); + }); + + it("scheduledAlpha: the pre-registered close-arm tightening tiers; merge stays fixed at 0.005", () => { + expect(scheduledAlpha("close", 0)).toBe(0.05); + expect(scheduledAlpha("close", 349)).toBe(0.05); + expect(scheduledAlpha("close", 350)).toBe(0.025); + expect(scheduledAlpha("close", 699)).toBe(0.025); + expect(scheduledAlpha("close", 700)).toBe(0.015); + expect(scheduledAlpha("merge", 0)).toBe(0.005); + expect(scheduledAlpha("merge", 5000)).toBe(0.005); }); }); From a9a4121898373490c339f1b86dd240cfeaebc753 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 26 Jul 2026 07:25:16 -0700 Subject: [PATCH 3/3] docs(scripts): finish the holdout population note cleanly --- scripts/backfill-decision-labels-core.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/backfill-decision-labels-core.ts b/scripts/backfill-decision-labels-core.ts index 4121db63e8..e3a4e82410 100644 --- a/scripts/backfill-decision-labels-core.ts +++ b/scripts/backfill-decision-labels-core.ts @@ -14,10 +14,8 @@ // for calibration — apply must skip them. Adjudicating the 2026-07 backfill proved they are NOT valid // close counterfactuals: a hold that survived means some non-confidence criterion (split vs consensus, // author tier) blocked the close, so "would close have been right" samples a different population than -// the acted-close guarantee governs. The LIVE ε-holdout (#8831) samples would-close PRs and stays valid. These are the pairs that let the sweep certify a λ̂ BELOW today's static -// floor — pairs at/above the floor alone can only re-bless the status quo. Holds with any non-AI blocker -// (CI red, conflicts, policy) are EXCLUDED: "would closing have been right" is unanswerable when the -// hold was about something other than the finding. +// the acted-close guarantee governs. The LIVE ε-holdout (#8831) samples would-close PRs and stays valid. +// Holds with any non-AI blocker (CI red, conflicts, policy) are excluded from staging entirely. // // Confidence reconstruction mirrors the live writer (processors.ts finalize site): the FIRST finding whose // code is in AI_JUDGMENT_BLOCKER_CODES supplies the confidence (`gate.blockers.find(...)`), and findings_json