diff --git a/scripts/backfill-decision-labels-core.ts b/scripts/backfill-decision-labels-core.ts new file mode 100644 index 0000000000..e3a4e82410 --- /dev/null +++ b/scripts/backfill-decision-labels-core.ts @@ -0,0 +1,150 @@ +// Backfill the risk-control calibration set from decision history (#8828 epic follow-through) — PURE core. +// +// The risk-control guarantee (#8835) needs adjudicated (confidence, correct?) pairs, and until 2026-07-26 +// decision records did not persist confidence — but ai_review_cache has retained per-finding confidence +// since 2026-06-28, and review_audit holds every acted gate decision. This core reconstructs the calibration +// candidates from those two histories so the label floor (199 clean close labels) can be cleared from +// EXISTING evidence instead of waiting weeks for live accrual. +// +// Population rules, both load-bearing: +// • ACTED CLOSES (stratum close_arm): the decision the guarantee governs. A realized outcome of `merged` +// on a target the gate CLOSED means the PR was later reopened and merged — the definitive incorrect-close +// class — so those candidates carry `definitiveAdjudication: "incorrect"` (no judgment needed). +// • HOLDS whose ONLY blockers were AI-judgment findings (stratum holdout_close): staged for ANALYSIS, not +// for calibration — apply must skip them. Adjudicating the 2026-07 backfill proved they are NOT valid +// close counterfactuals: a hold that survived means some non-confidence criterion (split vs consensus, +// author tier) blocked the close, so "would close have been right" samples a different population than +// the acted-close guarantee governs. The LIVE ε-holdout (#8831) samples would-close PRs and stays valid. +// Holds with any non-AI blocker (CI red, conflicts, policy) are excluded from staging entirely. +// +// Confidence reconstruction mirrors the live writer (processors.ts finalize site): the FIRST finding whose +// code is in AI_JUDGMENT_BLOCKER_CODES supplies the confidence (`gate.blockers.find(...)`), and findings_json +// preserves the derivation order the blocker list was built from. Candidates without a confidence-bearing +// AI-judgment finding are skipped (rule-only decisions cannot join a confidence-thresholded guarantee). +// +// PURE: decides WHAT to stage from rows the caller supplies. The thin IO wrapper owns stdin/stdout. +import { AI_JUDGMENT_BLOCKER_CODES } from "../src/rules/advisory"; + +/** One gate decision joined to its review's cached findings, as extracted from review_audit + ai_review_cache. */ +export type CandidateRow = { + targetId: string; // "owner/repo#123" + project: string; // "owner/repo" + pullNumber: number; + decision: "close" | "hold"; + headSha: string | null; + decidedAt: string; // review_audit.created_at of the gate_decision row + findingsJson: string; // ai_review_cache.findings_json for the target + /** gate_outcomes.blocker_codes_json for holds (the codes that held it); null/absent for closes. */ + blockerCodesJson?: string | null | undefined; + /** Latest realized pr_outcome for the target, when one exists. */ + realizedOutcome?: "merged" | "closed" | null | undefined; +}; + +export type StagedCalibrationTarget = { + targetId: string; + project: string; + pullNumber: number; + headSha: string; + stratum: "close_arm" | "holdout_close"; + verdict: "close"; + outcome: "merged" | "closed" | null; + aiConfidence: number; + reasonCode: string; // the shaping finding's code — the record's clause + findingTitle: string; // for the adjudication worklist, never persisted to the record + decidedAt: string; + /** Set when the realized outcome already decides the label (close then merged = reopened+merged). */ + definitiveAdjudication: "incorrect" | null; +}; + +export type BackfillLabelPlan = { + staged: StagedCalibrationTarget[]; + skipped: { + noShapingFinding: number; + unparseableFindings: number; + mixedBlockerHold: number; + missingHeadSha: number; + duplicateTarget: number; + }; +}; + +type Finding = { code?: unknown; confidence?: unknown; title?: unknown }; + +/** The live selection contract: first AI-judgment finding, its confidence required numeric in [0, 1]. */ +function shapingFinding(findingsJson: string): { code: string; confidence: number; title: string } | "unparseable" | null { + let findings: unknown; + try { + findings = JSON.parse(findingsJson); + } catch { + return "unparseable"; + } + if (!Array.isArray(findings)) return "unparseable"; + for (const raw of findings as Finding[]) { + if (typeof raw?.code !== "string" || !AI_JUDGMENT_BLOCKER_CODES.has(raw.code)) continue; + if (typeof raw.confidence !== "number" || !Number.isFinite(raw.confidence) || raw.confidence < 0 || raw.confidence > 1) continue; + return { code: raw.code, confidence: raw.confidence, title: typeof raw.title === "string" ? raw.title : "" }; + } + return null; +} + +/** True when every code that held the PR is an AI-judgment code — the pure would-close population. */ +function holdWasAiJudgmentOnly(blockerCodesJson: string | null | undefined): boolean { + if (typeof blockerCodesJson !== "string" || blockerCodesJson.trim() === "") return false; + try { + const codes = JSON.parse(blockerCodesJson); + return Array.isArray(codes) && codes.length > 0 && codes.every((code) => typeof code === "string" && AI_JUDGMENT_BLOCKER_CODES.has(code)); + } catch { + return false; + } +} + +/** + * Stage calibration targets from candidate rows. One target contributes at most ONE pair (the UNIQUE + * target_id contract of decision_audit_labels): when a target appears as both an acted close and an earlier + * hold, the ACTED decision wins — it is the one the guarantee is about. + */ +export function planDecisionLabelBackfill(rows: CandidateRow[]): BackfillLabelPlan { + const skipped = { noShapingFinding: 0, unparseableFindings: 0, mixedBlockerHold: 0, missingHeadSha: 0, duplicateTarget: 0 }; + const staged: StagedCalibrationTarget[] = []; + const seen = new Set(); + const ordered = [...rows].sort((a, b) => (a.decision === b.decision ? 0 : a.decision === "close" ? -1 : 1)); + for (const row of ordered) { + if (seen.has(row.targetId)) { + skipped.duplicateTarget += 1; + continue; + } + if (row.decision === "hold" && !holdWasAiJudgmentOnly(row.blockerCodesJson)) { + skipped.mixedBlockerHold += 1; + continue; + } + const finding = shapingFinding(row.findingsJson); + if (finding === "unparseable") { + skipped.unparseableFindings += 1; + continue; + } + if (finding === null) { + skipped.noShapingFinding += 1; + continue; + } + if (typeof row.headSha !== "string" || row.headSha.trim() === "") { + skipped.missingHeadSha += 1; + continue; + } + seen.add(row.targetId); + const outcome = row.decision === "close" ? (row.realizedOutcome ?? null) : null; + staged.push({ + targetId: row.targetId, + project: row.project, + pullNumber: row.pullNumber, + headSha: row.headSha, + stratum: row.decision === "close" ? "close_arm" : "holdout_close", + verdict: "close", + outcome, + aiConfidence: finding.confidence, + reasonCode: finding.code, + findingTitle: finding.title, + decidedAt: row.decidedAt, + definitiveAdjudication: row.decision === "close" && outcome === "merged" ? "incorrect" : null, + }); + } + return { staged, skipped }; +} diff --git a/scripts/backfill-decision-labels.ts b/scripts/backfill-decision-labels.ts new file mode 100644 index 0000000000..b7a4d335e3 --- /dev/null +++ b/scripts/backfill-decision-labels.ts @@ -0,0 +1,132 @@ +#!/usr/bin/env node +// Backfill the risk-control calibration set (#8828 follow-through) — thin IO around +// backfill-decision-labels-core.ts. Reads candidate rows as JSON on stdin, emits a staging bundle on +// stdout. Deliberately infrastructure-free: the operator owns extraction and application. +// +// EXTRACT (run against the instance's Postgres; produces the stdin payload): +// SELECT json_agg(t) FROM ( +// SELECT ra.target_id AS "targetId", +// split_part(ra.target_id, '#', 1) AS "project", +// c.pull_number AS "pullNumber", +// ra.decision AS "decision", +// ra.head_sha AS "headSha", +// ra.created_at AS "decidedAt", +// c.findings_json AS "findingsJson", +// go.blocker_codes_json AS "blockerCodesJson", +// (SELECT ra2.decision FROM review_audit ra2 +// WHERE ra2.event_type = 'pr_outcome' AND ra2.target_id = ra.target_id +// ORDER BY ra2.created_at DESC LIMIT 1) AS "realizedOutcome" +// -- the LAST pr_outcome is the definitive story: merged after a close = reopened + merged +// FROM review_audit ra +// JOIN LATERAL ( +// SELECT * FROM ai_review_cache c +// WHERE c.repo_full_name || '#' || c.pull_number = ra.target_id +// AND c.findings_json LIKE '%confidence%' +// ORDER BY (c.head_sha = ra.head_sha) DESC, c.created_at DESC LIMIT 1 +// ) c ON true +// LEFT JOIN gate_outcomes go +// ON go.repo_full_name || '#' || go.pull_number = ra.target_id AND go.head_sha = ra.head_sha +// WHERE ra.event_type = 'gate_decision' AND ra.decision IN ('close', 'hold') +// AND ra.created_at >= '2026-06-28' +// ) t; +// +// STAGE: +// node --experimental-strip-types scripts/backfill-decision-labels.ts < candidates.json > bundle.json +// +// APPLY (after adjudication sign-off; both idempotent via ON CONFLICT DO NOTHING). CLOSE_ARM LABELS ONLY — +// holdout_close rows are analysis output, not calibration rows (see the core header's population note): +// \set records `jq -c .records bundle.json` +// INSERT INTO decision_records SELECT * FROM jsonb_populate_recordset(NULL::decision_records, :'records'::jsonb) +// ON CONFLICT (id) DO NOTHING; +// -- likewise .labels -> decision_audit_labels (staged status='pending'; adjudications are a later UPDATE) +// +// Backfilled records carry configDigest "backfill:unavailable" — the historical resolved config is +// unknowable and a fabricated sha256 would corrupt the commitment semantics; the sentinel is also the +// record's provenance marker. They are NOT appended to the decision ledger: the ledger attests decisions +// as they were finalized live, and reconstructed history has no place in that chain. +import { planDecisionLabelBackfill, type CandidateRow } from "./backfill-decision-labels-core.js"; +import { buildDecisionRecord, canonicalJson } from "../src/review/decision-record.js"; +import { DECISION_AUDIT_RUBRIC_VERSION } from "../src/review/decision-audit.js"; + +async function readStdin(): Promise { + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) chunks.push(chunk as Buffer); + return Buffer.concat(chunks).toString("utf8"); +} + +export async function buildBundle(rows: CandidateRow[], stagedAt: string): Promise> { + const plan = planDecisionLabelBackfill(rows); + const records = []; + const labels = []; + const worklist = []; + for (const target of plan.staged) { + const { record, recordDigest } = await buildDecisionRecord({ + repoFullName: target.project, + pullNumber: target.pullNumber, + headSha: target.headSha, + action: target.stratum === "close_arm" ? "close" : "hold", + reasonCode: target.reasonCode, + configDigest: "backfill:unavailable", + modelId: null, + promptDigest: null, + aiConfidence: target.aiConfidence, + decidedAt: target.decidedAt, + }); + records.push({ + id: `record:${target.targetId}@${target.headSha}`.slice(0, 250), + repo_full_name: target.project.slice(0, 200), + pull_number: target.pullNumber, + head_sha: target.headSha, + action: record.action, + reason_code: record.reasonCode.slice(0, 200), + record_digest: recordDigest, + record_json: canonicalJson(record), + created_at: target.decidedAt, + }); + labels.push({ + id: `audit:${target.targetId}`.slice(0, 190), + project: target.project.slice(0, 200), + target_id: target.targetId, + verdict: target.verdict, + outcome: target.outcome, + stratum: target.stratum, + rubric_version: DECISION_AUDIT_RUBRIC_VERSION, + sampled_at: stagedAt, + status: "pending", + adjudication: null, + reason_category: null, + adjudicated_at: null, + }); + worklist.push({ + targetId: target.targetId, + stratum: target.stratum, + confidence: target.aiConfidence, + reasonCode: target.reasonCode, + findingTitle: target.findingTitle.slice(0, 300), + decidedAt: target.decidedAt, + outcome: target.outcome, + definitiveAdjudication: target.definitiveAdjudication, + }); + } + return { stagedAt, skipped: plan.skipped, records, labels, worklist }; +} + +const invokedDirectly = process.argv[1]?.endsWith("backfill-decision-labels.ts") === true; +if (invokedDirectly) { + readStdin() + .then(async (raw) => { + const rows = JSON.parse(raw) as CandidateRow[]; + const bundle = await buildBundle(rows, new Date().toISOString()); + process.stdout.write(`${JSON.stringify(bundle, null, 1)}\n`); + const skipped = bundle.skipped as Record; + console.error( + `backfill-decision-labels: ${rows.length} candidate(s) -> ${(bundle.records as unknown[]).length} staged; skipped: ${Object.entries(skipped) + .map(([key, count]) => `${key}=${count}`) + .join(" ")}`, + ); + }) + .catch((error: unknown) => { + console.error(`backfill-decision-labels: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + }); +} diff --git a/src/review/risk-control-wire.ts b/src/review/risk-control-wire.ts index 4af1aaccea..8243397687 100644 --- a/src/review/risk-control-wire.ts +++ b/src/review/risk-control-wire.ts @@ -31,15 +31,42 @@ export function parseBudget(raw: string | undefined, fallback: number, max: numb } /** Per-arm error budgets (#8835's Neyman–Pearson requirement) and the calibration confidence level. - * Defaults: close α=0.015 (~199-label floor), merge α=0.005 (~598 — stricter than close by 3x; the earlier - * 0.002 draft needed ~1,497 labels, a year of adjudication for the last 3x of strictness, which contradicts - * the minimal-human-involvement objective this instrument serves). */ -export function riskControlArms(env: Env): Array<{ arm: "close" | "merge"; verdict: "close" | "merge"; alpha: number }> { + * An explicit env α pins the arm; when unset, the CLOSE arm follows the pre-registered α(n) schedule below + * and the merge arm stays at 0.005 (~598-label floor — stricter than close by design; the earlier 0.002 + * draft needed ~1,497 labels, a year of adjudication for the last 3x of strictness, which contradicts the + * minimal-human-involvement objective this instrument serves). */ +export function riskControlArms(env: Env): Array<{ arm: "close" | "merge"; verdict: "close" | "merge"; alpha: number | null }> { return [ - { arm: "close", verdict: "close", alpha: parseBudget(env.LOOPOVER_RISK_CONTROL_CLOSE_ALPHA, 0.015, 0.05) }, - { arm: "merge", verdict: "merge", alpha: parseBudget(env.LOOPOVER_RISK_CONTROL_MERGE_ALPHA, 0.005, 0.05) }, + { arm: "close", verdict: "close", alpha: parseBudgetOrNull(env.LOOPOVER_RISK_CONTROL_CLOSE_ALPHA, 0.05) }, + { arm: "merge", verdict: "merge", alpha: parseBudgetOrNull(env.LOOPOVER_RISK_CONTROL_MERGE_ALPHA, 0.05) }, ]; } + +/** Like parseBudget but with no fallback: null when absent/garbage/outside (0, max] — "use the schedule". */ +export function parseBudgetOrNull(raw: string | undefined, max: number): number | null { + const value = Number((raw ?? "").trim()); + if (!Number.isFinite(value) || value <= 0 || value > max) return null; + return value; +} + +/** + * The PRE-REGISTERED α(n) tightening schedule: the strongest error budget the arm's usable label count can + * power, chosen as a deterministic function of SAMPLE SIZE alone. Because n is information size, not the + * test statistic, walking this schedule as labels accrue is not outcome-snooping and needs no multiplicity + * correction — each daily recalibration still runs exactly ONE level-δ test. The alternative (testing an + * α-grid each day and publishing the strongest pass) would need a Bonferroni δ-split that raises every + * floor by ~35% and, at today's label volume, certifies nothing at all. + * + * Tiers (close): α=0.05 under 350 usable pairs (guarantee arrives at the honest floor of ~59 clean at-λ + * labels), 0.025 from 350, 0.015 from 700 — the homepage claim self-tightens 95% → 97.5% → 98.5% with zero + * human steps. Merge: fixed 0.005 until its own volume justifies a schedule (tracked on #8828's epic). + */ +export function scheduledAlpha(arm: "close" | "merge", usablePairs: number): number { + if (arm === "merge") return 0.005; + if (usablePairs >= 700) return 0.015; + if (usablePairs >= 350) return 0.025; + return 0.05; +} export function riskControlDelta(env: Env): number { return parseBudget(env.LOOPOVER_RISK_CONTROL_DELTA, 0.05, 0.2); } @@ -86,9 +113,10 @@ export async function loadCalibrationPairs(env: Env, verdict: "close" | "merge", /** One arm's recalibration (global when `project` is null, else that repo's own labels): calibrate → * publish or retract. Best-effort per arm. */ -async function recalibrateArm(env: Env, arm: string, verdict: "close" | "merge", alpha: number, project: string | null): Promise { +async function recalibrateArm(env: Env, arm: "close" | "merge", verdict: "close" | "merge", envAlpha: number | null, project: string | null): Promise { const delta = riskControlDelta(env); const pairs = await loadCalibrationPairs(env, verdict, project); + const alpha = envAlpha ?? scheduledAlpha(arm, pairs.length); const result = calibrateActThreshold(pairs, alpha, delta); const scope = project === null ? arm : `${arm}:${project}`; if (result.status === "calibrated") { diff --git a/test/unit/backfill-decision-labels-core.test.ts b/test/unit/backfill-decision-labels-core.test.ts new file mode 100644 index 0000000000..8aa566dea7 --- /dev/null +++ b/test/unit/backfill-decision-labels-core.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; +import { planDecisionLabelBackfill, type CandidateRow } from "../../scripts/backfill-decision-labels-core"; +import { buildBundle } from "../../scripts/backfill-decision-labels"; +import { canonicalJson, contentDigest } from "../../src/review/decision-record"; + +// The backfill's contracts: acted closes stage as close_arm (merged outcome = definitive incorrect), +// AI-judgment-only holds stage as holdout_close, everything else is skipped with its reason counted, +// and one target contributes at most one pair with the acted decision winning. + +function candidate(overrides: Partial): CandidateRow { + return { + targetId: "o/r#1", + project: "o/r", + pullNumber: 1, + decision: "close", + headSha: "sha1", + decidedAt: "2026-07-01T00:00:00.000Z", + findingsJson: JSON.stringify([{ code: "ai_consensus_defect", confidence: 0.9, title: "defect" }]), + realizedOutcome: "closed", + ...overrides, + }; +} + +describe("planDecisionLabelBackfill", () => { + it("stages an acted close as close_arm carrying the shaping finding's confidence", () => { + const plan = planDecisionLabelBackfill([candidate({})]); + expect(plan.staged).toHaveLength(1); + expect(plan.staged[0]).toMatchObject({ + stratum: "close_arm", + verdict: "close", + aiConfidence: 0.9, + reasonCode: "ai_consensus_defect", + outcome: "closed", + definitiveAdjudication: null, + }); + }); + + it("a closed-then-MERGED target is the definitive incorrect-close class", () => { + const plan = planDecisionLabelBackfill([candidate({ realizedOutcome: "merged" })]); + expect(plan.staged[0]!.definitiveAdjudication).toBe("incorrect"); + expect(plan.staged[0]!.outcome).toBe("merged"); + }); + + it("stages a hold as holdout_close ONLY when every blocker code was an AI judgment; outcome stays null", () => { + const pure = candidate({ decision: "hold", blockerCodesJson: JSON.stringify(["ai_consensus_defect"]) }); + const mixed = candidate({ + targetId: "o/r#2", + pullNumber: 2, + decision: "hold", + blockerCodesJson: JSON.stringify(["ai_consensus_defect", "ci_failing"]), + }); + const absent = candidate({ targetId: "o/r#3", pullNumber: 3, decision: "hold", blockerCodesJson: null }); + const empty = candidate({ targetId: "o/r#4", pullNumber: 4, decision: "hold", blockerCodesJson: "[]" }); + const garbled = candidate({ targetId: "o/r#5", pullNumber: 5, decision: "hold", blockerCodesJson: "{nope" }); + const plan = planDecisionLabelBackfill([pure, mixed, absent, empty, garbled]); + expect(plan.staged).toHaveLength(1); + expect(plan.staged[0]).toMatchObject({ stratum: "holdout_close", outcome: null, definitiveAdjudication: null }); + expect(plan.skipped.mixedBlockerHold).toBe(4); + }); + + it("selects the FIRST AI-judgment finding (the live gate.blockers.find contract), skipping non-judgment codes", () => { + const plan = planDecisionLabelBackfill([ + candidate({ + findingsJson: JSON.stringify([ + { code: "slop_signal", confidence: 0.99, title: "not a judgment" }, + { code: "ai_review_split", confidence: 0.62, title: "split" }, + { code: "ai_consensus_defect", confidence: 0.97, title: "later" }, + ]), + }), + ]); + expect(plan.staged[0]).toMatchObject({ aiConfidence: 0.62, reasonCode: "ai_review_split" }); + }); + + it("skips candidates with no usable confidence: missing, non-numeric, out of range, or no judgment finding", () => { + const plan = planDecisionLabelBackfill([ + candidate({ findingsJson: JSON.stringify([{ code: "ai_consensus_defect", title: "no confidence" }]) }), + candidate({ targetId: "o/r#2", pullNumber: 2, findingsJson: JSON.stringify([{ code: "ai_consensus_defect", confidence: "high" }]) }), + candidate({ targetId: "o/r#3", pullNumber: 3, findingsJson: JSON.stringify([{ code: "ai_consensus_defect", confidence: 1.7 }]) }), + candidate({ targetId: "o/r#4", pullNumber: 4, findingsJson: JSON.stringify([{ code: "readiness_low" }]) }), + ]); + expect(plan.staged).toHaveLength(0); + expect(plan.skipped.noShapingFinding).toBe(4); + }); + + it("counts unparseable findings and missing head shas separately", () => { + const plan = planDecisionLabelBackfill([ + candidate({ findingsJson: "{broken" }), + candidate({ targetId: "o/r#2", pullNumber: 2, findingsJson: JSON.stringify({ not: "an array" }) }), + candidate({ targetId: "o/r#3", pullNumber: 3, headSha: null }), + candidate({ targetId: "o/r#4", pullNumber: 4, headSha: " " }), + ]); + expect(plan.staged).toHaveLength(0); + expect(plan.skipped.unparseableFindings).toBe(2); + expect(plan.skipped.missingHeadSha).toBe(2); + }); + + it("one pair per target: the ACTED close wins over the same target's hold regardless of input order", () => { + const hold = candidate({ decision: "hold", blockerCodesJson: JSON.stringify(["ai_consensus_defect"]) }); + const close = candidate({ findingsJson: JSON.stringify([{ code: "ai_consensus_defect", confidence: 0.8, title: "t" }]) }); + const plan = planDecisionLabelBackfill([hold, close]); + expect(plan.staged).toHaveLength(1); + expect(plan.staged[0]).toMatchObject({ stratum: "close_arm", aiConfidence: 0.8 }); + expect(plan.skipped.duplicateTarget).toBe(1); + }); +}); + +describe("buildBundle", () => { + it("emits records whose digest matches their canonical JSON, backfill-sentinel config, and pending labels", async () => { + const bundle = await buildBundle([candidate({}), candidate({ targetId: "o/r#7", pullNumber: 7, decision: "hold", blockerCodesJson: '["ai_consensus_defect"]' })], "2026-07-26T12:00:00.000Z"); + const records = bundle.records as Array>; + const labels = bundle.labels as Array>; + expect(records).toHaveLength(2); + for (const row of records) { + const record = JSON.parse(String(row.record_json)) as Record; + expect(await contentDigest(record)).toBe(row.record_digest); + expect(canonicalJson(record)).toBe(row.record_json); + expect(record.configDigest).toBe("backfill:unavailable"); + expect(record.schemaVersion).toBe("2"); + } + expect(records[0]).toMatchObject({ id: "record:o/r#1@sha1", action: "close", created_at: "2026-07-01T00:00:00.000Z" }); + expect(records[1]).toMatchObject({ action: "hold" }); + const parsed = JSON.parse(String(records[0]!.record_json)) as { aiConfidence: number }; + expect(parsed.aiConfidence).toBe(0.9); + expect(labels[0]).toMatchObject({ + id: "audit:o/r#1", + status: "pending", + stratum: "close_arm", + rubric_version: "1", + sampled_at: "2026-07-26T12:00:00.000Z", + adjudication: null, + }); + expect(labels[1]).toMatchObject({ stratum: "holdout_close", outcome: null }); + expect((bundle.worklist as unknown[]).length).toBe(2); + }); +}); diff --git a/test/unit/risk-control-wire.test.ts b/test/unit/risk-control-wire.test.ts index 74a5ef5f63..438e426598 100644 --- a/test/unit/risk-control-wire.test.ts +++ b/test/unit/risk-control-wire.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { isRiskControlEnabled, loadCalibrationPairs, parseBudget, readCalibratedThreshold, resolveAutomaticCloseConfidence, riskControlArms, riskControlFlagKey, runRiskControlRecalibration } from "../../src/review/risk-control-wire"; +import { isRiskControlEnabled, loadCalibrationPairs, parseBudget, parseBudgetOrNull, readCalibratedThreshold, resolveAutomaticCloseConfidence, riskControlArms, riskControlFlagKey, runRiskControlRecalibration, scheduledAlpha } from "../../src/review/risk-control-wire"; import { processJob } from "../../src/queue/processors"; import { createTestEnv } from "../helpers/d1"; @@ -69,7 +69,7 @@ describe("runRiskControlRecalibration", () => { const flag = await env.DB.prepare(`SELECT value FROM system_flags WHERE key = ?`).bind(riskControlFlagKey("close")).first(); expect(flag).toBeFalsy(); // a stale guarantee is a lie — retracted const audit = await env.DB.prepare(`SELECT detail FROM audit_events WHERE event_type = 'risk_control_insufficient' AND target_key = 'riskcontrol:close'`).first<{ detail: string }>(); - expect(audit!.detail).toContain("of 199 needed"); // close-arm alpha 0.015 floor + expect(audit!.detail).toContain("of 59 needed"); // scheduled close alpha 0.05 under 350 pairs → 59-label floor }); it("CALIBRATED: publishes lambda + the certified statement once the close arm clears its floor", async () => { @@ -77,13 +77,14 @@ describe("runRiskControlRecalibration", () => { for (let i = 1; i <= 210; i += 1) await seedLabeledDecision(env, i, "close", "correct", 0.9 + (i % 5) / 100); const summary = await runRiskControlRecalibration(env); expect(summary.close).toBe("calibrated"); - expect(summary.merge).toBe("insufficient_labels"); // merge alpha 0.002 needs 1497 — genuinely separate arms + expect(summary.merge).toBe("insufficient_labels"); // merge alpha 0.005 needs 598 — genuinely separate arms const flag = await env.DB.prepare(`SELECT value FROM system_flags WHERE key = ?`).bind(riskControlFlagKey("close")).first<{ value: string }>(); - const stored = JSON.parse(flag!.value) as { lambda: number; coverageAtLambda: number }; + const stored = JSON.parse(flag!.value) as { lambda: number; coverageAtLambda: number; alpha: number }; expect(stored.lambda).toBe(0.9); expect(stored.coverageAtLambda).toBe(1); + expect(stored.alpha).toBe(0.05); // 210 pairs < 350 → the schedule's first tier const audit = await env.DB.prepare(`SELECT detail FROM audit_events WHERE event_type = 'risk_control_calibrated'`).first<{ detail: string }>(); - expect(audit!.detail).toContain("P(wrong | acted) ≤ 0.015 guaranteed at 100% coverage"); + expect(audit!.detail).toContain("P(wrong | acted) ≤ 0.05 guaranteed at 100% coverage"); }); }); @@ -169,7 +170,25 @@ describe("env-configurable budgets", () => { expect(parseBudget("0.01", 0.015, 0.05)).toBe(0.01); const arms = riskControlArms(createTestEnv({ LOOPOVER_RISK_CONTROL_CLOSE_ALPHA: "0.02" })); expect(arms.find((a) => a.arm === "close")!.alpha).toBe(0.02); - expect(arms.find((a) => a.arm === "merge")!.alpha).toBe(0.005); + expect(arms.find((a) => a.arm === "merge")!.alpha).toBeNull(); // unset env = follow the schedule + }); + + it("parseBudgetOrNull: null on absent/garbage/out-of-range means 'use the schedule'", () => { + expect(parseBudgetOrNull(undefined, 0.05)).toBeNull(); + expect(parseBudgetOrNull("nope", 0.05)).toBeNull(); + expect(parseBudgetOrNull("0.2", 0.05)).toBeNull(); + expect(parseBudgetOrNull("0", 0.05)).toBeNull(); + expect(parseBudgetOrNull("0.03", 0.05)).toBe(0.03); + }); + + it("scheduledAlpha: the pre-registered close-arm tightening tiers; merge stays fixed at 0.005", () => { + expect(scheduledAlpha("close", 0)).toBe(0.05); + expect(scheduledAlpha("close", 349)).toBe(0.05); + expect(scheduledAlpha("close", 350)).toBe(0.025); + expect(scheduledAlpha("close", 699)).toBe(0.025); + expect(scheduledAlpha("close", 700)).toBe(0.015); + expect(scheduledAlpha("merge", 0)).toBe(0.005); + expect(scheduledAlpha("merge", 5000)).toBe(0.005); }); });