feat(scripts): backfill the risk-control calibration set from decision history

scripts/backfill-decision-labels-core.ts

@@ -0,0 +1,150 @@
+// Backfill the risk-control calibration set from decision history (#8828 epic follow-through) — PURE core.
+//
+// The risk-control guarantee (#8835) needs adjudicated (confidence, correct?) pairs, and until 2026-07-26
+// decision records did not persist confidence — but ai_review_cache has retained per-finding confidence
+// since 2026-06-28, and review_audit holds every acted gate decision. This core reconstructs the calibration
+// candidates from those two histories so the label floor (199 clean close labels) can be cleared from
+// EXISTING evidence instead of waiting weeks for live accrual.
+//
+// Population rules, both load-bearing:
+//   • ACTED CLOSES (stratum close_arm): the decision the guarantee governs. A realized outcome of `merged`
+//     on a target the gate CLOSED means the PR was later reopened and merged — the definitive incorrect-close
+//     class — so those candidates carry `definitiveAdjudication: "incorrect"` (no judgment needed).
+//   • HOLDS whose ONLY blockers were AI-judgment findings (stratum holdout_close): staged for ANALYSIS, not
+//     for calibration — apply must skip them. Adjudicating the 2026-07 backfill proved they are NOT valid
+//     close counterfactuals: a hold that survived means some non-confidence criterion (split vs consensus,
+//     author tier) blocked the close, so "would close have been right" samples a different population than
+//     the acted-close guarantee governs. The LIVE ε-holdout (#8831) samples would-close PRs and stays valid.
+//     Holds with any non-AI blocker (CI red, conflicts, policy) are excluded from staging entirely.
+//
+// Confidence reconstruction mirrors the live writer (processors.ts finalize site): the FIRST finding whose
+// code is in AI_JUDGMENT_BLOCKER_CODES supplies the confidence (`gate.blockers.find(...)`), and findings_json
+// preserves the derivation order the blocker list was built from. Candidates without a confidence-bearing
+// AI-judgment finding are skipped (rule-only decisions cannot join a confidence-thresholded guarantee).
+//
+// PURE: decides WHAT to stage from rows the caller supplies. The thin IO wrapper owns stdin/stdout.
+import { AI_JUDGMENT_BLOCKER_CODES } from "../src/rules/advisory";
+
+/** One gate decision joined to its review's cached findings, as extracted from review_audit + ai_review_cache. */
+export type CandidateRow = {
+  targetId: string; // "owner/repo#123"
+  project: string; // "owner/repo"
+  pullNumber: number;
+  decision: "close" | "hold";
+  headSha: string | null;
+  decidedAt: string; // review_audit.created_at of the gate_decision row
+  findingsJson: string; // ai_review_cache.findings_json for the target
+  /** gate_outcomes.blocker_codes_json for holds (the codes that held it); null/absent for closes. */
+  blockerCodesJson?: string | null | undefined;
+  /** Latest realized pr_outcome for the target, when one exists. */
+  realizedOutcome?: "merged" | "closed" | null | undefined;
+};
+
+export type StagedCalibrationTarget = {
+  targetId: string;
+  project: string;
+  pullNumber: number;
+  headSha: string;
+  stratum: "close_arm" | "holdout_close";
+  verdict: "close";
+  outcome: "merged" | "closed" | null;
+  aiConfidence: number;
+  reasonCode: string; // the shaping finding's code — the record's clause
+  findingTitle: string; // for the adjudication worklist, never persisted to the record
+  decidedAt: string;
+  /** Set when the realized outcome already decides the label (close then merged = reopened+merged). */
+  definitiveAdjudication: "incorrect" | null;
+};
+
+export type BackfillLabelPlan = {
+  staged: StagedCalibrationTarget[];
+  skipped: {
+    noShapingFinding: number;
+    unparseableFindings: number;
+    mixedBlockerHold: number;
+    missingHeadSha: number;
+    duplicateTarget: number;
+  };
+};
+
+type Finding = { code?: unknown; confidence?: unknown; title?: unknown };
+
+/** The live selection contract: first AI-judgment finding, its confidence required numeric in [0, 1]. */
+function shapingFinding(findingsJson: string): { code: string; confidence: number; title: string } | "unparseable" | null {
+  let findings: unknown;
+  try {
+    findings = JSON.parse(findingsJson);
+  } catch {
+    return "unparseable";
+  }
+  if (!Array.isArray(findings)) return "unparseable";
+  for (const raw of findings as Finding[]) {
+    if (typeof raw?.code !== "string" || !AI_JUDGMENT_BLOCKER_CODES.has(raw.code)) continue;
+    if (typeof raw.confidence !== "number" || !Number.isFinite(raw.confidence) || raw.confidence < 0 || raw.confidence > 1) continue;
+    return { code: raw.code, confidence: raw.confidence, title: typeof raw.title === "string" ? raw.title : "" };
+  }
+  return null;
+}
+
+/** True when every code that held the PR is an AI-judgment code — the pure would-close population. */
+function holdWasAiJudgmentOnly(blockerCodesJson: string | null | undefined): boolean {
+  if (typeof blockerCodesJson !== "string" || blockerCodesJson.trim() === "") return false;
+  try {
+    const codes = JSON.parse(blockerCodesJson);
+    return Array.isArray(codes) && codes.length > 0 && codes.every((code) => typeof code === "string" && AI_JUDGMENT_BLOCKER_CODES.has(code));
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Stage calibration targets from candidate rows. One target contributes at most ONE pair (the UNIQUE
+ * target_id contract of decision_audit_labels): when a target appears as both an acted close and an earlier
+ * hold, the ACTED decision wins — it is the one the guarantee is about.
+ */
+export function planDecisionLabelBackfill(rows: CandidateRow[]): BackfillLabelPlan {
+  const skipped = { noShapingFinding: 0, unparseableFindings: 0, mixedBlockerHold: 0, missingHeadSha: 0, duplicateTarget: 0 };
+  const staged: StagedCalibrationTarget[] = [];
+  const seen = new Set<string>();
+  const ordered = [...rows].sort((a, b) => (a.decision === b.decision ? 0 : a.decision === "close" ? -1 : 1));
+  for (const row of ordered) {
+    if (seen.has(row.targetId)) {
+      skipped.duplicateTarget += 1;
+      continue;
+    }
+    if (row.decision === "hold" && !holdWasAiJudgmentOnly(row.blockerCodesJson)) {
+      skipped.mixedBlockerHold += 1;
+      continue;
+    }
+    const finding = shapingFinding(row.findingsJson);
+    if (finding === "unparseable") {
+      skipped.unparseableFindings += 1;
+      continue;
+    }
+    if (finding === null) {
+      skipped.noShapingFinding += 1;
+      continue;
+    }
+    if (typeof row.headSha !== "string" || row.headSha.trim() === "") {
+      skipped.missingHeadSha += 1;
+      continue;
+    }
+    seen.add(row.targetId);
+    const outcome = row.decision === "close" ? (row.realizedOutcome ?? null) : null;
+    staged.push({
+      targetId: row.targetId,
+      project: row.project,
+      pullNumber: row.pullNumber,
+      headSha: row.headSha,
+      stratum: row.decision === "close" ? "close_arm" : "holdout_close",
+      verdict: "close",
+      outcome,
+      aiConfidence: finding.confidence,
+      reasonCode: finding.code,
+      findingTitle: finding.title,
+      decidedAt: row.decidedAt,
+      definitiveAdjudication: row.decision === "close" && outcome === "merged" ? "incorrect" : null,
+    });
+  }
+  return { staged, skipped };
+}

scripts/backfill-decision-labels.ts

@@ -0,0 +1,132 @@
+#!/usr/bin/env node
+// Backfill the risk-control calibration set (#8828 follow-through) — thin IO around
+// backfill-decision-labels-core.ts. Reads candidate rows as JSON on stdin, emits a staging bundle on
+// stdout. Deliberately infrastructure-free: the operator owns extraction and application.
+//
+// EXTRACT (run against the instance's Postgres; produces the stdin payload):
+//   SELECT json_agg(t) FROM (
+//     SELECT ra.target_id      AS "targetId",
+//            split_part(ra.target_id, '#', 1) AS "project",
+//            c.pull_number     AS "pullNumber",
+//            ra.decision       AS "decision",
+//            ra.head_sha       AS "headSha",
+//            ra.created_at     AS "decidedAt",
+//            c.findings_json   AS "findingsJson",
+//            go.blocker_codes_json AS "blockerCodesJson",
+//            (SELECT ra2.decision FROM review_audit ra2
+//              WHERE ra2.event_type = 'pr_outcome' AND ra2.target_id = ra.target_id
+//              ORDER BY ra2.created_at DESC LIMIT 1) AS "realizedOutcome"
+//              -- the LAST pr_outcome is the definitive story: merged after a close = reopened + merged
+//       FROM review_audit ra
+//       JOIN LATERAL (
+//              SELECT * FROM ai_review_cache c
+//               WHERE c.repo_full_name || '#' || c.pull_number = ra.target_id
+//                 AND c.findings_json LIKE '%confidence%'
+//               ORDER BY (c.head_sha = ra.head_sha) DESC, c.created_at DESC LIMIT 1
+//            ) c ON true
+//       LEFT JOIN gate_outcomes go
+//              ON go.repo_full_name || '#' || go.pull_number = ra.target_id AND go.head_sha = ra.head_sha
+//      WHERE ra.event_type = 'gate_decision' AND ra.decision IN ('close', 'hold')
+//        AND ra.created_at >= '2026-06-28'
+//   ) t;
+//
+// STAGE:
+//   node --experimental-strip-types scripts/backfill-decision-labels.ts < candidates.json > bundle.json
+//
+// APPLY (after adjudication sign-off; both idempotent via ON CONFLICT DO NOTHING). CLOSE_ARM LABELS ONLY —
+// holdout_close rows are analysis output, not calibration rows (see the core header's population note):
+//   \set records `jq -c .records bundle.json`
+//   INSERT INTO decision_records SELECT * FROM jsonb_populate_recordset(NULL::decision_records, :'records'::jsonb)
+//   ON CONFLICT (id) DO NOTHING;
+//   -- likewise .labels -> decision_audit_labels (staged status='pending'; adjudications are a later UPDATE)
+//
+// Backfilled records carry configDigest "backfill:unavailable" — the historical resolved config is
+// unknowable and a fabricated sha256 would corrupt the commitment semantics; the sentinel is also the
+// record's provenance marker. They are NOT appended to the decision ledger: the ledger attests decisions
+// as they were finalized live, and reconstructed history has no place in that chain.
+import { planDecisionLabelBackfill, type CandidateRow } from "./backfill-decision-labels-core.js";
+import { buildDecisionRecord, canonicalJson } from "../src/review/decision-record.js";
+import { DECISION_AUDIT_RUBRIC_VERSION } from "../src/review/decision-audit.js";
+
+async function readStdin(): Promise<string> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of process.stdin) chunks.push(chunk as Buffer);
+  return Buffer.concat(chunks).toString("utf8");
+}
+
+export async function buildBundle(rows: CandidateRow[], stagedAt: string): Promise<Record<string, unknown>> {
+  const plan = planDecisionLabelBackfill(rows);
+  const records = [];
+  const labels = [];
+  const worklist = [];
+  for (const target of plan.staged) {
+    const { record, recordDigest } = await buildDecisionRecord({
+      repoFullName: target.project,
+      pullNumber: target.pullNumber,
+      headSha: target.headSha,
+      action: target.stratum === "close_arm" ? "close" : "hold",
+      reasonCode: target.reasonCode,
+      configDigest: "backfill:unavailable",
+      modelId: null,
+      promptDigest: null,
+      aiConfidence: target.aiConfidence,
+      decidedAt: target.decidedAt,
+    });
+    records.push({
+      id: `record:${target.targetId}@${target.headSha}`.slice(0, 250),
+      repo_full_name: target.project.slice(0, 200),
+      pull_number: target.pullNumber,
+      head_sha: target.headSha,
+      action: record.action,
+      reason_code: record.reasonCode.slice(0, 200),
+      record_digest: recordDigest,
+      record_json: canonicalJson(record),
+      created_at: target.decidedAt,
+    });
+    labels.push({
+      id: `audit:${target.targetId}`.slice(0, 190),
+      project: target.project.slice(0, 200),
+      target_id: target.targetId,
+      verdict: target.verdict,
+      outcome: target.outcome,
+      stratum: target.stratum,
+      rubric_version: DECISION_AUDIT_RUBRIC_VERSION,
+      sampled_at: stagedAt,
+      status: "pending",
+      adjudication: null,
+      reason_category: null,
+      adjudicated_at: null,
+    });
+    worklist.push({
+      targetId: target.targetId,
+      stratum: target.stratum,
+      confidence: target.aiConfidence,
+      reasonCode: target.reasonCode,
+      findingTitle: target.findingTitle.slice(0, 300),
+      decidedAt: target.decidedAt,
+      outcome: target.outcome,
+      definitiveAdjudication: target.definitiveAdjudication,
+    });
+  }
+  return { stagedAt, skipped: plan.skipped, records, labels, worklist };
+}
+
+const invokedDirectly = process.argv[1]?.endsWith("backfill-decision-labels.ts") === true;
+if (invokedDirectly) {
+  readStdin()
+    .then(async (raw) => {
+      const rows = JSON.parse(raw) as CandidateRow[];
+      const bundle = await buildBundle(rows, new Date().toISOString());
+      process.stdout.write(`${JSON.stringify(bundle, null, 1)}\n`);
+      const skipped = bundle.skipped as Record<string, number>;
+      console.error(
+        `backfill-decision-labels: ${rows.length} candidate(s) -> ${(bundle.records as unknown[]).length} staged; skipped: ${Object.entries(skipped)
+          .map(([key, count]) => `${key}=${count}`)
+          .join(" ")}`,
+      );
+    })
+    .catch((error: unknown) => {
+      console.error(`backfill-decision-labels: ${error instanceof Error ? error.message : String(error)}`);
+      process.exitCode = 1;
+    });
+}

src/review/risk-control-wire.ts

@@ -31,15 +31,42 @@ export function parseBudget(raw: string | undefined, fallback: number, max: numb
 }
 
 /** Per-arm error budgets (#8835's Neyman–Pearson requirement) and the calibration confidence level.
- *  Defaults: close α=0.015 (~199-label floor), merge α=0.005 (~598 — stricter than close by 3x; the earlier
- *  0.002 draft needed ~1,497 labels, a year of adjudication for the last 3x of strictness, which contradicts
- *  the minimal-human-involvement objective this instrument serves). */
-export function riskControlArms(env: Env): Array<{ arm: "close" | "merge"; verdict: "close" | "merge"; alpha: number }> {
+ *  An explicit env α pins the arm; when unset, the CLOSE arm follows the pre-registered α(n) schedule below
+ *  and the merge arm stays at 0.005 (~598-label floor — stricter than close by design; the earlier 0.002
+ *  draft needed ~1,497 labels, a year of adjudication for the last 3x of strictness, which contradicts the
+ *  minimal-human-involvement objective this instrument serves). */
+export function riskControlArms(env: Env): Array<{ arm: "close" | "merge"; verdict: "close" | "merge"; alpha: number | null }> {
   return [
-    { arm: "close", verdict: "close", alpha: parseBudget(env.LOOPOVER_RISK_CONTROL_CLOSE_ALPHA, 0.015, 0.05) },
-    { arm: "merge", verdict: "merge", alpha: parseBudget(env.LOOPOVER_RISK_CONTROL_MERGE_ALPHA, 0.005, 0.05) },
+    { arm: "close", verdict: "close", alpha: parseBudgetOrNull(env.LOOPOVER_RISK_CONTROL_CLOSE_ALPHA, 0.05) },
+    { arm: "merge", verdict: "merge", alpha: parseBudgetOrNull(env.LOOPOVER_RISK_CONTROL_MERGE_ALPHA, 0.05) },
   ];
 }
+
+/** Like parseBudget but with no fallback: null when absent/garbage/outside (0, max] — "use the schedule". */
+export function parseBudgetOrNull(raw: string | undefined, max: number): number | null {
+  const value = Number((raw ?? "").trim());
+  if (!Number.isFinite(value) || value <= 0 || value > max) return null;
+  return value;
+}
+
+/**
+ * The PRE-REGISTERED α(n) tightening schedule: the strongest error budget the arm's usable label count can
+ * power, chosen as a deterministic function of SAMPLE SIZE alone. Because n is information size, not the
+ * test statistic, walking this schedule as labels accrue is not outcome-snooping and needs no multiplicity
+ * correction — each daily recalibration still runs exactly ONE level-δ test. The alternative (testing an
+ * α-grid each day and publishing the strongest pass) would need a Bonferroni δ-split that raises every
+ * floor by ~35% and, at today's label volume, certifies nothing at all.
+ *
+ * Tiers (close): α=0.05 under 350 usable pairs (guarantee arrives at the honest floor of ~59 clean at-λ
+ * labels), 0.025 from 350, 0.015 from 700 — the homepage claim self-tightens 95% → 97.5% → 98.5% with zero
+ * human steps. Merge: fixed 0.005 until its own volume justifies a schedule (tracked on #8828's epic).
+ */
+export function scheduledAlpha(arm: "close" | "merge", usablePairs: number): number {
+  if (arm === "merge") return 0.005;
+  if (usablePairs >= 700) return 0.015;
+  if (usablePairs >= 350) return 0.025;
+  return 0.05;
+}
 export function riskControlDelta(env: Env): number {
   return parseBudget(env.LOOPOVER_RISK_CONTROL_DELTA, 0.05, 0.2);
 }
@@ -86,9 +113,10 @@ export async function loadCalibrationPairs(env: Env, verdict: "close" | "merge",
 
 /** One arm's recalibration (global when `project` is null, else that repo's own labels): calibrate →
  *  publish or retract. Best-effort per arm. */
-async function recalibrateArm(env: Env, arm: string, verdict: "close" | "merge", alpha: number, project: string | null): Promise<CalibrationResult> {
+async function recalibrateArm(env: Env, arm: "close" | "merge", verdict: "close" | "merge", envAlpha: number | null, project: string | null): Promise<CalibrationResult> {
   const delta = riskControlDelta(env);
   const pairs = await loadCalibrationPairs(env, verdict, project);
+  const alpha = envAlpha ?? scheduledAlpha(arm, pairs.length);
   const result = calibrateActThreshold(pairs, alpha, delta);
   const scope = project === null ? arm : `${arm}:${project}`;
   if (result.status === "calibrated") {