feat(orb): persist the AI judgment's confidence and prompt commitment in the decision record

src/queue/processors.ts

@@ -644,6 +644,8 @@ import {
   recordPrOutcome,
   recordReversalSignals,
 } from "../review/outcomes-wire";
+import { AI_JUDGMENT_BLOCKER_CODES } from "../rules/advisory";
+import { REVIEW_PROMPT_VERSION, REVIEW_SYSTEM_PROMPT } from "../services/ai-review";
 import { maybeApplyCloseAuditHoldout } from "../review/close-audit-holdout";
 import { buildDecisionRecord, contentDigest, loadDecisionRecordCollapsible, persistDecisionRecord } from "../review/decision-record";
 import { neutralHoldReasonCode, nativeGateActionFromConclusion, recordNativeGateDecision } from "../review/parity-wire";
@@ -3322,12 +3324,16 @@ async function runAgentMaintenancePlanAndExecute(
     // erase a prior miner-authored prediction for the same head.
     minerAuthored: breakerMinerAuthored,
   });
-  // #8836: the content-addressed decision record — the public commitment that lets a contributor argue
-  // "clause X of config abc123 decided this". Persist-only here (best-effort inside); the same reasonCode
-  // expression as recordNativeGateDecision above so the record and the calibration row can never disagree
-  // about WHY. Model/prompt commitments are wired when the AI-review contribution lands (tracked on #8836's
-  // follow-up note); rule-only decisions carry null there by design.
+  // #8836/#8834: the content-addressed decision record — the public commitment that lets a contributor
+  // argue "clause X of config abc123 decided this". Persist-only here (best-effort inside); the same
+  // reasonCode expression as recordNativeGateDecision above so the record and the calibration row can never
+  // disagree about WHY. When an AI JUDGMENT shaped this decision (an AI_JUDGMENT_BLOCKER_CODES finding is
+  // present), the record carries the prompt-template commitment and the finding's calibrated confidence —
+  // the confidence is what joins every decision to the risk-control calibration set (#8835). modelId stays
+  // null at this site: the finding does not carry which concrete models ran, and recording a guess would be
+  // worse than recording nothing (the reviewDiagnostics ledger holds the per-run model identities).
   {
+    const aiJudgment = gate.blockers.find((blocker) => AI_JUDGMENT_BLOCKER_CODES.has(blocker.code));
     const { record, recordDigest } = await buildDecisionRecord({
       repoFullName,
       pullNumber: pr.number,
@@ -3344,7 +3350,8 @@ async function runAgentMaintenancePlanAndExecute(
       gatePack: settings.gatePack,
       ciState: null,
       modelId: null,
-      promptDigest: null,
+      promptDigest: aiJudgment !== undefined ? await contentDigest({ version: REVIEW_PROMPT_VERSION, template: REVIEW_SYSTEM_PROMPT }) : null,
+      aiConfidence: aiJudgment?.confidence ?? null,
     });
     await persistDecisionRecord(env, record, recordDigest);
   }

src/review/decision-record.ts

@@ -14,7 +14,7 @@
 import { errorMessage, nowIso } from "../utils/json";
 
 /** Bump when the record's FIELD SET changes meaning — consumers compare records only within a version. */
-export const DECISION_RECORD_SCHEMA_VERSION = "1";
+export const DECISION_RECORD_SCHEMA_VERSION = "2"; // v2 (#8834): + aiConfidence, model/prompt commitments live
 
 /**
  * Canonical JSON: recursively key-sorted, no insignificant whitespace — the ONE serialization every digest
@@ -72,17 +72,22 @@ export type DecisionRecord = {
   /** Model + prompt commitments when an AI review contributed; null for rule-only decisions. */
   modelId: string | null;
   promptDigest: string | null;
+  /** #8834: the calibrated confidence of the AI-judgment finding that shaped this decision (consensus
+   *  defect / split), null when no AI judgment contributed. Persisted so every decision joins the
+   *  risk-control calibration set (#8835) with its confidence attached. */
+  aiConfidence: number | null;
   decidedAt: string;
 };
 
 /** Assemble the record and its own content digest. PURE given pre-computed digests. Normalizes the
  *  optional-shaped caller fields (undefined -> null) HERE so call sites carry no fallback arms of their own. */
 export async function buildDecisionRecord(
-  input: Omit<DecisionRecord, "schemaVersion" | "decidedAt" | "gatePack" | "ciState" | "baseSha"> & {
+  input: Omit<DecisionRecord, "schemaVersion" | "decidedAt" | "gatePack" | "ciState" | "baseSha" | "aiConfidence"> & {
     decidedAt?: string;
     gatePack?: string | null | undefined;
     ciState?: string | null | undefined;
     baseSha?: string | null | undefined;
+    aiConfidence?: number | null | undefined;
   },
 ): Promise<{ record: DecisionRecord; recordDigest: string }> {
   const record: DecisionRecord = {
@@ -92,6 +97,7 @@ export async function buildDecisionRecord(
     gatePack: input.gatePack ?? null,
     ciState: input.ciState ?? null,
     baseSha: input.baseSha ?? null,
+    aiConfidence: input.aiConfidence ?? null,
   };
   return { record, recordDigest: await contentDigest(record) };
 }
@@ -139,7 +145,7 @@ export function renderDecisionRecordSection(record: DecisionRecord, recordDigest
     `- **action**: ${record.action} · **clause**: \`${record.reasonCode}\``,
     `- **config**: \`${short(record.configDigest)}\`${record.gatePack ? ` · **pack**: ${record.gatePack}` : ""}${record.ciState ? ` · **ci**: ${record.ciState}` : ""}`,
     ...(record.modelId !== null || record.promptDigest !== null
-      ? [`- **model**: ${record.modelId ?? "n/a"}${record.promptDigest !== null ? ` · **prompt**: \`${short(record.promptDigest)}\`` : ""}`]
+      ? [`- **model**: ${record.modelId ?? "n/a"}${record.promptDigest !== null ? ` · **prompt**: \`${short(record.promptDigest)}\`` : ""}${record.aiConfidence !== null ? ` · **confidence**: ${record.aiConfidence}` : ""}`]
       : []),
     `- **record**: \`${short(recordDigest)}\` (schema v${record.schemaVersion}, head \`${record.headSha.slice(0, 7)}\`)`,
   ];

src/services/ai-review.ts

@@ -72,7 +72,9 @@ export const SCOPE_MISMATCH_ASSESSMENT =
  *  live reclassifiable case (PR #8735) carried a ~300-char rationale naming specific call sites. */
 export const SCOPE_RECLASSIFY_MIN_RATIONALE_CHARS = 40;
 
-const REVIEW_SYSTEM_PROMPT = [
+// Exported for the decision record (#8834): the template commitment digest is computed over this constant
+// plus REVIEW_PROMPT_VERSION, so a silent template edit changes every subsequent record digest.
+export const REVIEW_SYSTEM_PROMPT = [
   "You are a senior open-source maintainer giving a FOCUSED, high-signal code review of a single pull request diff.",
   "Read each meaningful hunk and review like a careful human; judge ONLY the diff and the context provided.",
   "Respond with ONLY a JSON object of this exact shape (no prose, no code fence):",

test/unit/decision-record.test.ts

@@ -60,6 +60,7 @@ function recordInput(over: Partial<DecisionRecord> = {}): Omit<DecisionRecord, "
     ciState: "failed",
     modelId: null,
     promptDigest: null,
+    aiConfidence: null,
     ...over,
   };
 }
@@ -71,10 +72,14 @@ describe("buildDecisionRecord / persistDecisionRecord", () => {
     expect(typeof record.decidedAt).toBe("string");
     expect(recordDigest).toBe(await contentDigest(record));
     // Call sites pass optional-shaped settings fields raw; normalization happens HERE, once.
-    const { record: bare } = await buildDecisionRecord({ ...recordInput(), gatePack: undefined, ciState: undefined, baseSha: undefined });
+    const { record: bare } = await buildDecisionRecord({ ...recordInput(), gatePack: undefined, ciState: undefined, baseSha: undefined, aiConfidence: undefined });
     expect(bare.gatePack).toBeNull();
     expect(bare.ciState).toBeNull();
     expect(bare.baseSha).toBeNull();
+    expect(bare.aiConfidence).toBeNull();
+    // #8834: a stated confidence (including explicit 0) survives normalization.
+    const { record: withConf } = await buildDecisionRecord({ ...recordInput(), aiConfidence: 0 });
+    expect(withConf.aiConfidence).toBe(0);
   });
 
   it("persists with latest-finalize-wins per (target, head sha)", async () => {
@@ -114,10 +119,11 @@ describe("renderDecisionRecordSection", () => {
     expect(body).toContain(recordDigest.slice(0, 12));
     expect(body).not.toContain("**model**");
 
-    const ai = await buildDecisionRecord(recordInput({ modelId: "claude-sonnet-5", promptDigest: "p".repeat(64) }));
+    const ai = await buildDecisionRecord(recordInput({ modelId: "claude-sonnet-5", promptDigest: "p".repeat(64), aiConfidence: 0.97 }));
     const aiBody = renderDecisionRecordSection(ai.record, ai.recordDigest);
     expect(aiBody).toContain("**model**: claude-sonnet-5");
     expect(aiBody).toContain("`pppppppppppp`");
+    expect(aiBody).toContain("**confidence**: 0.97");
     // Bounded: a record section must stay a small fixed-size block.
     expect(aiBody.length).toBeLessThan(700);
 

test/unit/queue-2.test.ts

@@ -364,6 +364,13 @@ describe("queue processors", () => {
     // two-pass test), so the disposition planner's own audit record is the observable proof instead.
     const close = await env.DB.prepare("select outcome, detail from audit_events where event_type = ? order by rowid desc limit 1").bind("agent.action.close").first<{ outcome: string; detail: string }>();
     expect(close?.outcome).toBe("completed");
+    // #8834: an AI-judgment-shaped decision persists its confidence + prompt-template commitment in the
+    // decision record — the row every future calibration read keys on.
+    const record = await env.DB.prepare("select record_json from decision_records where repo_full_name = 'owner/agent-repo' and pull_number = 9 order by created_at desc limit 1").first<{ record_json: string }>();
+    const parsedRecord = JSON.parse(record!.record_json) as { aiConfidence: number | null; promptDigest: string | null; schemaVersion: string };
+    expect(parsedRecord.schemaVersion).toBe("2");
+    expect(parsedRecord.aiConfidence).toBe(0.3); // the cached sub-floor defect's calibrated confidence
+    expect(typeof parsedRecord.promptDigest).toBe("string");
   });
 
   it("posts the 🟪 reviewing placeholder before the AI review runs, then overwrites it with the verdict (#reviewing-placeholder)", async () => {