From 0d631a1824614297d6a04aafd3a51db6543790c1 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 26 Jul 2026 05:47:54 -0700 Subject: [PATCH] feat(orb): persist the AI judgment's confidence and prompt commitment in the decision record When an AI-judgment finding (ai_consensus_defect / ai_review_split) shaped a decision, the decision record now carries the finding's calibrated confidence and a digest of the prompt template commitment (REVIEW_PROMPT_VERSION + REVIEW_SYSTEM_PROMPT) -- the fields that join every decision to the risk-control calibration set (#8835) and complete #8842's tracked follow-up. Advances #8834 (the persistence half its issue text names as the core deliverable; the flag-gated rotated-exemplar N-run variant remains -- the existing dual-model consensus already IS a two-sample agreement signal, so the extra runs are a cost knob, not the prerequisite). - DecisionRecord schema v2: + aiConfidence (null when no AI judgment contributed); normalized once at the builder like its siblings - processors: source the confidence from the gate's own AI-judgment blocker; modelId deliberately stays null at this site -- the finding does not carry which concrete models ran, and a guess would be worse than nothing (reviewDiagnostics holds per-run identities) - render: the model line gains the confidence when present --- src/queue/processors.ts | 19 +++++++++++++------ src/review/decision-record.ts | 12 +++++++++--- src/services/ai-review.ts | 4 +++- test/unit/decision-record.test.ts | 10 ++++++++-- test/unit/queue-2.test.ts | 7 +++++++ 5 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 1a3ca2c191..818cd6671f 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -644,6 +644,8 @@ import { recordPrOutcome, recordReversalSignals, } from "../review/outcomes-wire"; +import { AI_JUDGMENT_BLOCKER_CODES } from "../rules/advisory"; +import { REVIEW_PROMPT_VERSION, REVIEW_SYSTEM_PROMPT } from "../services/ai-review"; import { maybeApplyCloseAuditHoldout } from "../review/close-audit-holdout"; import { buildDecisionRecord, contentDigest, loadDecisionRecordCollapsible, persistDecisionRecord } from "../review/decision-record"; import { neutralHoldReasonCode, nativeGateActionFromConclusion, recordNativeGateDecision } from "../review/parity-wire"; @@ -3322,12 +3324,16 @@ async function runAgentMaintenancePlanAndExecute( // erase a prior miner-authored prediction for the same head. minerAuthored: breakerMinerAuthored, }); - // #8836: the content-addressed decision record — the public commitment that lets a contributor argue - // "clause X of config abc123 decided this". Persist-only here (best-effort inside); the same reasonCode - // expression as recordNativeGateDecision above so the record and the calibration row can never disagree - // about WHY. Model/prompt commitments are wired when the AI-review contribution lands (tracked on #8836's - // follow-up note); rule-only decisions carry null there by design. + // #8836/#8834: the content-addressed decision record — the public commitment that lets a contributor + // argue "clause X of config abc123 decided this". Persist-only here (best-effort inside); the same + // reasonCode expression as recordNativeGateDecision above so the record and the calibration row can never + // disagree about WHY. When an AI JUDGMENT shaped this decision (an AI_JUDGMENT_BLOCKER_CODES finding is + // present), the record carries the prompt-template commitment and the finding's calibrated confidence — + // the confidence is what joins every decision to the risk-control calibration set (#8835). modelId stays + // null at this site: the finding does not carry which concrete models ran, and recording a guess would be + // worse than recording nothing (the reviewDiagnostics ledger holds the per-run model identities). { + const aiJudgment = gate.blockers.find((blocker) => AI_JUDGMENT_BLOCKER_CODES.has(blocker.code)); const { record, recordDigest } = await buildDecisionRecord({ repoFullName, pullNumber: pr.number, @@ -3344,7 +3350,8 @@ async function runAgentMaintenancePlanAndExecute( gatePack: settings.gatePack, ciState: null, modelId: null, - promptDigest: null, + promptDigest: aiJudgment !== undefined ? await contentDigest({ version: REVIEW_PROMPT_VERSION, template: REVIEW_SYSTEM_PROMPT }) : null, + aiConfidence: aiJudgment?.confidence ?? null, }); await persistDecisionRecord(env, record, recordDigest); } diff --git a/src/review/decision-record.ts b/src/review/decision-record.ts index 7cc6e43b18..64606af08c 100644 --- a/src/review/decision-record.ts +++ b/src/review/decision-record.ts @@ -14,7 +14,7 @@ import { errorMessage, nowIso } from "../utils/json"; /** Bump when the record's FIELD SET changes meaning — consumers compare records only within a version. */ -export const DECISION_RECORD_SCHEMA_VERSION = "1"; +export const DECISION_RECORD_SCHEMA_VERSION = "2"; // v2 (#8834): + aiConfidence, model/prompt commitments live /** * Canonical JSON: recursively key-sorted, no insignificant whitespace — the ONE serialization every digest @@ -72,17 +72,22 @@ export type DecisionRecord = { /** Model + prompt commitments when an AI review contributed; null for rule-only decisions. */ modelId: string | null; promptDigest: string | null; + /** #8834: the calibrated confidence of the AI-judgment finding that shaped this decision (consensus + * defect / split), null when no AI judgment contributed. Persisted so every decision joins the + * risk-control calibration set (#8835) with its confidence attached. */ + aiConfidence: number | null; decidedAt: string; }; /** Assemble the record and its own content digest. PURE given pre-computed digests. Normalizes the * optional-shaped caller fields (undefined -> null) HERE so call sites carry no fallback arms of their own. */ export async function buildDecisionRecord( - input: Omit & { + input: Omit & { decidedAt?: string; gatePack?: string | null | undefined; ciState?: string | null | undefined; baseSha?: string | null | undefined; + aiConfidence?: number | null | undefined; }, ): Promise<{ record: DecisionRecord; recordDigest: string }> { const record: DecisionRecord = { @@ -92,6 +97,7 @@ export async function buildDecisionRecord( gatePack: input.gatePack ?? null, ciState: input.ciState ?? null, baseSha: input.baseSha ?? null, + aiConfidence: input.aiConfidence ?? null, }; return { record, recordDigest: await contentDigest(record) }; } @@ -139,7 +145,7 @@ export function renderDecisionRecordSection(record: DecisionRecord, recordDigest `- **action**: ${record.action} · **clause**: \`${record.reasonCode}\``, `- **config**: \`${short(record.configDigest)}\`${record.gatePack ? ` · **pack**: ${record.gatePack}` : ""}${record.ciState ? ` · **ci**: ${record.ciState}` : ""}`, ...(record.modelId !== null || record.promptDigest !== null - ? [`- **model**: ${record.modelId ?? "n/a"}${record.promptDigest !== null ? ` · **prompt**: \`${short(record.promptDigest)}\`` : ""}`] + ? [`- **model**: ${record.modelId ?? "n/a"}${record.promptDigest !== null ? ` · **prompt**: \`${short(record.promptDigest)}\`` : ""}${record.aiConfidence !== null ? ` · **confidence**: ${record.aiConfidence}` : ""}`] : []), `- **record**: \`${short(recordDigest)}\` (schema v${record.schemaVersion}, head \`${record.headSha.slice(0, 7)}\`)`, ]; diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 964396231e..b746d67ab8 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -72,7 +72,9 @@ export const SCOPE_MISMATCH_ASSESSMENT = * live reclassifiable case (PR #8735) carried a ~300-char rationale naming specific call sites. */ export const SCOPE_RECLASSIFY_MIN_RATIONALE_CHARS = 40; -const REVIEW_SYSTEM_PROMPT = [ +// Exported for the decision record (#8834): the template commitment digest is computed over this constant +// plus REVIEW_PROMPT_VERSION, so a silent template edit changes every subsequent record digest. +export const REVIEW_SYSTEM_PROMPT = [ "You are a senior open-source maintainer giving a FOCUSED, high-signal code review of a single pull request diff.", "Read each meaningful hunk and review like a careful human; judge ONLY the diff and the context provided.", "Respond with ONLY a JSON object of this exact shape (no prose, no code fence):", diff --git a/test/unit/decision-record.test.ts b/test/unit/decision-record.test.ts index a8187f353d..b972182d48 100644 --- a/test/unit/decision-record.test.ts +++ b/test/unit/decision-record.test.ts @@ -60,6 +60,7 @@ function recordInput(over: Partial = {}): Omit { expect(typeof record.decidedAt).toBe("string"); expect(recordDigest).toBe(await contentDigest(record)); // Call sites pass optional-shaped settings fields raw; normalization happens HERE, once. - const { record: bare } = await buildDecisionRecord({ ...recordInput(), gatePack: undefined, ciState: undefined, baseSha: undefined }); + const { record: bare } = await buildDecisionRecord({ ...recordInput(), gatePack: undefined, ciState: undefined, baseSha: undefined, aiConfidence: undefined }); expect(bare.gatePack).toBeNull(); expect(bare.ciState).toBeNull(); expect(bare.baseSha).toBeNull(); + expect(bare.aiConfidence).toBeNull(); + // #8834: a stated confidence (including explicit 0) survives normalization. + const { record: withConf } = await buildDecisionRecord({ ...recordInput(), aiConfidence: 0 }); + expect(withConf.aiConfidence).toBe(0); }); it("persists with latest-finalize-wins per (target, head sha)", async () => { @@ -114,10 +119,11 @@ describe("renderDecisionRecordSection", () => { expect(body).toContain(recordDigest.slice(0, 12)); expect(body).not.toContain("**model**"); - const ai = await buildDecisionRecord(recordInput({ modelId: "claude-sonnet-5", promptDigest: "p".repeat(64) })); + const ai = await buildDecisionRecord(recordInput({ modelId: "claude-sonnet-5", promptDigest: "p".repeat(64), aiConfidence: 0.97 })); const aiBody = renderDecisionRecordSection(ai.record, ai.recordDigest); expect(aiBody).toContain("**model**: claude-sonnet-5"); expect(aiBody).toContain("`pppppppppppp`"); + expect(aiBody).toContain("**confidence**: 0.97"); // Bounded: a record section must stay a small fixed-size block. expect(aiBody.length).toBeLessThan(700); diff --git a/test/unit/queue-2.test.ts b/test/unit/queue-2.test.ts index 8d99d954c9..0d53daa52a 100644 --- a/test/unit/queue-2.test.ts +++ b/test/unit/queue-2.test.ts @@ -364,6 +364,13 @@ describe("queue processors", () => { // two-pass test), so the disposition planner's own audit record is the observable proof instead. const close = await env.DB.prepare("select outcome, detail from audit_events where event_type = ? order by rowid desc limit 1").bind("agent.action.close").first<{ outcome: string; detail: string }>(); expect(close?.outcome).toBe("completed"); + // #8834: an AI-judgment-shaped decision persists its confidence + prompt-template commitment in the + // decision record — the row every future calibration read keys on. + const record = await env.DB.prepare("select record_json from decision_records where repo_full_name = 'owner/agent-repo' and pull_number = 9 order by created_at desc limit 1").first<{ record_json: string }>(); + const parsedRecord = JSON.parse(record!.record_json) as { aiConfidence: number | null; promptDigest: string | null; schemaVersion: string }; + expect(parsedRecord.schemaVersion).toBe("2"); + expect(parsedRecord.aiConfidence).toBe(0.3); // the cached sub-floor defect's calibrated confidence + expect(typeof parsedRecord.promptDigest).toBe("string"); }); it("posts the 🟪 reviewing placeholder before the AI review runs, then overwrites it with the verdict (#reviewing-placeholder)", async () => {