JSONbored · loopover-orb · Jul 28, 2026 · Jul 28, 2026
@@ -0,0 +1,22 @@
+-- #9028 (epic #8828, Replay v2): the exact system prompt sent to the model, for re-query action matching.
+--
+-- The public decision record commits to the prompt via `promptDigest` (sha256 of the ACTUAL
+-- buildSystemPrompt output for that call, #9124) -- a contributor can verify the commitment, but a digest
+-- cannot be re-queried. `scripts/replay-decision.ts --requery` needs the text itself to re-run the model for
+-- the same target and report an ACTION-MATCH RATE (never "reproducibility": hosted inference is not
+-- bit-deterministic even at temperature 0, and the docs say so).
+--
+-- DELIBERATELY A PRIVATE SIBLING TABLE, mirroring decision_replay_inputs' posture exactly (#8838): the
+-- prompt embeds the full diff plus contributor content, so it must never live in the public record. Row-size
+-- is why it is ALSO not a decision_replay_inputs column: prompts run to hundreds of KB, and pinning them to
+-- the replay-input row would drag that table's every read through the blob.
+--
+-- Retention is 30 days (src/db/retention.ts), deliberately SHORTER than decision_replay_inputs' 180: the
+-- re-query mode is an operator debugging tool for RECENT decisions, the blob is the largest and most
+-- sensitive artifact in the replay family, and the public promptDigest commitment outlives the text forever.
+CREATE TABLE IF NOT EXISTS decision_replay_prompts (
+  record_id   TEXT PRIMARY KEY,            -- decision_records.id (record:<owner/repo>#<pr>@<head sha>)
+  prompt_json TEXT NOT NULL,               -- { systemPrompt } -- exactly what was sent, nothing derived
+  created_at  TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_decision_replay_prompts_created_at ON decision_replay_prompts(created_at);
@@ -46,6 +46,10 @@ export const RAW_SQL_ONLY_TABLES: Set<string> = new Set([
   "decision_ledger_anchors",
   "decision_records",
   "decision_replay_inputs",
+  // #9028: same raw-SQL-only posture as its sibling above — an operator-private blob table the drizzle
+  // schema never queries (writes go through persistDecisionReplayPrompt's raw statement; reads happen only
+  // in the operator's own extract SQL for the replay CLI).
+  "decision_replay_prompts",
   "ai_review_verdict_flips",
   "global_agent_controls",
   "global_contributor_blacklist",

@@ -27,8 +27,38 @@
 // Exit codes: 0 = replayed, same verdict ("here is the clause"); 1 = DIVERGENCE — a bug by definition,
 // file it with the printed stage diff; 2 = unusable input. Replay mode cannot re-query the model or touch
 // any network/DB by construction: replayDecision is a pure function of the two JSON values.
+//
+// #9028: `--requery <n>` is the ONE deliberately-networked mode, for the one stage pure replay cannot cover.
+// It re-runs the model n times against the EXACT persisted prompt and reports an ACTION-MATCH RATE — never
+// "reproducibility": hosted inference is not bit-deterministic even at temperature 0 (batching, kernel
+// scheduling, silent model revisions), so bit-comparing outputs would measure the provider's scheduler, not
+// the decision. What CAN be honestly measured is whether fresh runs land in the same verdict CLASS the
+// recorded decision acted on (defect vs clean, through the same parseModelReview the live pipeline uses).
+//
+// The bundle gains an optional `prompt` for this mode. EXTRACT (joins the private prompts sibling by the
+// BASE record id, so a supersession's `:rev<N>` row still finds its head's prompt):
+//   SELECT json_build_object(
+//            'record', to_jsonb(dr) || dr.record_json::jsonb,
+//            'replayInput', dri.replay_json::jsonb,
+//            'prompt', drp.prompt_json::jsonb
+//          )
+//     FROM decision_records dr
+//     JOIN decision_replay_inputs dri ON dri.record_id = dr.id
+//     LEFT JOIN decision_replay_prompts drp
+//            ON drp.record_id = 'record:' || dr.repo_full_name || '#' || dr.pull_number || '@' || dr.head_sha
+//    WHERE dr.id = 'record:<owner/repo>#<pr>@<head sha>';
+//
+// Provider config (explicit env, no defaults — this is a debugging tool, not a service):
+//   REPLAY_AI_BASE_URL + REPLAY_AI_MODEL [+ REPLAY_AI_API_KEY]   -> any OpenAI-compatible endpoint (Ollama etc.)
+//   ANTHROPIC_API_KEY + REPLAY_AI_MODEL                          -> Anthropic
+//
+// Requery exit codes: 0 = report produced (a low rate is a FINDING, not a failure); 2 = unusable input or
+// missing provider config. Prompts above the persistence cap were skipped at capture time, never truncated —
+// a truncated prompt re-queried would report a rate for a prompt that was never sent.
 import { readFileSync } from "node:fs";
 import { replayDecision, type DecisionReplayInput, type ReplayableRecord } from "../src/review/decision-replay";
+import { recordedJudgmentClass, runRequery } from "../src/review/decision-requery";
+import { createAnthropicAi, createOpenAiCompatibleAi, type SelfHostAi } from "../src/selfhost/ai";
 
 /** Parse + normalize a bundle (snake_case SQL rows accepted) and replay it. Exported for tests.
  *
@@ -61,9 +91,86 @@ export function runReplayBundle(raw: string, atMs?: number): { outcome: ReturnTy
   return { outcome: replayDecision(record, replayInput, atMs === undefined ? {} : { nowMs: atMs }) };
 }
 
+/** Parse the bundle's optional prompt + replay input for requery. Exported for tests. */
+export function parseRequeryBundle(raw: string): { systemPrompt: string; userPrompt: string; recordedClass: "defect" | "clean" } | { error: string } {
+  let bundle: { replayInput?: DecisionReplayInput; prompt?: { systemPrompt?: unknown; userPrompt?: unknown } };
+  try {
+    bundle = JSON.parse(raw) as never;
+  } catch (error) {
+    return { error: `unparseable bundle JSON: ${error instanceof Error ? error.message : String(error)}` };
+  }
+  const systemPrompt = bundle.prompt?.systemPrompt;
+  const userPrompt = bundle.prompt?.userPrompt;
+  if (typeof systemPrompt !== "string" || systemPrompt.length === 0 || typeof userPrompt !== "string" || userPrompt.length === 0) {
+    return { error: "requery needs bundle.prompt.{systemPrompt,userPrompt} — extract with the prompts LEFT JOIN in this file's header (rows age out after 30 days; older decisions cannot be re-queried, only replayed)" };
+  }
+  if (!bundle.replayInput || !Array.isArray(bundle.replayInput.findings)) {
+    return { error: "requery needs bundle.replayInput.findings to derive the recorded verdict class" };
+  }
+  return { systemPrompt, userPrompt, recordedClass: recordedJudgmentClass(bundle.replayInput) };
+}
+
+/** Build the provider client from explicit env. Exported for tests; returns an error string when unconfigured. */
+export function requeryClientFromEnv(env: Record<string, string | undefined>): { ai: SelfHostAi; model: string } | { error: string } {
+  const model = env.REPLAY_AI_MODEL;
+  if (!model) return { error: "requery needs REPLAY_AI_MODEL (explicit — this tool never guesses which model to spend against)" };
+  if (env.REPLAY_AI_BASE_URL) {
+    return { ai: createOpenAiCompatibleAi({ baseUrl: env.REPLAY_AI_BASE_URL, apiKey: env.REPLAY_AI_API_KEY, model }), model };
+  }
+  if (env.ANTHROPIC_API_KEY) {
+    return { ai: createAnthropicAi({ apiKey: env.ANTHROPIC_API_KEY, model }), model };
+  }
+  return { error: "requery needs REPLAY_AI_BASE_URL (OpenAI-compatible) or ANTHROPIC_API_KEY" };
+}
+
 const invokedDirectly = process.argv[1]?.endsWith("replay-decision.ts") === true;
 if (invokedDirectly) {
   const argv = process.argv.slice(2);
+  const requeryIndex = argv.indexOf("--requery");
+  if (requeryIndex !== -1) {
+    const runsRaw = argv[requeryIndex + 1];
+    const runs = Number(runsRaw);
+    if (!Number.isInteger(runs) || runs < 1 || runs > 25) {
+      console.error("replay-decision: --requery requires a run count between 1 and 25");
+      process.exit(2);
+    }
+    const requerySource = argv.filter((_arg, index) => index !== requeryIndex && index !== requeryIndex + 1)[0];
+    if (!requerySource) {
+      console.error("usage: replay-decision.ts <bundle.json | -> --requery <n>");
+      process.exit(2);
+    }
+    const rawBundle = requerySource === "-" ? readFileSync(0, "utf8") : readFileSync(requerySource, "utf8");
+    const parsed = parseRequeryBundle(rawBundle);
+    if ("error" in parsed) {
+      console.error(`replay-decision: ${parsed.error}`);
+      process.exit(2);
+    }
+    const client = requeryClientFromEnv(process.env);
+    if ("error" in client) {
+      console.error(`replay-decision: ${client.error}`);
+      process.exit(2);
+    }
+    const report = await runRequery({
+      systemPrompt: parsed.systemPrompt,
+      userPrompt: parsed.userPrompt,
+      runs,
+      recordedClass: parsed.recordedClass,
+      // The live call's own shape (ai-review.ts): a system turn plus a user turn, temperature 0. Matching it
+      // is what makes the rate a statement about the DECISION and not about a different way of asking.
+      callModel: async (systemPrompt, userPrompt) => {
+        const result = await client.ai.run(client.model, {
+          temperature: 0,
+          messages: [
+            { role: "system", content: systemPrompt },
+            { role: "user", content: userPrompt },
+          ],
+        });
+        return result.response ?? "";
+      },
+    });
+    console.log(JSON.stringify(report, null, 2));
+    process.exit(0);
+  }
   const atIndex = argv.indexOf("--at");
   const atRaw = atIndex === -1 ? undefined : argv[atIndex + 1];
   if (atIndex !== -1 && (atRaw === undefined || !Number.isFinite(Number(atRaw)))) {

@@ -101,6 +101,10 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [
   { table: "predicted_gate_calibration_ledger", column: "created_at", days: 90 },
   { table: "contributor_gate_history", column: "created_at", days: 90 },
   { table: "decision_replay_inputs", column: "created_at", days: 180 },
+  // #9028: deliberately SHORTER than decision_replay_inputs' 180d -- the prompt blob embeds the full diff
+  // plus contributor content (the largest, most sensitive artifact in the replay family), the re-query mode
+  // is a debugging tool for RECENT decisions, and the public promptDigest commitment outlives the text.
+  { table: "decision_replay_prompts", column: "created_at", days: 30 },
 ];
 
 // #9083: a real, single-column, indexable primary key for the ordered-range delete below, keyed by table
@@ -142,6 +146,8 @@ export const RETENTION_PK_COLUMN: Readonly<Record<string, string>> = {
   contributor_gate_history: "id",
   // decision_replay_inputs keys on record_id (decision_records.id), not an `id` column.
   decision_replay_inputs: "record_id",
+  // Same key shape as decision_replay_inputs above, for the same reason.
+  decision_replay_prompts: "record_id",
 };
 
 /**

@@ -26,6 +26,7 @@ import { recordAuditEvent, getDecryptedRepositoryAiKey, getRepository, listCheck
 import { registerHeldLock, unregisterHeldLock } from "./held-lock-registry";
 import { recordRoutingShadow } from "../services/reviewer-routing";
 import { scoreJudgmentAgreement } from "../review/judgment-agreement";
+import { persistDecisionReplayPrompt } from "../review/decision-replay";
 import { createInstallationToken } from "../github/app";
 import type { AgentActionMode } from "../settings/agent-execution";
 import { buildAiReviewDiff } from "../review/review-diff";
@@ -927,6 +928,24 @@ export async function runAiReviewForAdvisory(
     // the REAL reviewer identities and the REAL system prompt into DecisionRecord instead of hardcoding null.
     const aiJudgmentModelIds = parsedReviewModelIds(result.reviewDiagnostics ?? []);
     const aiJudgmentPromptDigest = result.systemPromptDigest;
+    // #9028: capture the ACTUAL prompt text the digest above commits to, so the replay harness's re-query
+    // mode can re-run the model for this exact target. Written here (not at the decision-record persist)
+    // because the text must never travel through findings -- findings reach public render surfaces, and the
+    // prompt embeds the full diff. Keyed by the derivable base record id; an orphan row from a pass that
+    // never finalizes a decision ages out with the table's 30-day retention. Every verdict class is captured
+    // -- a CLEAN decision's action-match rate matters exactly as much as a defect's.
+    /* v8 ignore next -- the no-head arm is a plain skip, not a protection: without a SHA there is nothing to
+     * key the prompt row by, and a ghost PR's decision record is head-keyed too, so requery would have no row
+     * to join even if one were written. */
+    if (args.advisory.headSha) {
+      await persistDecisionReplayPrompt(env, {
+        repoFullName: args.repoFullName,
+        pullNumber: args.pr.number,
+        headSha: args.advisory.headSha,
+        systemPrompt: result.systemPrompt,
+        userPrompt: result.userPrompt,
+      });
+    }
     // #8834: inter-run agreement over the stances this review ALREADY produced (#8229's reviewerVotes) —
     // zero additional AI spend. Computed once and attached to whichever AI-judgment finding is built below,
     // so the decision record carries a per-decision confidence signal for the calibration set (#8835).

@@ -214,6 +214,50 @@ export async function persistDecisionReplayInputForGate(
   });
 }
 
+/** #9028: prompts above this size are SKIPPED, never truncated -- a truncated prompt re-queried against the
+ *  model would report an action-match rate for a prompt that was never sent, which is worse than reporting
+ *  nothing. The public promptDigest commitment is unaffected either way. Sized well under D1's row ceiling. */
+export const DECISION_REPLAY_PROMPT_MAX_CHARS = 900_000;
+
+/**
+ * #9028: persist the EXACT system prompt sent to the model, keyed by the BASE record id
+ * (`record:<repo>#<pr>@<head sha>`) rather than a supersession's `:rev<N>` id -- the prompt is a property of
+ * the (target, head, resolved-config) triple, not of which revision row happened to land, and upsert-last-wins
+ * means a superseding pass that rebuilt the prompt (config changed between passes) leaves the one that
+ * matches the LATEST decision. Private sibling of decision_replay_inputs with the same posture and a shorter
+ * (30-day) retention; the text must never reach the public record or any rendered surface.
+ *
+ * Best-effort like every persist in this family: prompt capture must never break the review pass that
+ * produced it.
+ */
+export async function persistDecisionReplayPrompt(
+  env: Env,
+  args: { repoFullName: string; pullNumber: number; headSha: string; systemPrompt: string; userPrompt: string },
+): Promise<void> {
+  if (args.systemPrompt.length + args.userPrompt.length > DECISION_REPLAY_PROMPT_MAX_CHARS) {
+    console.warn(
+      JSON.stringify({
+        event: "decision_replay_prompt_skipped_oversize",
+        repoFullName: args.repoFullName,
+        pullNumber: args.pullNumber,
+        chars: args.systemPrompt.length + args.userPrompt.length,
+      }),
+    );
+    return;
+  }
+  const recordId = `record:${args.repoFullName}#${args.pullNumber}@${args.headSha}`.slice(0, 250);
+  try {
+    await env.DB.prepare(
+      `INSERT INTO decision_replay_prompts (record_id, prompt_json, created_at) VALUES (?, ?, ?)
+       ON CONFLICT(record_id) DO UPDATE SET prompt_json = excluded.prompt_json, created_at = excluded.created_at`,
+    )
+      .bind(recordId, JSON.stringify({ systemPrompt: args.systemPrompt, userPrompt: args.userPrompt }), nowIso())
+      .run();
+  } catch (error) {
+    console.warn(JSON.stringify({ event: "decision_replay_prompt_persist_error", recordId: recordId.slice(0, 120), message: errorMessage(error).slice(0, 160) }));
+  }
+}
+
 export async function persistDecisionReplayInput(env: Env, recordId: string, input: DecisionReplayInput): Promise<void> {
   try {
     await env.DB.prepare(