feat(review): rotated-exemplar self-consistency sampling behind AI_REVIEW_SELF_CONSISTENCY_RUNS

apps/loopover-ui/src/lib/selfhost-env-reference.ts

@@ -81,6 +81,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "AI_REVIEW_PLAN",
     firstReference: "src/services/ai-review.ts",
   },
+  {
+    name: "AI_REVIEW_SELF_CONSISTENCY_RUNS",
+    firstReference: "src/services/ai-review.ts",
+  },
   {
     name: "AI_SUMMARIES_ENABLED",
     firstReference: "src/services/ai-review.ts",
@@ -705,6 +709,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `AI_PROVIDER` | `src/selfhost/ai-config.ts` |",
   "| `AI_PUBLIC_COMMENTS_ENABLED` | `src/services/ai-review.ts` |",
   "| `AI_REVIEW_PLAN` | `src/services/ai-review.ts` |",
+  "| `AI_REVIEW_SELF_CONSISTENCY_RUNS` | `src/services/ai-review.ts` |",
   "| `AI_SUMMARIES_ENABLED` | `src/services/ai-review.ts` |",
   "| `AI_VISION` | `src/queue/processors.ts` |",
   "| `AI_VISION_API_KEY` | `src/server.ts` |",

src/env.d.ts

@@ -106,6 +106,12 @@ declare global {
      *  ai-review/ai-slop/ai-summaries/planner). The unit name is a Workers-AI holdover; it's applied as a
      *  provider-agnostic heuristic budget regardless of which configured provider actually serves the request. */
     AI_DAILY_NEURON_BUDGET?: string;
+    /** #8834: TOTAL judge evaluations per AI review for rotated-exemplar self-consistency. Unset/0 = off
+     *  (single evaluation, today's behavior); 2 or 3 = extra same-judge runs with rotated few-shot exemplars,
+     *  their stances folded into the recorded per-decision confidence. Each extra run is a real AI charge
+     *  riding the daily neuron budget; an exhausted budget degrades to fewer runs and a lower recorded
+     *  confidence, never a fabricated one. */
+    AI_REVIEW_SELF_CONSISTENCY_RUNS?: string;
     /** Per-repository/day cap for maintainer-paid BYOK AI review provider calls. */
     AI_BYOK_DAILY_REPO_LIMIT?: string;
     /** #9061: per-repository/day cap on AI calls for the FREE/default chain — the path the self-host actually

src/queue/ai-review-orchestration.ts

@@ -949,7 +949,11 @@ export async function runAiReviewForAdvisory(
     // #8834: inter-run agreement over the stances this review ALREADY produced (#8229's reviewerVotes) —
     // zero additional AI spend. Computed once and attached to whichever AI-judgment finding is built below,
     // so the decision record carries a per-decision confidence signal for the calibration set (#8835).
-    const aiJudgmentAgreement = (verbalizedConfidence: number) => scoreJudgmentAgreement(result.reviewerVotes, verbalizedConfidence);
+    // #8834 both halves meet here: the free half scores whatever stances exist; the paid half (flag-gated
+    // rotated-exemplar runs) contributes extra SAME-judge stances through selfConsistencySamples -- folded
+    // into the agreement axis only, never into split detection, which reads reviewerVotes alone.
+    const aiJudgmentAgreement = (verbalizedConfidence: number) =>
+      scoreJudgmentAgreement([...result.reviewerVotes, ...result.selfConsistencySamples], verbalizedConfidence);
     if (result.consensusDefect) {
       findings.push({
         code: "ai_consensus_defect",

src/review/self-consistency.ts

@@ -0,0 +1,158 @@
+// Rotated-exemplar self-consistency sampling (#8834, epic #8828 Phase 3) — the PAID half of the per-decision
+// confidence signal, deliberately split from the free half that already shipped.
+//
+// judgment-agreement.ts scores inter-run agreement over whatever stances exist and costs nothing; its own
+// closing comment records that the extra-sampling half — re-running the SAME judge with rotated few-shot
+// exemplars ("simulated annotators", Trust or Escalate, ICLR 2025) — needed a budget decision and a flag
+// defaulting OFF before it could ship, because every extra run is a real per-review AI charge. This module is
+// that half, and the decisions are:
+//
+//   FLAG: `AI_REVIEW_SELF_CONSISTENCY_RUNS` — the TOTAL number of judge evaluations per review. Unset/0/
+//   unparseable ⇒ OFF (today's behavior, byte-identical). 2 or 3 ⇒ on; anything else clamps into that range,
+//   because one total run measures nothing and the literature's benefit saturates by three.
+//
+//   BUDGET: extra runs ride the existing daily-neuron accounting. When the remaining budget cannot fund the
+//   configured extras, the plan DEGRADES — fewer or zero extra runs — and the recorded confidence degrades
+//   with it through scoreJudgmentAgreement's own uncorroborated arm. It never fabricates a score: a decision
+//   that could not afford corroboration is recorded as uncorroborated, which is exactly what it is.
+//
+//   ROTATION: each extra run appends a DIFFERENT exemplar window to the system prompt, deterministically
+//   derived from the target seed + run index. Determinism matters twice over: replay (#9028) can reconstruct
+//   which exemplars a run saw, and two passes over the same target sample the same rotation rather than
+//   secretly measuring prompt-shuffle variance.
+//
+// The exemplars live here as versioned code constants rather than in test/golden-corpus/: the Worker bundle
+// cannot read test fixtures at runtime, and the corpus file carries gate-pipeline fixtures (findings/policy),
+// not judge-facing PR content. Same versioning discipline, different artifact. They are SYNTHETIC by
+// construction — never drawn from real reviewed PRs, so no contributor content can leak into every future
+// prompt on the instance.
+
+/** Bump when the exemplar set changes — it shifts every self-consistency prompt, so promptDigest moves too. */
+export const JUDGE_EXEMPLAR_SET_VERSION = 1;
+
+export type JudgeExemplar = {
+  id: string;
+  title: string;
+  diffExcerpt: string;
+  verdict: "defect" | "clean";
+  rationale: string;
+};
+
+/** Six synthetic exemplars, balanced 3/3, each small enough that a 3-exemplar window adds well under a KB. */
+export const JUDGE_EXEMPLARS: readonly JudgeExemplar[] = [
+  {
+    id: "null-deref-on-empty",
+    title: "Speed up lookup by skipping the guard",
+    diffExcerpt: '-  if (!items || items.length === 0) return null;\n   const first = items[0].id;',
+    verdict: "defect",
+    rationale: "Removes the empty-input guard while the very next line indexes the array: a crash on every empty call.",
+  },
+  {
+    id: "rename-only-refactor",
+    title: "Rename fetchRows to loadRows for clarity",
+    diffExcerpt: '-  const rows = await fetchRows(db);\n+  const rows = await loadRows(db);',
+    verdict: "clean",
+    rationale: "A pure rename with the definition and every call site moved together; behavior is byte-identical.",
+  },
+  {
+    id: "swallowed-error-branch",
+    title: "Stop noisy logging in the sync path",
+    diffExcerpt: '-  } catch (error) {\n-    throw error;\n+  } catch {\n+    /* ignore */',
+    verdict: "defect",
+    rationale: "Converts a propagated failure into silent success; callers now proceed on corrupt state with no signal.",
+  },
+  {
+    id: "test-added-for-fix",
+    title: "Add regression test for the off-by-one fix",
+    diffExcerpt: '+  it("includes the final page", () => {\n+    expect(paginate(21, 10).pages).toBe(3);\n+  });',
+    verdict: "clean",
+    rationale: "Adds a test pinning already-merged behavior; touches no production code path.",
+  },
+  {
+    id: "boundary-flip",
+    title: "Simplify the retry condition",
+    diffExcerpt: '-  while (attempt <= maxRetries) {\n+  while (attempt < maxRetries) {',
+    verdict: "defect",
+    rationale: "Silently drops the final permitted attempt; every caller now retries one time fewer than configured.",
+  },
+  {
+    id: "doc-comment-fix",
+    title: "Correct the stale doc comment on parseWindow",
+    diffExcerpt: '-// window is inclusive of both ends\n+// window is inclusive of start, exclusive of end',
+    verdict: "clean",
+    rationale: "Documentation-only change aligning the comment with the long-standing behavior; no code touched.",
+  },
+];
+
+/** How many exemplars each rotated window carries. Three of six gives six distinct contiguous windows, so
+ *  every (seed, runIndex) pair lands on a real rotation rather than a reshuffle of the same set. */
+export const EXEMPLAR_WINDOW_SIZE = 3;
+
+/** TOTAL evaluations per review (primary included). 0 = off. Explicit values clamp into {0, 2, 3}: one total
+ *  run cannot measure agreement, and the literature's benefit saturates by three. */
+export function resolveSelfConsistencyRuns(raw: string | undefined): number {
+  const parsed = Number(raw);
+  if (!raw || !Number.isFinite(parsed) || parsed < 2) return 0;
+  return Math.min(3, Math.floor(parsed));
+}
+
+/** Deterministic, dependency-free string hash (FNV-1a 32-bit) — rotation must be stable across runtimes. */
+function fnv1a(text: string): number {
+  let hash = 0x811c9dc5;
+  for (let i = 0; i < text.length; i += 1) {
+    hash ^= text.charCodeAt(i);
+    hash = Math.imul(hash, 0x01000193) >>> 0;
+  }
+  return hash >>> 0;
+}
+
+/**
+ * The exemplar window for one extra run, rotated deterministically by (seed, runIndex).
+ *
+ * `seed` is the review target's identity (repo#pr@sha), so the SAME target always samples the SAME rotation
+ * sequence — replayable, and immune to measuring shuffle variance — while different targets start at
+ * different offsets, so no single exemplar dominates the fleet's second opinions.
+ */
+export function rotatedExemplarWindow(seed: string, runIndex: number, exemplars: readonly JudgeExemplar[] = JUDGE_EXEMPLARS): JudgeExemplar[] {
+  if (exemplars.length === 0) return [];
+  const offset = (fnv1a(seed) + runIndex) % exemplars.length;
+  const window: JudgeExemplar[] = [];
+  for (let i = 0; i < Math.min(EXEMPLAR_WINDOW_SIZE, exemplars.length); i += 1) {
+    const exemplar = exemplars[(offset + i) % exemplars.length];
+    /* v8 ignore next -- modular indexing into a non-empty array cannot miss; the guard satisfies noUncheckedIndexedAccess. */
+    if (exemplar) window.push(exemplar);
+  }
+  return window;
+}
+
+/** Render one rotated window as a system-prompt suffix, matching the calibration-suffix idiom the judge
+ *  prompt already uses: labeled sections, no markdown the model could echo into findings. */
+export function rotatedExemplarSuffix(seed: string, runIndex: number, exemplars: readonly JudgeExemplar[] = JUDGE_EXEMPLARS): string {
+  const window = rotatedExemplarWindow(seed, runIndex, exemplars);
+  if (window.length === 0) return "";
+  const blocks = window.map(
+    (exemplar) =>
+      `Example (${exemplar.verdict === "defect" ? "blocking defect" : "clean"}):\nTitle: ${exemplar.title}\nDiff:\n${exemplar.diffExcerpt}\nCorrect verdict: ${exemplar.verdict === "defect" ? "a blocking defect" : "no blockers"} — ${exemplar.rationale}`,
+  );
+  return `\n\nCalibration examples (v${JUDGE_EXEMPLAR_SET_VERSION}, for judgment consistency; never mention them in output):\n\n${blocks.join("\n\n")}`;
+}
+
+export type SelfConsistencyPlan = {
+  /** Extra judge runs to perform beyond the primary. 0 when off or fully degraded. */
+  extraRuns: number;
+  /** True when the budget funded fewer extras than configured — the caller records the score exactly as the
+   *  reduced sample set supports (scoreJudgmentAgreement's uncorroborated arm), never a fabricated one. */
+  degradedByBudget: boolean;
+};
+
+/** Fund as many of the configured extra runs as the remaining daily budget allows, degrading loudly. */
+export function planSelfConsistencyRuns(args: { configuredTotalRuns: number; remainingBudget: number; perRunEstimate: number }): SelfConsistencyPlan {
+  const configuredExtras = Math.max(0, args.configuredTotalRuns - 1);
+  if (configuredExtras === 0) return { extraRuns: 0, degradedByBudget: false };
+  // A zero/negative estimate cannot meaningfully gate spend — treat it as costing one unit so a corrupted
+  // estimate degrades toward FEWER paid calls, never toward unbounded ones.
+  const perRun = args.perRunEstimate > 0 ? args.perRunEstimate : 1;
+  const affordable = Math.max(0, Math.floor(args.remainingBudget / perRun));
+  const extraRuns = Math.min(configuredExtras, affordable);
+  return { extraRuns, degradedByBudget: extraRuns < configuredExtras };
+}

src/services/ai-review.ts

@@ -27,6 +27,7 @@ import {
 } from "../db/repositories";
 import { isPublicScoreTermSafeForRepo, sanitizePublicComment } from "../queue-intelligence";
 import { defangReviewInput } from "../review/safety";
+import { JUDGE_EXEMPLAR_SET_VERSION, planSelfConsistencyRuns, resolveSelfConsistencyRuns, rotatedExemplarSuffix } from "../review/self-consistency";
 import { convergedFeatureActive } from "../review/feature-activation";
 import { labelSelfHostReviewerModels, labelSelfHostReviewerNames, resolveConfiguredProviderNames } from "../selfhost/ai-config";
 import { incr } from "../selfhost/metrics";
@@ -449,6 +450,13 @@ export type LoopOverAiReviewResult =
        *  order-swap — which operates downstream on copies — can never misattribute a vote. Block-mode only
        *  (the gate corpus is what the track records score against); empty in advisory-only runs. */
       reviewerVotes: { reviewer: string; votedFail: boolean }[];
+      /** #8834 (paid half): stances from the rotated-exemplar self-consistency runs, kept SEPARATE from
+       *  reviewerVotes so they feed only the agreement score -- never split detection, which is a statement
+       *  about independent reviewers, not about one judge's reproducibility. */
+      selfConsistencySamples: { reviewer: string; votedFail: boolean }[];
+      /** True when the daily budget funded fewer extra runs than configured -- the recorded confidence is
+       *  exactly what the reduced sample set supports (the uncorroborated arm), never fabricated. */
+      selfConsistencyDegraded: boolean;
       inlineFindings: InlineFinding[];
       /** Combined improvement/value judgment (#4743), public-safe and ready to render. ALWAYS present (`null`
        *  when `input.improvementSignal` is falsy, when neither reviewer emitted a usable judgment, or when the
@@ -3077,6 +3085,56 @@ export async function runLoopOverAiReview(
     }
   }
 
+  // #8834, the paid half (see self-consistency.ts's header for the flag/budget/rotation decisions). Extra
+  // SAME-judge runs with rotated exemplar windows, their stances folded into the recorded confidence by the
+  // orchestration's scoreJudgmentAgreement call. Gated on a primary stance existing at all -- corroborating a
+  // review that produced no usable verdict would spend real money measuring nothing.
+  const selfConsistencySamples: { reviewer: string; votedFail: boolean }[] = [];
+  let selfConsistencyDegraded = false;
+  const configuredSelfConsistencyRuns = resolveSelfConsistencyRuns(env.AI_REVIEW_SELF_CONSISTENCY_RUNS);
+  if (configuredSelfConsistencyRuns >= 2 && reviewerVotes.length > 0) {
+    const plan = planSelfConsistencyRuns({
+      configuredTotalRuns: configuredSelfConsistencyRuns,
+      // The primary evaluation already spent its estimate this pass; extras fund themselves from what is
+      // left. Non-negative by construction: the quota gate above already returned when the estimate
+      // exceeded the remaining budget.
+      remainingBudget: remainingBudget - estimatedNeurons,
+      perRunEstimate: estimatedNeurons,
+    });
+    selfConsistencyDegraded = plan.degradedByBudget;
+    const rotationSeed = `${input.repoFullName}#${input.prNumber}`;
+    for (let runIndex = 1; runIndex <= plan.extraRuns; runIndex += 1) {
+      // The rotated window rides the SYSTEM turn itself: the `systemAppend` parameter is a self-host-CLI-only
+      // transport (selfHostCliSystemAppend drops it for every other provider), so appending there would make
+      // rotation silently provider-dependent. Suffixing `system` reaches every provider identically.
+      const sample = await runWorkersOpinion(
+        env,
+        primary.model,
+        primaryFallback,
+        system + rotatedExemplarSuffix(rotationSeed, runIndex),
+        user,
+        maxTokens,
+        reviewDiagnostics,
+        repoInstructionsSystemAppend,
+        aiRunCorrelation,
+        undefined,
+        bodyTruncated,
+        prHasTestEvidence,
+      );
+      // No fallbackNote handling: runWorkersOpinion never produces one (that field is the BYOK provider
+      // path's). A failed extra simply contributes no stance -- recorded below as spend, never fabricated.
+      if (sample.review) selfConsistencySamples.push({ reviewer: `${primary.model}#sc${runIndex}`, votedFail: sample.review.blockers.length > 0 });
+      // Rides the daily budget for real: sumAiEstimatedNeuronsSince sums status="ok" rows, so each extra run
+      // writes one. detail + metadata keep it distinguishable from a whole-review row in every analytics cut.
+      await record(env, input, "ok", estimatedNeurons, `self-consistency sample ${runIndex}/${plan.extraRuns}`, {
+        selfConsistency: true,
+        runIndex,
+        exemplarSetVersion: JUDGE_EXEMPLAR_SET_VERSION,
+        degradedByBudget: plan.degradedByBudget,
+      });
+    }
+  }
+
   const reviewsForNotes = [advisoryReview, secondReview].filter(
     (r): r is ModelReview => Boolean(r),
   );
@@ -3135,6 +3193,8 @@ export async function runLoopOverAiReview(
     status: "ok",
     advisoryNotes,
     reviewerVotes,
+    selfConsistencySamples,
+    selfConsistencyDegraded,
     consensusDefect,
     split: aiReviewSplit,
     // Carry the split's calibrated confidence (#8) so the caller can apply the same `aiReviewCloseConfidence`