From ff262536689c32d40c23bd1365101d7ad8434ebc Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:37:28 -0700 Subject: [PATCH 1/2] feat(review): rotated-exemplar self-consistency sampling behind AI_REVIEW_SELF_CONSISTENCY_RUNS The paid half of #8834. The free half (judgment-agreement.ts) already scores inter-run agreement over whatever stances a review produces; this adds the deliberate extra stances: flag-gated re-runs of the SAME judge with deterministically rotated few-shot exemplar windows ('simulated annotators', Trust or Escalate, ICLR 2025), folded into the recorded confidence through the existing scoreJudgmentAgreement axis -- never into split detection. Decisions the free half's closing comment deferred, now made: - Flag: AI_REVIEW_SELF_CONSISTENCY_RUNS = TOTAL evaluations per review. Unset/0/garbage = OFF (today's behavior byte-identical); values clamp into {2,3} -- one run measures nothing, benefit saturates by three. - Budget: extras ride the daily neuron budget for real -- each records a status-ok ai_usage_events row (the same sum the quota gate reads) with selfConsistency metadata. An underfunded plan DEGRADES to fewer runs and the score records as uncorroborated; nothing is ever fabricated. - Rotation: window = f(repo#pr, runIndex) via FNV-1a, so replay can reconstruct what each run saw and re-reviews sample the same rotation instead of measuring prompt-shuffle variance. Exemplars are synthetic, versioned (JUDGE_EXEMPLAR_SET_VERSION), balanced 3 defect / 3 clean. - The rotated window rides the SYSTEM turn itself: the systemAppend parameter is a self-host-CLI-only transport (selfHostCliSystemAppend drops it for every other provider), so appending there would have made rotation silently provider-dependent. - Extras are skipped when the primary pass produced no usable stance: corroborating a non-verdict is pure spend. A failed extra run records its spend but contributes no stance; a dissenting one lands measurably in decision_records' aiAgreement. Closes #8834 --- src/env.d.ts | 6 + src/queue/ai-review-orchestration.ts | 6 +- src/review/self-consistency.ts | 158 ++++++++++++ src/services/ai-review.ts | 60 +++++ test/unit/self-consistency.test.ts | 369 +++++++++++++++++++++++++++ 5 files changed, 598 insertions(+), 1 deletion(-) create mode 100644 src/review/self-consistency.ts create mode 100644 test/unit/self-consistency.test.ts diff --git a/src/env.d.ts b/src/env.d.ts index 14e3d62871..94148c0c3f 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -106,6 +106,12 @@ declare global { * ai-review/ai-slop/ai-summaries/planner). The unit name is a Workers-AI holdover; it's applied as a * provider-agnostic heuristic budget regardless of which configured provider actually serves the request. */ AI_DAILY_NEURON_BUDGET?: string; + /** #8834: TOTAL judge evaluations per AI review for rotated-exemplar self-consistency. Unset/0 = off + * (single evaluation, today's behavior); 2 or 3 = extra same-judge runs with rotated few-shot exemplars, + * their stances folded into the recorded per-decision confidence. Each extra run is a real AI charge + * riding the daily neuron budget; an exhausted budget degrades to fewer runs and a lower recorded + * confidence, never a fabricated one. */ + AI_REVIEW_SELF_CONSISTENCY_RUNS?: string; /** Per-repository/day cap for maintainer-paid BYOK AI review provider calls. */ AI_BYOK_DAILY_REPO_LIMIT?: string; /** #9061: per-repository/day cap on AI calls for the FREE/default chain — the path the self-host actually diff --git a/src/queue/ai-review-orchestration.ts b/src/queue/ai-review-orchestration.ts index 346df25ea5..5d2d2dd97c 100644 --- a/src/queue/ai-review-orchestration.ts +++ b/src/queue/ai-review-orchestration.ts @@ -949,7 +949,11 @@ export async function runAiReviewForAdvisory( // #8834: inter-run agreement over the stances this review ALREADY produced (#8229's reviewerVotes) — // zero additional AI spend. Computed once and attached to whichever AI-judgment finding is built below, // so the decision record carries a per-decision confidence signal for the calibration set (#8835). - const aiJudgmentAgreement = (verbalizedConfidence: number) => scoreJudgmentAgreement(result.reviewerVotes, verbalizedConfidence); + // #8834 both halves meet here: the free half scores whatever stances exist; the paid half (flag-gated + // rotated-exemplar runs) contributes extra SAME-judge stances through selfConsistencySamples -- folded + // into the agreement axis only, never into split detection, which reads reviewerVotes alone. + const aiJudgmentAgreement = (verbalizedConfidence: number) => + scoreJudgmentAgreement([...result.reviewerVotes, ...result.selfConsistencySamples], verbalizedConfidence); if (result.consensusDefect) { findings.push({ code: "ai_consensus_defect", diff --git a/src/review/self-consistency.ts b/src/review/self-consistency.ts new file mode 100644 index 0000000000..6653694005 --- /dev/null +++ b/src/review/self-consistency.ts @@ -0,0 +1,158 @@ +// Rotated-exemplar self-consistency sampling (#8834, epic #8828 Phase 3) — the PAID half of the per-decision +// confidence signal, deliberately split from the free half that already shipped. +// +// judgment-agreement.ts scores inter-run agreement over whatever stances exist and costs nothing; its own +// closing comment records that the extra-sampling half — re-running the SAME judge with rotated few-shot +// exemplars ("simulated annotators", Trust or Escalate, ICLR 2025) — needed a budget decision and a flag +// defaulting OFF before it could ship, because every extra run is a real per-review AI charge. This module is +// that half, and the decisions are: +// +// FLAG: `AI_REVIEW_SELF_CONSISTENCY_RUNS` — the TOTAL number of judge evaluations per review. Unset/0/ +// unparseable ⇒ OFF (today's behavior, byte-identical). 2 or 3 ⇒ on; anything else clamps into that range, +// because one total run measures nothing and the literature's benefit saturates by three. +// +// BUDGET: extra runs ride the existing daily-neuron accounting. When the remaining budget cannot fund the +// configured extras, the plan DEGRADES — fewer or zero extra runs — and the recorded confidence degrades +// with it through scoreJudgmentAgreement's own uncorroborated arm. It never fabricates a score: a decision +// that could not afford corroboration is recorded as uncorroborated, which is exactly what it is. +// +// ROTATION: each extra run appends a DIFFERENT exemplar window to the system prompt, deterministically +// derived from the target seed + run index. Determinism matters twice over: replay (#9028) can reconstruct +// which exemplars a run saw, and two passes over the same target sample the same rotation rather than +// secretly measuring prompt-shuffle variance. +// +// The exemplars live here as versioned code constants rather than in test/golden-corpus/: the Worker bundle +// cannot read test fixtures at runtime, and the corpus file carries gate-pipeline fixtures (findings/policy), +// not judge-facing PR content. Same versioning discipline, different artifact. They are SYNTHETIC by +// construction — never drawn from real reviewed PRs, so no contributor content can leak into every future +// prompt on the instance. + +/** Bump when the exemplar set changes — it shifts every self-consistency prompt, so promptDigest moves too. */ +export const JUDGE_EXEMPLAR_SET_VERSION = 1; + +export type JudgeExemplar = { + id: string; + title: string; + diffExcerpt: string; + verdict: "defect" | "clean"; + rationale: string; +}; + +/** Six synthetic exemplars, balanced 3/3, each small enough that a 3-exemplar window adds well under a KB. */ +export const JUDGE_EXEMPLARS: readonly JudgeExemplar[] = [ + { + id: "null-deref-on-empty", + title: "Speed up lookup by skipping the guard", + diffExcerpt: '- if (!items || items.length === 0) return null;\n const first = items[0].id;', + verdict: "defect", + rationale: "Removes the empty-input guard while the very next line indexes the array: a crash on every empty call.", + }, + { + id: "rename-only-refactor", + title: "Rename fetchRows to loadRows for clarity", + diffExcerpt: '- const rows = await fetchRows(db);\n+ const rows = await loadRows(db);', + verdict: "clean", + rationale: "A pure rename with the definition and every call site moved together; behavior is byte-identical.", + }, + { + id: "swallowed-error-branch", + title: "Stop noisy logging in the sync path", + diffExcerpt: '- } catch (error) {\n- throw error;\n+ } catch {\n+ /* ignore */', + verdict: "defect", + rationale: "Converts a propagated failure into silent success; callers now proceed on corrupt state with no signal.", + }, + { + id: "test-added-for-fix", + title: "Add regression test for the off-by-one fix", + diffExcerpt: '+ it("includes the final page", () => {\n+ expect(paginate(21, 10).pages).toBe(3);\n+ });', + verdict: "clean", + rationale: "Adds a test pinning already-merged behavior; touches no production code path.", + }, + { + id: "boundary-flip", + title: "Simplify the retry condition", + diffExcerpt: '- while (attempt <= maxRetries) {\n+ while (attempt < maxRetries) {', + verdict: "defect", + rationale: "Silently drops the final permitted attempt; every caller now retries one time fewer than configured.", + }, + { + id: "doc-comment-fix", + title: "Correct the stale doc comment on parseWindow", + diffExcerpt: '-// window is inclusive of both ends\n+// window is inclusive of start, exclusive of end', + verdict: "clean", + rationale: "Documentation-only change aligning the comment with the long-standing behavior; no code touched.", + }, +]; + +/** How many exemplars each rotated window carries. Three of six gives six distinct contiguous windows, so + * every (seed, runIndex) pair lands on a real rotation rather than a reshuffle of the same set. */ +export const EXEMPLAR_WINDOW_SIZE = 3; + +/** TOTAL evaluations per review (primary included). 0 = off. Explicit values clamp into {0, 2, 3}: one total + * run cannot measure agreement, and the literature's benefit saturates by three. */ +export function resolveSelfConsistencyRuns(raw: string | undefined): number { + const parsed = Number(raw); + if (!raw || !Number.isFinite(parsed) || parsed < 2) return 0; + return Math.min(3, Math.floor(parsed)); +} + +/** Deterministic, dependency-free string hash (FNV-1a 32-bit) — rotation must be stable across runtimes. */ +function fnv1a(text: string): number { + let hash = 0x811c9dc5; + for (let i = 0; i < text.length; i += 1) { + hash ^= text.charCodeAt(i); + hash = Math.imul(hash, 0x01000193) >>> 0; + } + return hash >>> 0; +} + +/** + * The exemplar window for one extra run, rotated deterministically by (seed, runIndex). + * + * `seed` is the review target's identity (repo#pr@sha), so the SAME target always samples the SAME rotation + * sequence — replayable, and immune to measuring shuffle variance — while different targets start at + * different offsets, so no single exemplar dominates the fleet's second opinions. + */ +export function rotatedExemplarWindow(seed: string, runIndex: number, exemplars: readonly JudgeExemplar[] = JUDGE_EXEMPLARS): JudgeExemplar[] { + if (exemplars.length === 0) return []; + const offset = (fnv1a(seed) + runIndex) % exemplars.length; + const window: JudgeExemplar[] = []; + for (let i = 0; i < Math.min(EXEMPLAR_WINDOW_SIZE, exemplars.length); i += 1) { + const exemplar = exemplars[(offset + i) % exemplars.length]; + /* v8 ignore next -- modular indexing into a non-empty array cannot miss; the guard satisfies noUncheckedIndexedAccess. */ + if (exemplar) window.push(exemplar); + } + return window; +} + +/** Render one rotated window as a system-prompt suffix, matching the calibration-suffix idiom the judge + * prompt already uses: labeled sections, no markdown the model could echo into findings. */ +export function rotatedExemplarSuffix(seed: string, runIndex: number, exemplars: readonly JudgeExemplar[] = JUDGE_EXEMPLARS): string { + const window = rotatedExemplarWindow(seed, runIndex, exemplars); + if (window.length === 0) return ""; + const blocks = window.map( + (exemplar) => + `Example (${exemplar.verdict === "defect" ? "blocking defect" : "clean"}):\nTitle: ${exemplar.title}\nDiff:\n${exemplar.diffExcerpt}\nCorrect verdict: ${exemplar.verdict === "defect" ? "a blocking defect" : "no blockers"} — ${exemplar.rationale}`, + ); + return `\n\nCalibration examples (v${JUDGE_EXEMPLAR_SET_VERSION}, for judgment consistency; never mention them in output):\n\n${blocks.join("\n\n")}`; +} + +export type SelfConsistencyPlan = { + /** Extra judge runs to perform beyond the primary. 0 when off or fully degraded. */ + extraRuns: number; + /** True when the budget funded fewer extras than configured — the caller records the score exactly as the + * reduced sample set supports (scoreJudgmentAgreement's uncorroborated arm), never a fabricated one. */ + degradedByBudget: boolean; +}; + +/** Fund as many of the configured extra runs as the remaining daily budget allows, degrading loudly. */ +export function planSelfConsistencyRuns(args: { configuredTotalRuns: number; remainingBudget: number; perRunEstimate: number }): SelfConsistencyPlan { + const configuredExtras = Math.max(0, args.configuredTotalRuns - 1); + if (configuredExtras === 0) return { extraRuns: 0, degradedByBudget: false }; + // A zero/negative estimate cannot meaningfully gate spend — treat it as costing one unit so a corrupted + // estimate degrades toward FEWER paid calls, never toward unbounded ones. + const perRun = args.perRunEstimate > 0 ? args.perRunEstimate : 1; + const affordable = Math.max(0, Math.floor(args.remainingBudget / perRun)); + const extraRuns = Math.min(configuredExtras, affordable); + return { extraRuns, degradedByBudget: extraRuns < configuredExtras }; +} diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 2555b28865..50fd3b7d88 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -27,6 +27,7 @@ import { } from "../db/repositories"; import { isPublicScoreTermSafeForRepo, sanitizePublicComment } from "../queue-intelligence"; import { defangReviewInput } from "../review/safety"; +import { JUDGE_EXEMPLAR_SET_VERSION, planSelfConsistencyRuns, resolveSelfConsistencyRuns, rotatedExemplarSuffix } from "../review/self-consistency"; import { convergedFeatureActive } from "../review/feature-activation"; import { labelSelfHostReviewerModels, labelSelfHostReviewerNames, resolveConfiguredProviderNames } from "../selfhost/ai-config"; import { incr } from "../selfhost/metrics"; @@ -449,6 +450,13 @@ export type LoopOverAiReviewResult = * order-swap — which operates downstream on copies — can never misattribute a vote. Block-mode only * (the gate corpus is what the track records score against); empty in advisory-only runs. */ reviewerVotes: { reviewer: string; votedFail: boolean }[]; + /** #8834 (paid half): stances from the rotated-exemplar self-consistency runs, kept SEPARATE from + * reviewerVotes so they feed only the agreement score -- never split detection, which is a statement + * about independent reviewers, not about one judge's reproducibility. */ + selfConsistencySamples: { reviewer: string; votedFail: boolean }[]; + /** True when the daily budget funded fewer extra runs than configured -- the recorded confidence is + * exactly what the reduced sample set supports (the uncorroborated arm), never fabricated. */ + selfConsistencyDegraded: boolean; inlineFindings: InlineFinding[]; /** Combined improvement/value judgment (#4743), public-safe and ready to render. ALWAYS present (`null` * when `input.improvementSignal` is falsy, when neither reviewer emitted a usable judgment, or when the @@ -3077,6 +3085,56 @@ export async function runLoopOverAiReview( } } + // #8834, the paid half (see self-consistency.ts's header for the flag/budget/rotation decisions). Extra + // SAME-judge runs with rotated exemplar windows, their stances folded into the recorded confidence by the + // orchestration's scoreJudgmentAgreement call. Gated on a primary stance existing at all -- corroborating a + // review that produced no usable verdict would spend real money measuring nothing. + const selfConsistencySamples: { reviewer: string; votedFail: boolean }[] = []; + let selfConsistencyDegraded = false; + const configuredSelfConsistencyRuns = resolveSelfConsistencyRuns(env.AI_REVIEW_SELF_CONSISTENCY_RUNS); + if (configuredSelfConsistencyRuns >= 2 && reviewerVotes.length > 0) { + const plan = planSelfConsistencyRuns({ + configuredTotalRuns: configuredSelfConsistencyRuns, + // The primary evaluation already spent its estimate this pass; extras fund themselves from what is + // left. Non-negative by construction: the quota gate above already returned when the estimate + // exceeded the remaining budget. + remainingBudget: remainingBudget - estimatedNeurons, + perRunEstimate: estimatedNeurons, + }); + selfConsistencyDegraded = plan.degradedByBudget; + const rotationSeed = `${input.repoFullName}#${input.prNumber}`; + for (let runIndex = 1; runIndex <= plan.extraRuns; runIndex += 1) { + // The rotated window rides the SYSTEM turn itself: the `systemAppend` parameter is a self-host-CLI-only + // transport (selfHostCliSystemAppend drops it for every other provider), so appending there would make + // rotation silently provider-dependent. Suffixing `system` reaches every provider identically. + const sample = await runWorkersOpinion( + env, + primary.model, + primaryFallback, + system + rotatedExemplarSuffix(rotationSeed, runIndex), + user, + maxTokens, + reviewDiagnostics, + repoInstructionsSystemAppend, + aiRunCorrelation, + undefined, + bodyTruncated, + prHasTestEvidence, + ); + // No fallbackNote handling: runWorkersOpinion never produces one (that field is the BYOK provider + // path's). A failed extra simply contributes no stance -- recorded below as spend, never fabricated. + if (sample.review) selfConsistencySamples.push({ reviewer: `${primary.model}#sc${runIndex}`, votedFail: sample.review.blockers.length > 0 }); + // Rides the daily budget for real: sumAiEstimatedNeuronsSince sums status="ok" rows, so each extra run + // writes one. detail + metadata keep it distinguishable from a whole-review row in every analytics cut. + await record(env, input, "ok", estimatedNeurons, `self-consistency sample ${runIndex}/${plan.extraRuns}`, { + selfConsistency: true, + runIndex, + exemplarSetVersion: JUDGE_EXEMPLAR_SET_VERSION, + degradedByBudget: plan.degradedByBudget, + }); + } + } + const reviewsForNotes = [advisoryReview, secondReview].filter( (r): r is ModelReview => Boolean(r), ); @@ -3135,6 +3193,8 @@ export async function runLoopOverAiReview( status: "ok", advisoryNotes, reviewerVotes, + selfConsistencySamples, + selfConsistencyDegraded, consensusDefect, split: aiReviewSplit, // Carry the split's calibrated confidence (#8) so the caller can apply the same `aiReviewCloseConfidence` diff --git a/test/unit/self-consistency.test.ts b/test/unit/self-consistency.test.ts new file mode 100644 index 0000000000..b0a6142e35 --- /dev/null +++ b/test/unit/self-consistency.test.ts @@ -0,0 +1,369 @@ +import { describe, expect, it } from "vitest"; +import { + EXEMPLAR_WINDOW_SIZE, + JUDGE_EXEMPLARS, + planSelfConsistencyRuns, + resolveSelfConsistencyRuns, + rotatedExemplarSuffix, + rotatedExemplarWindow, +} from "../../src/review/self-consistency"; +import { scoreJudgmentAgreement } from "../../src/review/judgment-agreement"; +import { vi, afterEach, beforeEach } from "vitest"; +import { + upsertInstallation, + upsertOfficialMinerDetection, + upsertPullRequestFromGitHub, + upsertRepositoryFromGitHub, + upsertRepositorySettings, +} from "../../src/db/repositories"; +import { reReviewStoredPullRequest } from "../../src/queue/processors"; +import { upsertRepoFocusManifest } from "../../src/signals/focus-manifest-loader"; +import { normalizeRegistryPayload } from "../../src/registry/normalize"; +import { persistRegistrySnapshot } from "../../src/registry/sync"; +import { asCloudEnv, createTestEnv } from "../helpers/d1"; +import { generatePrivateKeyPem } from "../helpers/github-app-key"; + +// #8834, the paid half. The free half (judgment-agreement.ts) scores whatever stances exist; this half buys +// extra SAME-judge stances with rotated exemplar windows, flag-gated OFF, riding the daily neuron budget and +// degrading to fewer runs -- never a fabricated score -- when the budget cannot fund them. +describe("rotated-exemplar self-consistency (#8834)", () => { + describe("resolveSelfConsistencyRuns — the OFF-by-default flag", () => { + it("unset/empty/zero/garbage are all OFF — today's behavior stays byte-identical", () => { + for (const raw of [undefined, "", "0", "1", "nope", "-3"]) { + expect(resolveSelfConsistencyRuns(raw)).toBe(0); + } + }); + + it("clamps into {2,3}: one total run cannot measure agreement, and the benefit saturates by three", () => { + expect(resolveSelfConsistencyRuns("2")).toBe(2); + expect(resolveSelfConsistencyRuns("3")).toBe(3); + expect(resolveSelfConsistencyRuns("7")).toBe(3); + expect(resolveSelfConsistencyRuns("2.9")).toBe(2); // floor, never round up into extra spend + }); + }); + + describe("rotatedExemplarWindow — deterministic rotation, real rotation", () => { + it("PROPERTY: the same (seed, runIndex) always yields the same window — replay can reconstruct what a run saw", () => { + for (let run = 1; run <= 3; run += 1) { + expect(rotatedExemplarWindow("o/r#7", run)).toEqual(rotatedExemplarWindow("o/r#7", run)); + } + }); + + it("PROPERTY: consecutive run indices yield DIFFERENT windows — otherwise the runs measure sampling noise, not exemplar-rotated consistency", () => { + const first = rotatedExemplarWindow("o/r#7", 1).map((e) => e.id); + const second = rotatedExemplarWindow("o/r#7", 2).map((e) => e.id); + expect(first).not.toEqual(second); + }); + + it("PROPERTY: different targets start at different offsets, so no exemplar dominates the fleet's second opinions", () => { + const seeds = ["a/a#1", "b/b#2", "c/c#3", "d/d#4", "e/e#5", "f/f#6", "g/g#7"]; + const firstIds = new Set(seeds.map((seed) => rotatedExemplarWindow(seed, 1)[0]?.id)); + expect(firstIds.size).toBeGreaterThan(1); + }); + + it("windows are the declared size, drawn from the declared set, and survive an empty set without crashing", () => { + const window = rotatedExemplarWindow("o/r#7", 1); + expect(window).toHaveLength(EXEMPLAR_WINDOW_SIZE); + for (const exemplar of window) expect(JUDGE_EXEMPLARS.map((e) => e.id)).toContain(exemplar.id); + expect(rotatedExemplarWindow("o/r#7", 1, [])).toEqual([]); + }); + + it("the shipped exemplar set is balanced — an unbalanced set would bias every second opinion toward one verdict", () => { + const defects = JUDGE_EXEMPLARS.filter((e) => e.verdict === "defect").length; + expect(defects).toBe(JUDGE_EXEMPLARS.length - defects); + }); + }); + + describe("rotatedExemplarSuffix", () => { + it("renders the window with both verdict classes representable, and tells the model never to echo it", () => { + const suffix = rotatedExemplarSuffix("o/r#7", 1); + expect(suffix).toContain("Calibration examples"); + expect(suffix).toContain("never mention them in output"); + // Across the full rotation both verdict phrasings appear (balance test above guarantees the set has both). + const all = [1, 2, 3, 4, 5, 6].map((k) => rotatedExemplarSuffix("o/r#7", k)).join(""); + expect(all).toContain("a blocking defect"); + expect(all).toContain("no blockers"); + }); + + it("an empty exemplar set renders an empty suffix — the prompt is unchanged, not decorated with a header", () => { + expect(rotatedExemplarSuffix("o/r#7", 1, [])).toBe(""); + }); + }); + + describe("planSelfConsistencyRuns — budget honesty", () => { + it("funds the configured extras when the budget allows", () => { + expect(planSelfConsistencyRuns({ configuredTotalRuns: 3, remainingBudget: 1000, perRunEstimate: 100 })).toEqual({ + extraRuns: 2, + degradedByBudget: false, + }); + }); + + it("INVARIANT: degrades to FEWER runs when the budget cannot fund them — and says so, never fabricating", () => { + expect(planSelfConsistencyRuns({ configuredTotalRuns: 3, remainingBudget: 150, perRunEstimate: 100 })).toEqual({ + extraRuns: 1, + degradedByBudget: true, + }); + expect(planSelfConsistencyRuns({ configuredTotalRuns: 2, remainingBudget: 0, perRunEstimate: 100 })).toEqual({ + extraRuns: 0, + degradedByBudget: true, + }); + }); + + it("INVARIANT: a zero/negative per-run estimate degrades toward FEWER paid calls, never unbounded ones", () => { + const plan = planSelfConsistencyRuns({ configuredTotalRuns: 3, remainingBudget: 5, perRunEstimate: 0 }); + expect(plan.extraRuns).toBeLessThanOrEqual(2); + }); + + it("off (or one total run) plans nothing and reports no degradation", () => { + expect(planSelfConsistencyRuns({ configuredTotalRuns: 0, remainingBudget: 1000, perRunEstimate: 1 })).toEqual({ + extraRuns: 0, + degradedByBudget: false, + }); + expect(planSelfConsistencyRuns({ configuredTotalRuns: 1, remainingBudget: 1000, perRunEstimate: 1 })).toEqual({ + extraRuns: 0, + degradedByBudget: false, + }); + }); + }); + + describe("the confidence contract with judgment-agreement (the disagreement→hold path)", () => { + it("a disagreeing extra sample DROPS the folded confidence — which is what routes the decision to hold via the existing floor", () => { + const unanimous = scoreJudgmentAgreement( + [ + { votedFail: true }, + { votedFail: true }, // agreeing self-consistency sample + ], + 0.9, + ); + const split = scoreJudgmentAgreement( + [ + { votedFail: true }, + { votedFail: false }, // disagreeing self-consistency sample + ], + 0.9, + ); + expect(unanimous.confidence).toBeGreaterThan(split.confidence); + expect(split.agreement).toBe(0.5); + }); + + it("a fully-degraded plan leaves a single stance, which scores as UNCORROBORATED — the lower recorded confidence the issue requires", () => { + const degraded = scoreJudgmentAgreement([{ votedFail: true }], 0.9); + expect(degraded.uncorroborated).toBe(true); + expect(degraded.confidence).toBeLessThan(0.9); + }); + }); + + // End-to-end: the flag actually buys extra judge calls, accounts every one of them against the daily + // budget, and the whole pass still completes. Same minimal seed as the sibling review-flow suites (the + // per-file fixture-duplication convention). + describe("end-to-end with the flag on", () => { + beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date("2026-07-28T00:00:00.000Z")); + }); + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + async function seed(env: Env) { + await persistRegistrySnapshot( + asCloudEnv(env), + normalizeRegistryPayload( + { "JSONbored/gittensory": { emission_share: 0.01, issue_discovery_share: 0 } }, + { kind: "raw-github", url: "https://example.test" }, + "2026-05-23T00:00:00.000Z", + ), + ); + await upsertInstallation(env, { action: "created", installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" }, target_type: "User", repository_selection: "selected", permissions: {}, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, 123); + await upsertRepositorySettings(env, { repoFullName: "JSONbored/gittensory", autoLabelEnabled: false, gatePack: "oss-anti-slop", autonomy: { label: "auto" } }); + await upsertRepoFocusManifest(env, "JSONbored/gittensory", { + settings: { commentMode: "all_prs", publicSurface: "comment_only", checkRunMode: "off", reviewCheckMode: "required", aiReviewMode: "block" }, + }); + await upsertOfficialMinerDetection(env, "contributor", { status: "confirmed", snapshot: { + source: "gittensor_api" as const, githubId: "123", githubUsername: "contributor", isEligible: true, credibility: 1, + eligibleRepoCount: 1, issueDiscoveryScore: 0, issueTokenScore: 0, issueCredibility: 1, isIssueEligible: false, + issueEligibleRepoCount: 0, alphaPerDay: 0, taoPerDay: 0, usdPerDay: 0, + totals: { pullRequests: 3, mergedPullRequests: 2, openPullRequests: 1, closedPullRequests: 0, openIssues: 0, closedIssues: 0, solvedIssues: 0, validSolvedIssues: 0 }, + repositories: [], pullRequests: [], issueLabels: [], + } }, 60_000); + await upsertPullRequestFromGitHub(env, "JSONbored/gittensory", { + number: 900, title: "Self-consistency PR", state: "open", user: { login: "contributor" }, + author_association: "CONTRIBUTOR", head: { sha: "shaSc" }, base: { ref: "main" }, labels: [], body: "Closes #1", + }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "fake-installation-token" }); + if (url.includes("/pulls/900/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = true;" }]); + if (url.endsWith("/pulls/900")) return Response.json({ number: 900, title: "Self-consistency PR", state: "open", user: { login: "contributor" }, head: { sha: "shaSc" }, labels: [], body: "Closes #1", mergeable_state: "clean" }); + if (url.includes("/commits/shaSc/check-runs")) return Response.json({ total_count: 1, check_runs: [{ name: "test", status: "completed", conclusion: "success", app: { slug: "github-actions" } }] }); + if (url.includes("/commits/shaSc/status")) return Response.json({ state: "success", statuses: [] }); + if (url.includes("/issues/1")) return Response.json({ number: 1, title: "Issue", state: "open", labels: [], user: { login: "reporter" } }); + if (url.includes("/issues/900/comments") && (method === "POST" || method === "PATCH")) return Response.json({ id: 1 }, { status: 201 }); + if (url.includes("/issues/900/comments")) return Response.json([]); + if (url.includes("/branches/")) return Response.json({ protected: false, protection: { required_status_checks: { contexts: [] } } }); + return Response.json({}); + }); + } + + it("REGRESSION: flag=3 spends exactly two extra judge calls, each with a DIFFERENT rotated exemplar window, all budget-accounted", async () => { + const prompts: string[] = []; + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async (_model: string, options: { messages?: Array<{ role: string; content: string }> }) => { + prompts.push(options.messages?.find((m) => m.role === "system")?.content ?? ""); + return { response: JSON.stringify({ assessment: "Fine.", blockers: [], nits: [], suggestions: [] }) }; + } } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + AI_REVIEW_SELF_CONSISTENCY_RUNS: "3", + }); + await seed(env); + + await reReviewStoredPullRequest(env, "sc-900", 123, "JSONbored/gittensory", 900); + + const judgeCalls = prompts.filter((prompt) => prompt.length > 0); + expect(judgeCalls.length).toBeGreaterThanOrEqual(3); // 1 primary + 2 self-consistency extras + const exemplarPrompts = judgeCalls.filter((prompt) => prompt.includes("Calibration examples")); + expect(exemplarPrompts).toHaveLength(2); // the extras carry the rotated windows; the primary never does + expect(exemplarPrompts[0]).not.toBe(exemplarPrompts[1]); // rotation, not repetition + const usage = await env.DB.prepare( + "select count(*) as n from ai_usage_events where status = 'ok' and json_extract(metadata_json, '$.selfConsistency') = 1", + ).first<{ n: number }>(); + expect(usage?.n).toBe(2); // every extra run rides the SAME budget sum the quota gate reads + }); + + it("INVARIANT: flag off (default) spends nothing extra and adds no self-consistency usage rows", async () => { + let aiCalls = 0; + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => { aiCalls += 1; return { response: JSON.stringify({ assessment: "Fine.", blockers: [], nits: [], suggestions: [] }) }; } } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + await seed(env); + + await reReviewStoredPullRequest(env, "sc-off-900", 123, "JSONbored/gittensory", 900); + + const usage = await env.DB.prepare( + "select count(*) as n from ai_usage_events where json_extract(metadata_json, '$.selfConsistency') = 1", + ).first<{ n: number }>(); + expect(usage?.n).toBe(0); + expect(aiCalls).toBeGreaterThan(0); // the pass itself still reviewed + }); + + it("INVARIANT: an exhausted budget degrades to zero extras — the pass completes, nothing is fabricated", async () => { + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => ({ response: JSON.stringify({ assessment: "Fine.", blockers: [], nits: [], suggestions: [] }) }) } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + // Enough for the primary evaluation (estimated ~3.5k neurons for this fixture), but the remainder + // cannot fund a single extra run at the same per-run estimate. + AI_DAILY_NEURON_BUDGET: "5000", + AI_REVIEW_SELF_CONSISTENCY_RUNS: "3", + }); + await seed(env); + + await reReviewStoredPullRequest(env, "sc-budget-900", 123, "JSONbored/gittensory", 900); + + const extras = await env.DB.prepare( + "select count(*) as n from ai_usage_events where json_extract(metadata_json, '$.selfConsistency') = 1", + ).first<{ n: number }>(); + expect(extras?.n).toBe(0); // degraded, not fabricated + const reviewed = await env.DB.prepare("select count(*) as n from ai_usage_events where status = 'ok'").first<{ n: number }>(); + expect(reviewed?.n).toBeGreaterThan(0); // the primary review itself still happened and was recorded + }); + + it("REGRESSION: a dissenting extra sample lands in the persisted decision record's agreement", async () => { + // Primaries flag a consensus defect; extra run 1 dissents (clean), extra run 2 agrees (blockers). + // Stances fold as [fail, fail, clean, fail] -> agreement 0.75 on the ai_consensus_defect finding, + // which persistDecisionRecord carries into decision_records.ai_agreement. + let exemplarCalls = 0; + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async (_model: string, options: { messages?: Array<{ role: string; content: string }> }) => { + const systemContent = options.messages?.find((m) => m.role === "system")?.content ?? ""; + if (systemContent.includes("Calibration examples")) { + exemplarCalls += 1; + return { response: JSON.stringify(exemplarCalls === 1 + ? { assessment: "Fine.", blockers: [], nits: [], suggestions: [] } + : { assessment: "Broken.", blockers: ["Null deref: crashes on empty input"], nits: [], suggestions: [] }) }; + } + return { response: JSON.stringify({ assessment: "Broken.", blockers: ["Null deref: crashes on empty input"], nits: [], suggestions: [] }) }; + } } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + AI_REVIEW_SELF_CONSISTENCY_RUNS: "3", + }); + await seed(env); + + await reReviewStoredPullRequest(env, "sc-dissent-900", 123, "JSONbored/gittensory", 900); + + expect(exemplarCalls).toBe(2); + const row = await env.DB.prepare( + "select record_json from decision_records order by created_at desc limit 1", + ).first<{ record_json: string }>(); + const record = JSON.parse(row?.record_json ?? "{}") as { aiAgreement: { agreement: number; sampleCount: number } | null }; + expect(record.aiAgreement?.agreement).toBe(0.75); // 3 of 4 stances failed -- the dissent is measured, not discarded + expect(record.aiAgreement?.sampleCount).toBe(4); // two primaries + two extra samples + }); + + it("INVARIANT: an extra run that fails outright records its spend but fabricates no stance", async () => { + // Every exemplar-suffixed call throws (all attempts, both models): the sample yields no review, so no + // stance folds -- but the usage row is still written, because the calls were genuinely spent. + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async (_model: string, options: { messages?: Array<{ role: string; content: string }> }) => { + const systemContent = options.messages?.find((m) => m.role === "system")?.content ?? ""; + if (systemContent.includes("Calibration examples")) throw new Error("provider down"); + return { response: JSON.stringify({ assessment: "Broken.", blockers: ["Null deref: crashes on empty input"], nits: [], suggestions: [] }) }; + } } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + AI_REVIEW_SELF_CONSISTENCY_RUNS: "2", + }); + await seed(env); + + await reReviewStoredPullRequest(env, "sc-fail-900", 123, "JSONbored/gittensory", 900); + + const usage = await env.DB.prepare( + "select count(*) as n from ai_usage_events where json_extract(metadata_json, '$.selfConsistency') = 1", + ).first<{ n: number }>(); + expect(usage?.n).toBe(1); // the spend is accounted + const row = await env.DB.prepare( + "select record_json from decision_records order by created_at desc limit 1", + ).first<{ record_json: string }>(); + const record = JSON.parse(row?.record_json ?? "{}") as { aiAgreement: { agreement: number; sampleCount: number } | null }; + // Both primaries failed the PR unanimously and the broken extra contributed NOTHING -- had a stance + // been fabricated from the failed run, the sample count would be 3 and agreement below 1. + expect(record.aiAgreement?.sampleCount).toBe(2); + expect(record.aiAgreement?.agreement).toBe(1); + }); + + it("INVARIANT: flag on but no usable primary stance spends nothing -- corroborating a non-verdict is pure waste", async () => { + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => ({ response: "I am not JSON at all" }) } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + AI_REVIEW_SELF_CONSISTENCY_RUNS: "3", + }); + await seed(env); + + await reReviewStoredPullRequest(env, "sc-novote-900", 123, "JSONbored/gittensory", 900); + + const usage = await env.DB.prepare( + "select count(*) as n from ai_usage_events where json_extract(metadata_json, '$.selfConsistency') = 1", + ).first<{ n: number }>(); + expect(usage?.n).toBe(0); + }); + }); +}); From 920358a172a089a6ca67668549885f2908a4827b Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:47:52 -0700 Subject: [PATCH 2/2] chore(selfhost): regenerate env reference for AI_REVIEW_SELF_CONSISTENCY_RUNS --- apps/loopover-ui/src/lib/selfhost-env-reference.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/loopover-ui/src/lib/selfhost-env-reference.ts b/apps/loopover-ui/src/lib/selfhost-env-reference.ts index 088ec6bd88..c3d6d1f589 100644 --- a/apps/loopover-ui/src/lib/selfhost-env-reference.ts +++ b/apps/loopover-ui/src/lib/selfhost-env-reference.ts @@ -81,6 +81,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "AI_REVIEW_PLAN", firstReference: "src/services/ai-review.ts", }, + { + name: "AI_REVIEW_SELF_CONSISTENCY_RUNS", + firstReference: "src/services/ai-review.ts", + }, { name: "AI_SUMMARIES_ENABLED", firstReference: "src/services/ai-review.ts", @@ -705,6 +709,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `AI_PROVIDER` | `src/selfhost/ai-config.ts` |", "| `AI_PUBLIC_COMMENTS_ENABLED` | `src/services/ai-review.ts` |", "| `AI_REVIEW_PLAN` | `src/services/ai-review.ts` |", + "| `AI_REVIEW_SELF_CONSISTENCY_RUNS` | `src/services/ai-review.ts` |", "| `AI_SUMMARIES_ENABLED` | `src/services/ai-review.ts` |", "| `AI_VISION` | `src/queue/processors.ts` |", "| `AI_VISION_API_KEY` | `src/server.ts` |",