diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json
index 3f55e9a5e..bfc3bf69d 100644
--- a/apps/loopover-ui/public/openapi.json
+++ b/apps/loopover-ui/public/openapi.json
@@ -567,7 +567,8 @@
"type": "number"
},
"gamingFlagsCaught": {
- "type": "number"
+ "type": "number",
+ "nullable": true
},
"accuracyCiPct": {
"type": "object",
@@ -645,18 +646,23 @@
"lambda": {
"type": "number"
},
- "coveragePct": {
+ "n": {
"type": "number"
},
- "n": {
+ "aiJudgedCoveragePct": {
"type": "number"
+ },
+ "backfilledPct": {
+ "type": "number",
+ "nullable": true
}
},
"required": [
"alpha",
"lambda",
- "coveragePct",
- "n"
+ "aiJudgedCoveragePct",
+ "n",
+ "backfilledPct"
]
},
"merge": {
@@ -669,18 +675,23 @@
"lambda": {
"type": "number"
},
- "coveragePct": {
+ "n": {
"type": "number"
},
- "n": {
+ "aiJudgedCoveragePct": {
"type": "number"
+ },
+ "backfilledPct": {
+ "type": "number",
+ "nullable": true
}
},
"required": [
"alpha",
"lambda",
- "coveragePct",
- "n"
+ "aiJudgedCoveragePct",
+ "n",
+ "backfilledPct"
]
}
},
diff --git a/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx b/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx
index 577ce81f3..d6cfbc212 100644
--- a/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx
+++ b/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx
@@ -193,6 +193,20 @@ describe("FairnessReportPage (#fairness-analytics)", () => {
expect(screen.getByText("2 human-reversed, lifetime")).toBeTruthy();
});
+ it("#9068: renders the insufficient-instances state (not a fabricated zero) when gamingFlagsCaught is null", async () => {
+ apiFetch.mockResolvedValue({
+ ok: true,
+ data: { ...FIXTURE, fleetAccuracy: { ...FIXTURE.fleetAccuracy, gamingFlagsCaught: null } },
+ status: 200,
+ durationMs: 10,
+ });
+ renderWithClient();
+ await waitFor(() => expect(screen.getByText("Anti-gaming flags caught")).toBeTruthy());
+ const gamingCard = screen.getByText("Anti-gaming flags caught").closest("div")!.parentElement!;
+ expect(gamingCard.textContent).toContain("—");
+ expect(gamingCard.textContent).toContain("not enough registered instances");
+ });
+
it("REGRESSION: does not crash when the API response predates the fleetAccuracy field (old backend/new frontend deployment skew)", async () => {
const { fleetAccuracy: _omitted, ...payloadWithoutFleetAccuracy } = FIXTURE;
apiFetch.mockResolvedValue({
diff --git a/apps/loopover-ui/src/components/site/fairness-report-page.tsx b/apps/loopover-ui/src/components/site/fairness-report-page.tsx
index 1cf46e61b..cebc2164c 100644
--- a/apps/loopover-ui/src/components/site/fairness-report-page.tsx
+++ b/apps/loopover-ui/src/components/site/fairness-report-page.tsx
@@ -123,11 +123,16 @@ export function FairnessReportPage() {
Anti-gaming flags caught
- {data.fleetAccuracy ? intFmt.format(data.fleetAccuracy.gamingFlagsCaught) : "—"}
+ {data.fleetAccuracy?.gamingFlagsCaught != null
+ ? intFmt.format(data.fleetAccuracy.gamingFlagsCaught)
+ : "—"}
- self-hosted instances flagged for mass-submitting easy PRs to inflate their own
- precision
+ {/* #9068: null (not 0) below the fleet's own eligibility floor -- a structural zero must
+ never read as "checked, found none". */}
+ {data.fleetAccuracy?.gamingFlagsCaught != null
+ ? "self-hosted instances flagged for mass-submitting easy PRs to inflate their own precision"
+ : "not enough registered instances yet to compare for a gaming pattern"}
diff --git a/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts b/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts
index e383f428e..dbf371f82 100644
--- a/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts
+++ b/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts
@@ -39,13 +39,32 @@ export type PublicStats = {
closePrecisionCiPct?: { lo: number; hi: number } | null;
coveragePct?: number | null;
decidedCount?: number;
+ /** #9050: `aiJudgedCoveragePct` (renamed from `coveragePct`) is the share of the arm's AI-JUDGED
+ * sub-population the guarantee covers, not a share of all decided signals (that's the sibling
+ * `coveragePct` above) -- render the population it's actually over, not a bare percentage.
+ * `backfilledPct` is null when the stored calibration predates that field. */
guaranteed?: {
- close: { alpha: number; lambda: number; coveragePct: number; n: number } | null;
- merge: { alpha: number; lambda: number; coveragePct: number; n: number } | null;
+ close: {
+ alpha: number;
+ lambda: number;
+ aiJudgedCoveragePct: number;
+ n: number;
+ backfilledPct: number | null;
+ } | null;
+ merge: {
+ alpha: number;
+ lambda: number;
+ aiJudgedCoveragePct: number;
+ n: number;
+ backfilledPct: number | null;
+ } | null;
};
instanceCount: number;
windowDays: number;
- gamingFlagsCaught: number;
+ /** #9068: null (not 0) when the fleet has fewer than GAMING_MIN_ELIGIBLE eligible instances -- the
+ * anti-farming detector cannot run below that floor, so a structural zero must never render as "checked,
+ * found none". */
+ gamingFlagsCaught: number | null;
};
/** Trailing weekly history of totals.accuracyPct's SAME formula (#4447). */
accuracyTrend: Array<{
diff --git a/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx b/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx
index c11187ddc..c15624e1d 100644
--- a/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx
+++ b/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx
@@ -230,6 +230,41 @@ describe("ProofOfPowerStats", () => {
expect(screen.getAllByRole("img", { name: "Trend over the last 8 weeks" })).toHaveLength(3);
});
+ it("#9050: disambiguates the published guarantee's coverage as the AI-judged sub-population, with the backfilled-vs-live split", async () => {
+ apiFetch.mockResolvedValue({
+ ok: true,
+ status: 200,
+ durationMs: 1,
+ data: {
+ ...PAYLOAD,
+ fleetAccuracy: {
+ accuracyPct: 80,
+ coveragePct: 64.3,
+ instanceCount: 3,
+ windowDays: 90,
+ gamingFlagsCaught: null, // #9068: below the fleet's own eligibility floor
+ guaranteed: {
+ close: {
+ alpha: 0.05,
+ lambda: 0.9,
+ aiJudgedCoveragePct: 57.1,
+ n: 129,
+ backfilledPct: 98.7,
+ },
+ merge: null,
+ },
+ },
+ },
+ });
+ renderWithClient();
+ await screen.findByText("Decision accuracy");
+ expect(
+ screen.getByText(
+ "merge/close calls confirmed by outcome · at 64.3% coverage · closes ≥95% guaranteed at 57.1% coverage of AI-judged closes (98.7% backfilled) · 3 self-hosted instances",
+ ),
+ ).toBeTruthy();
+ });
+
it("settles the count-up on the real reviewed total (not stuck at 0 when rAF never fires)", async () => {
// Deterministic (#flake): force prefers-reduced-motion so useCountUp lands the final value synchronously on
// mount, instead of running the requestAnimationFrame tween. jsdom has no matchMedia, so the unfixed test took
diff --git a/apps/loopover-ui/src/components/site/proof-of-power-stats.tsx b/apps/loopover-ui/src/components/site/proof-of-power-stats.tsx
index 90e24051d..3415083c7 100644
--- a/apps/loopover-ui/src/components/site/proof-of-power-stats.tsx
+++ b/apps/loopover-ui/src/components/site/proof-of-power-stats.tsx
@@ -168,7 +168,13 @@ export function ProofOfPowerStats({ className }: { className?: string }) {
fleetEligible
? // #8829: a bare accuracy scalar at unstated coverage is gameable (holding more raises it), so
// the tile names the coverage it was earned at whenever the backend supplies it.
- `merge/close calls confirmed by outcome${data.fleetAccuracy.coveragePct != null ? ` · at ${data.fleetAccuracy.coveragePct}% coverage` : ""}${data.fleetAccuracy.guaranteed?.close ? ` · closes ≥${Math.round((1 - data.fleetAccuracy.guaranteed.close.alpha) * 1000) / 10}% guaranteed at ${data.fleetAccuracy.guaranteed.close.coveragePct}% coverage` : ""} · ${intFmt.format(data.fleetAccuracy.instanceCount)} self-hosted instance${data.fleetAccuracy.instanceCount === 1 ? "" : "s"}${data.fleetAccuracy.gamingFlagsCaught > 0 ? ` · ${intFmt.format(data.fleetAccuracy.gamingFlagsCaught)} gaming pattern${data.fleetAccuracy.gamingFlagsCaught === 1 ? "" : "s"} flagged` : ""}`
+ // #9050: the guarantee's own coverage is named "of AI-judged closes" -- it's the share of the
+ // AI-judgment sub-population the threshold covers, not a share of all closes (that reading
+ // sits confusingly next to the sibling coveragePct just before it, which IS over all decided
+ // signals) -- plus the backfilled-vs-live split behind the evidence, when supplied.
+ // #9068: gamingFlagsCaught is null (not 0) below the fleet's own eligibility floor, so the
+ // suffix is omitted entirely rather than rendering a structural zero as "checked, found none".
+ `merge/close calls confirmed by outcome${data.fleetAccuracy.coveragePct != null ? ` · at ${data.fleetAccuracy.coveragePct}% coverage` : ""}${data.fleetAccuracy.guaranteed?.close ? ` · closes ≥${Math.round((1 - data.fleetAccuracy.guaranteed.close.alpha) * 1000) / 10}% guaranteed at ${data.fleetAccuracy.guaranteed.close.aiJudgedCoveragePct}% coverage of AI-judged closes${data.fleetAccuracy.guaranteed.close.backfilledPct != null ? ` (${data.fleetAccuracy.guaranteed.close.backfilledPct}% backfilled)` : ""}` : ""} · ${intFmt.format(data.fleetAccuracy.instanceCount)} self-hosted instance${data.fleetAccuracy.instanceCount === 1 ? "" : "s"}${data.fleetAccuracy.gamingFlagsCaught != null && data.fleetAccuracy.gamingFlagsCaught > 0 ? ` · ${intFmt.format(data.fleetAccuracy.gamingFlagsCaught)} gaming pattern${data.fleetAccuracy.gamingFlagsCaught === 1 ? "" : "s"} flagged` : ""}`
: totals.reversed > 0
? `${intFmt.format(totals.reversed)} human-reversed`
: "reversal-grounded"
diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts
index 16ca01b59..d222f9873 100644
--- a/src/openapi/schemas.ts
+++ b/src/openapi/schemas.ts
@@ -141,10 +141,13 @@ export const PublicStatsSchema = z
closePrecisionCiPct: z.object({ lo: z.number(), hi: z.number() }).nullable(),
coveragePct: z.number().nullable(),
decidedCount: z.number(),
- guaranteed: z.object({ close: z.object({ alpha: z.number(), lambda: z.number(), coveragePct: z.number(), n: z.number() }).nullable(), merge: z.object({ alpha: z.number(), lambda: z.number(), coveragePct: z.number(), n: z.number() }).nullable() }),
+ guaranteed: z.object({
+ close: z.object({ alpha: z.number(), lambda: z.number(), aiJudgedCoveragePct: z.number(), n: z.number(), backfilledPct: z.number().nullable() }).nullable(),
+ merge: z.object({ alpha: z.number(), lambda: z.number(), aiJudgedCoveragePct: z.number(), n: z.number(), backfilledPct: z.number().nullable() }).nullable(),
+ }),
instanceCount: z.number(),
windowDays: z.number(),
- gamingFlagsCaught: z.number(),
+ gamingFlagsCaught: z.number().nullable(),
}),
/** Trailing weekly history of totals.accuracyPct's SAME formula (#4447) -- null counts/accuracyPct on a week means
* too few decided (merged+closed) PRs to publish meaningful or non-identifying details. */
diff --git a/src/orb/analytics.ts b/src/orb/analytics.ts
index 761a721af..0ebe7488c 100644
--- a/src/orb/analytics.ts
+++ b/src/orb/analytics.ts
@@ -34,6 +34,18 @@ const OUTLIER_BAND = 0.25; // |instance precision − fleet median| beyond this
const GAMING_VOLUME_MULTIPLIER = 2; // an instance's decided count more than this many times the fleet median
const GAMING_PRECISION_BAND = OUTLIER_BAND; // mergePrecision this far ABOVE the fleet median (one-sided)
const GAMING_REVERSAL_RATIO = 0.5; // reversalRate below this fraction of the fleet median
+// #9068: below this many eligible instances, "an instance's volume/precision this far above the fleet
+// median" is UNSATISFIABLE by construction (with 1 eligible instance it IS the median; with 2, either could
+// be "above" the other trivially) — the detector cannot distinguish "gaming" from "the sole/only comparable
+// data point" below this floor, so it does not run at all rather than publish a structurally-guaranteed zero
+// as if it were a clean bill of health.
+export const GAMING_MIN_ELIGIBLE = 3;
+// #9068: an instance's reversalRate is a fraction of ALL its decided signals, so a genuinely well-run fleet
+// commonly has a fleet-median reversalRate of exactly 0 — `reversalRate < 0 * GAMING_REVERSAL_RATIO` can never
+// be true, so the "low reversal" conjunct (and therefore the whole flag) was structurally unfireable whenever
+// the median was 0. This absolute floor gives "low reversal" a meaning even then: at/under 2% is suspiciously
+// clean in absolute terms regardless of what the fleet median happens to be.
+const GAMING_REVERSAL_ABSOLUTE_FLOOR = 0.02;
/** Per-instance confusion-matrix cell as stored. */
export interface Cell {
@@ -105,8 +117,22 @@ export interface FleetAnalytics {
reversalRate: number | null;
/** Share of AUTONOMOUS decisions the realized outcome confirmed — the honest "decision accuracy"
* (#8820). See InstanceMetrics.decisionAccuracy for why this, not 1 − reversalRate, is the number to
- * publish. */
+ * publish.
+ *
+ * #9068: this is the POOLED proportion (mergeConfirmed+closeConfirmed)/(mergeVerdicts+closeVerdicts)
+ * over eligible instances, NOT the median of per-instance decisionAccuracy — a per-instance MEDIAN and a
+ * POOLED proportion are different estimands that only coincide by coincidence with equal per-instance
+ * volumes (true of today's single-instance fleet, not guaranteed once a second instance registers). This
+ * is also exactly what accuracyCiPct is a Wilson interval OVER downstream (public-stats.ts), so the two
+ * published figures are now guaranteed to describe the same population instead of merely usually
+ * agreeing. See decisionAccuracyMedian below for the per-instance-robust diagnostic this replaces as the
+ * published point estimate. */
decisionAccuracy: number | null;
+ /** #9068: the per-instance MEDIAN of decisionAccuracy — robust to a single instance's volume swamping the
+ * fleet figure, kept as a diagnostic now that `decisionAccuracy` above publishes the pooled proportion
+ * instead. Not itself published on the public surface; available to internal consumers that want the
+ * robust view. */
+ decisionAccuracyMedian: number | null;
cycleP50Ms: number | null;
cycleP95Ms: number | null;
/** #8829: confusion counts POOLED over eligible instances. Medians are robust to a bad contributor but
@@ -118,6 +144,11 @@ export interface FleetAnalytics {
instances: InstanceMetrics[];
outliers: Array<{ instanceId: string; metric: string; value: number; fleetMedian: number }>;
gamingPatternFlags: GamingPatternFlag[];
+ /** #9068: whether the anti-farming detector ran at all (eligible.length >= GAMING_MIN_ELIGIBLE). An empty
+ * `gamingPatternFlags` is ambiguous between "the detector ran and found nothing" and "the detector cannot
+ * run yet" — this disambiguates, so a public surface can publish null ("not enough instances to compare")
+ * instead of a structurally-guaranteed zero presented as a positive safety signal. */
+ gamingDetectionEligible: boolean;
}
function median(xs: number[]): number | null {
@@ -241,7 +272,15 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
const reg = await env.DB.prepare(`SELECT instance_id FROM orb_instances WHERE registered = 1`).all<{ instance_id: string }>();
registered = new Set((reg.results ?? []).map((r) => r.instance_id));
} catch {
- return { windowDays, instanceCount: 0, fleet: { mergePrecision: null, closePrecision: null, fpRate: null, reversalRate: null, decisionAccuracy: null, cycleP50Ms: null, cycleP95Ms: null, pooled: emptyPooled() }, instances: [], outliers: [], gamingPatternFlags: [] };
+ return {
+ windowDays,
+ instanceCount: 0,
+ fleet: { mergePrecision: null, closePrecision: null, fpRate: null, reversalRate: null, decisionAccuracy: null, decisionAccuracyMedian: null, cycleP50Ms: null, cycleP95Ms: null, pooled: emptyPooled() },
+ instances: [],
+ outliers: [],
+ gamingPatternFlags: [],
+ gamingDetectionEligible: false,
+ };
}
// Group cells by instance, fold each.
@@ -271,17 +310,23 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
}
}
- // #2350: gamingPatternFlags. Gated on fleetMergeP !== null (at least one eligible instance made a comparable
- // merge verdict) — decided/reversalRate are never null per-instance, so once `eligible` is known non-empty
- // (implied by fleetMergeP being resolvable), both medians below are guaranteed non-null too.
+ // #2350/#9068: gamingPatternFlags. Gated on fleetMergeP !== null (at least one eligible instance made a
+ // comparable merge verdict) AND eligible.length >= GAMING_MIN_ELIGIBLE — below that floor, "this far above
+ // the fleet median" is unsatisfiable by construction (an instance IS the median at n=1; either trivially
+ // "wins" at n=2), so the detector must not run at all rather than publish a guaranteed zero as a clean bill
+ // of health. decided/reversalRate are never null per-instance, so once `eligible` clears both gates, both
+ // medians below are guaranteed non-null too.
+ const gamingDetectionEligible = fleetMergeP !== null && eligible.length >= GAMING_MIN_ELIGIBLE;
const gamingPatternFlags: FleetAnalytics["gamingPatternFlags"] = [];
- if (fleetMergeP !== null) {
+ if (gamingDetectionEligible) {
const fleetMedianDecided = median(eligible.map((i) => i.decided))!;
const fleetReversalRate = median(eligible.map((i) => i.reversalRate))!;
for (const i of eligible) {
const highVolume = i.decided > fleetMedianDecided * GAMING_VOLUME_MULTIPLIER;
- const highPrecision = i.mergePrecision !== null && i.mergePrecision - fleetMergeP > GAMING_PRECISION_BAND;
- const lowReversal = i.reversalRate < fleetReversalRate * GAMING_REVERSAL_RATIO;
+ const highPrecision = i.mergePrecision !== null && i.mergePrecision - fleetMergeP! > GAMING_PRECISION_BAND;
+ // #9068: a fleet-median reversalRate of 0 is common (a healthy fleet), and a FRACTION of zero can never
+ // be undercut — fall back to an absolute floor in that case so "low reversal" can still mean something.
+ const lowReversal = fleetReversalRate > 0 ? i.reversalRate < fleetReversalRate * GAMING_REVERSAL_RATIO : i.reversalRate <= GAMING_REVERSAL_ABSOLUTE_FLOOR;
if (highVolume && highPrecision && lowReversal) {
gamingPatternFlags.push({
instanceId: i.instanceId,
@@ -289,7 +334,7 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
mergePrecision: i.mergePrecision!,
reversalRate: i.reversalRate,
fleetMedianDecided,
- fleetMergePrecision: fleetMergeP,
+ fleetMergePrecision: fleetMergeP!,
fleetReversalRate,
});
}
@@ -307,6 +352,10 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
}
const pooledVerdicts = pooled.mergeVerdicts + pooled.closeVerdicts;
pooled.coverage = pooledVerdicts + pooled.holds > 0 ? pooledVerdicts / (pooledVerdicts + pooled.holds) : null;
+ // #9068: the published point estimate is the POOLED proportion (same population accuracyCiPct's Wilson
+ // interval is computed over downstream in public-stats.ts), not the per-instance median — see the
+ // decisionAccuracy field doc above for why the two are different estimands.
+ const pooledDecisionAccuracy = pooledVerdicts > 0 ? (pooled.mergeConfirmed + pooled.closeConfirmed) / pooledVerdicts : null;
return {
windowDays,
@@ -316,7 +365,8 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
closePrecision: fleetCloseP,
fpRate: median(nums((i) => i.fpRate)),
reversalRate: median(nums((i) => i.reversalRate)),
- decisionAccuracy: median(nums((i) => i.decisionAccuracy)),
+ decisionAccuracy: pooledDecisionAccuracy,
+ decisionAccuracyMedian: median(nums((i) => i.decisionAccuracy)),
cycleP50Ms: percentile(cycle, 50),
cycleP95Ms: percentile(cycle, 95),
pooled,
@@ -324,6 +374,7 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
instances,
outliers,
gamingPatternFlags,
+ gamingDetectionEligible,
};
}
diff --git a/src/orb/ingest.ts b/src/orb/ingest.ts
index 005d936fc..176497039 100644
--- a/src/orb/ingest.ts
+++ b/src/orb/ingest.ts
@@ -3,6 +3,7 @@
// No raw repo names, owner identifiers, commit SHAs, or PR content — only HMAC-anonymized hashes +
// aggregate calibration metadata (verdict, outcome, reversal, bucketed reason, cycle time).
import { hashToken } from "../auth/security";
+import { validateCalibrationPayload } from "../review/risk-control";
const MAX_BATCH = 500;
const MAX_INSTANCE_ID_CHARS = 64;
@@ -215,7 +216,8 @@ export async function handleOrbIngest(body: string, db: D1Database, presentedIns
// shared fleet-wide bearer token (proof only of "some fleet member") may be enough to plant or delete one.
// Per-arm rows are scoped to THIS instance (orb_risk_control_arms); public-stats aggregates across
// registered instances at read time, so one compromised or miscalibrated peer can only ever touch its own
- // row. Bounded: two known arms, value stored verbatim as JSON for public-stats to render.
+ // row. Bounded: two known arms, shape/range-validated (#9068, validateCalibrationPayload) before storage,
+ // then kept as JSON for public-stats to render.
const riskControl = (payload as { risk_control?: unknown }).risk_control;
if (riskControl !== undefined && riskControl !== null && typeof riskControl === "object" && !Array.isArray(riskControl)) {
try {
@@ -239,7 +241,11 @@ export async function handleOrbIngest(body: string, db: D1Database, presentedIns
if (value === undefined) continue;
if (value === null) {
await db.prepare(`DELETE FROM orb_risk_control_arms WHERE instance_id = ? AND arm = ?`).bind(instance_id, arm).run();
- } else if (typeof value === "object" && !Array.isArray(value)) {
+ } else if (typeof value === "object" && !Array.isArray(value) && validateCalibrationPayload(value) !== null) {
+ // #9068: shape/range-validated (status === "calibrated", alpha/lambda/coverage in range, nAtLambda
+ // clears the zero-error floor for the payload's own alpha/delta) before it's allowed anywhere near
+ // storage or the public surface — a sender claiming an uncertifiable guarantee is silently dropped,
+ // the same best-effort posture as every other malformed row in this ingest.
await db
.prepare(
`INSERT INTO orb_risk_control_arms (instance_id, arm, payload_json, updated_at) VALUES (?, ?, ?, CURRENT_TIMESTAMP)
diff --git a/src/review/public-stats.ts b/src/review/public-stats.ts
index d5eecdd28..49a749beb 100644
--- a/src/review/public-stats.ts
+++ b/src/review/public-stats.ts
@@ -51,6 +51,7 @@
// are over disjoint PR sets and can be added directly.
import { getOrbGlobalStats } from "../orb/outcomes";
import { computeFleetAnalytics, wilsonInterval } from "../orb/analytics";
+import { validateCalibrationPayload } from "./risk-control";
import { loadRepoFocusManifest } from "../signals/focus-manifest-loader";
import { resolveLoopOverSelfRepoFullName } from "../config/loopover-repo-focus-manifest";
import { errorMessage } from "../utils/json";
@@ -257,15 +258,28 @@ export interface PublicStatsPayload {
/** merge + close verdicts behind accuracyPct — the denominator a reader needs to judge the claim. */
decidedCount: number;
/** #8835: the live finite-sample guarantees, per arm, when a registered instance publishes one —
- * "P(wrong | acted) ≤ alpha at coveragePct" with the certification's own sample size. Null arms mean no
- * guarantee is currently live (insufficient labels, or the instrument retracted it). */
- guaranteed: { close: { alpha: number; lambda: number; coveragePct: number; n: number } | null; merge: { alpha: number; lambda: number; coveragePct: number; n: number } | null };
+ * "P(wrong | acted) ≤ alpha at aiJudgedCoveragePct" with the certification's own sample size. Null arms
+ * mean no guarantee is currently live (insufficient labels, or the instrument retracted it).
+ *
+ * #9050: `aiJudgedCoveragePct` (renamed from `coveragePct`, which read as "% of all closes" sitting right
+ * next to the sibling `coveragePct` above that IS a share of all decided signals) is actually the share
+ * of the ARM'S AI-JUDGED sub-population the threshold covers — `loadCalibrationPairs` can only join a
+ * confidence to decisions an AI-judgment blocker actually ran on, which is a minority of real closes.
+ * `backfilledPct` surfaces how much of the certifying evidence is reconstructed history (the 2026-07
+ * calibration-corpus backfill) rather than live-accruing labels — null when the stored calibration
+ * predates that field. */
+ guaranteed: {
+ close: { alpha: number; lambda: number; aiJudgedCoveragePct: number; n: number; backfilledPct: number | null } | null;
+ merge: { alpha: number; lambda: number; aiJudgedCoveragePct: number; n: number; backfilledPct: number | null } | null;
+ };
instanceCount: number;
windowDays: number;
- /** Self-hosted instances currently flagged by computeFleetAnalytics's anti-farming detector
- * (gamingPatternFlags, src/orb/analytics.ts) -- proof the fleet actively polices for gaming, not just a
- * claim of it. Never identifies which instance; a bare count is public-safe. */
- gamingFlagsCaught: number;
+ /** #9068: self-hosted instances flagged by computeFleetAnalytics's anti-farming detector
+ * (gamingPatternFlags, src/orb/analytics.ts). null (not 0) when the fleet has fewer than
+ * GAMING_MIN_ELIGIBLE eligible instances — below that floor the detector cannot run at all (an instance
+ * IS the fleet median at n=1), so a structural zero must never be presented as "checked, found none".
+ * Never identifies which instance; a bare count is public-safe. */
+ gamingFlagsCaught: number | null;
};
}
@@ -475,21 +489,47 @@ export async function getPublicStats(
// largest sample size (nAtLambda) — more data is a more trustworthy estimate, so one low-n or stale peer
// can no longer eclipse a well-calibrated one just by writing last. Fail-open null — a flags blip hides
// the guarantee rather than fabricating or freezing one.
- const readGuarantee = async (arm: string): Promise<{ alpha: number; lambda: number; coveragePct: number; n: number } | null> => {
+ //
+ // #9068: rows are re-validated HERE (not just at ingest, validateCalibrationPayload from ./risk-control) —
+ // defense in depth against a row written before this gate existed, or any other path into the table. Reads
+ // every registered row for the arm (not just the top one) and walks them in nAtLambda-descending order,
+ // skipping any that fail validation, so a single malformed/stale peer can no longer hide a good one behind
+ // it in the ORDER BY.
+ const readGuarantee = async (arm: string): Promise<{ alpha: number; lambda: number; aiJudgedCoveragePct: number; n: number; backfilledPct: number | null } | null> => {
try {
- const row = await env.DB.prepare(
+ const { results } = await env.DB.prepare(
`SELECT o.payload_json AS value FROM orb_risk_control_arms o
JOIN orb_instances i ON i.instance_id = o.instance_id AND i.registered = 1
WHERE o.arm = ?
ORDER BY CAST(json_extract(o.payload_json, '$.nAtLambda') AS REAL) DESC, o.updated_at DESC
- LIMIT 1`,
+ LIMIT 50`,
)
.bind(arm)
- .first<{ value: string }>();
- if (!row?.value) return null;
- const parsed = JSON.parse(row.value) as { alpha?: unknown; lambda?: unknown; coverageAtLambda?: unknown; nAtLambda?: unknown };
- if (typeof parsed.alpha !== "number" || typeof parsed.lambda !== "number" || typeof parsed.coverageAtLambda !== "number" || typeof parsed.nAtLambda !== "number") return null;
- return { alpha: parsed.alpha, lambda: parsed.lambda, coveragePct: Math.round(parsed.coverageAtLambda * 1000) / 10, n: parsed.nAtLambda };
+ .all<{ value: string }>();
+ for (const row of results ?? []) {
+ if (!row?.value) continue;
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(row.value);
+ } catch {
+ continue;
+ }
+ const validated = validateCalibrationPayload(parsed);
+ if (!validated) continue;
+ // #9050: totalPairs/backfilledPairs are supplementary display fields, not part of the core guarantee
+ // — a calibration stored before they existed simply has no backfilled split to show.
+ const p = parsed as { totalPairs?: unknown; backfilledPairs?: unknown };
+ const totalPairs = typeof p.totalPairs === "number" && p.totalPairs > 0 ? p.totalPairs : null;
+ const backfilledPairs = typeof p.backfilledPairs === "number" ? p.backfilledPairs : null;
+ return {
+ alpha: validated.alpha,
+ lambda: validated.lambda,
+ aiJudgedCoveragePct: Math.round(validated.coverageAtLambda * 1000) / 10,
+ n: validated.nAtLambda,
+ backfilledPct: totalPairs !== null && backfilledPairs !== null ? Math.round((backfilledPairs / totalPairs) * 1000) / 10 : null,
+ };
+ }
+ return null;
} catch {
return null;
}
@@ -541,7 +581,7 @@ export async function getPublicStats(
guaranteed,
instanceCount: fleet.instanceCount,
windowDays: fleet.windowDays,
- gamingFlagsCaught: fleet.gamingPatternFlags.length,
+ gamingFlagsCaught: fleet.gamingDetectionEligible ? fleet.gamingPatternFlags.length : null,
},
};
}
diff --git a/src/review/risk-control-wire.ts b/src/review/risk-control-wire.ts
index 824339768..016827aaa 100644
--- a/src/review/risk-control-wire.ts
+++ b/src/review/risk-control-wire.ts
@@ -2,10 +2,14 @@
//
// Reads (adjudication, decision-time confidence) pairs — human labels from decision_audit_labels (#8830/
// #8831) joined to the confidence each decision persisted in its decision record (#8834) — runs the
-// fixed-sequence calibration PER ARM (Neyman–Pearson: a wrong merge costs more than a wrong close, so each
-// arm carries its own α), and publishes the result: a calibrated λ̂ lands in system_flags plus an audit
-// event carrying the certified statement; an under-powered set DELETES any stale λ̂ (a stale guarantee is a
-// lie) and audits the shortfall so the label-collection burn-down is visible.
+// calibration PER ARM (Neyman–Pearson: a wrong merge costs more than a wrong close, so each arm carries its
+// own α), and publishes the result: a calibrated λ̂ lands in system_flags plus an audit event carrying the
+// certified statement. A non-calibrated result DELETES any stale λ̂ (a stale guarantee is a lie) and audits
+// the shortfall — but #9048 splits WHICH shortfall into two distinct events, because they need opposite
+// remediations: `risk_control_insufficient` (too few usable labels — go collect more) vs
+// `risk_control_no_certifiable_threshold` (plenty of labels, but no λ clears alpha — investigate precision,
+// or accept a weaker alpha). Conflating the two under one message previously sent the label-collection
+// burn-down (#8828) chasing labels a repo already had plenty of.
//
// DELIBERATELY CONSULT-ONLY in this change: the calibrated λ̂ is not yet wired into the live gate floor,
// because ai_review_close_confidence already has an automatic writer (the backtest-gated knob loosening,
@@ -84,13 +88,20 @@ export function riskControlFlagKey(arm: string): string {
* PR reviewed across several pushes accumulates several rows): the label adjudicates the decision that was
* ACTED — the latest finalized record, the same latest-row semantics loadDecisionRecordCollapsible renders —
* and a bare target_id join would fan one label out into N pairs with different confidences, breaking the
- * one-label-one-trial contract Clopper–Pearson depends on. `id DESC` breaks created_at ties. */
+ * one-label-one-trial contract Clopper–Pearson depends on. `id DESC` breaks created_at ties.
+ *
+ * #9050 (latent-risk hardening, not a live bug as of the audit that filed it — every row today already joins
+ * the right action): `AND dr2.action = dal.verdict` requires the latest record to be the SAME disposition the
+ * label adjudicates, so a later HOLD (or MERGE) record on the same PR — a later push whose review cycle ended
+ * differently — can never shadow the CLOSE record the label is actually about, mirroring the sampler's own
+ * `decision IN ('merge','close')` restriction (decision-audit.ts). */
export async function loadCalibrationPairs(env: Env, verdict: "close" | "merge", project: string | null = null): Promise {
const base = `SELECT dal.adjudication AS adjudication, dr.record_json AS recordJson
FROM decision_audit_labels dal
JOIN decision_records dr ON dr.id = (
SELECT dr2.id FROM decision_records dr2
WHERE dr2.repo_full_name || '#' || dr2.pull_number = dal.target_id
+ AND dr2.action = dal.verdict
ORDER BY dr2.created_at DESC, dr2.id DESC LIMIT 1)
WHERE dal.status = 'adjudicated'
AND dal.adjudication IN ('correct', 'incorrect')
@@ -100,9 +111,12 @@ export async function loadCalibrationPairs(env: Env, verdict: "close" | "merge",
const pairs: CalibrationPair[] = [];
for (const row of results) {
try {
- const record = JSON.parse(row.recordJson) as { aiConfidence?: number | null };
+ const record = JSON.parse(row.recordJson) as { aiConfidence?: number | null; configDigest?: unknown };
if (typeof record.aiConfidence === "number") {
- pairs.push({ confidence: record.aiConfidence, correct: row.adjudication === "correct" });
+ // #9050: the 2026-07 calibration-corpus backfill stamps this exact sentinel on every record it
+ // reconstructed (see scripts/backfill-decision-labels.ts) — it is also that backfill's own
+ // provenance marker, deliberately, since the historical resolved config is unrecoverable.
+ pairs.push({ confidence: record.aiConfidence, correct: row.adjudication === "correct", backfilled: record.configDigest === "backfill:unavailable" });
}
} catch (error) {
console.warn(JSON.stringify({ event: "risk_control_pair_parse_error", message: errorMessage(error).slice(0, 120) }));
@@ -131,7 +145,7 @@ async function recalibrateArm(env: Env, arm: "close" | "merge", verdict: "close"
detail: `${scope}: P(wrong | acted) ≤ ${alpha} guaranteed at ${Math.round(result.coverageAtLambda * 1000) / 10}% coverage (λ=${result.lambda}, n=${result.nAtLambda}, 1−δ=${1 - delta})`,
metadata: { arm, ...result },
});
- } else {
+ } else if (result.status === "insufficient_labels") {
// A stale guarantee is a lie: retract any previously-published λ̂ the moment the data stops supporting it.
await env.DB.prepare(`DELETE FROM system_flags WHERE key = ?`).bind(riskControlFlagKey(scope)).run();
await recordAuditEvent(env, {
@@ -142,6 +156,20 @@ async function recalibrateArm(env: Env, arm: "close" | "merge", verdict: "close"
detail: `${scope}: cannot certify α=${alpha} — ${result.have} usable label(s) of ${result.needed} needed`,
metadata: { arm, ...result },
});
+ } else {
+ // #9048: a DISTINCT outcome from insufficient_labels — this scope has AMPLE labels (result.totalPairs
+ // cleared the floor) but no λ ever certified alpha. A distinct event_type keeps the label-collection
+ // burn-down (which reads risk_control_insufficient) from conflating "needs labels" with "needs a better
+ // error rate" — very different remediations, and only one of them "more labels" can fix.
+ await env.DB.prepare(`DELETE FROM system_flags WHERE key = ?`).bind(riskControlFlagKey(scope)).run();
+ await recordAuditEvent(env, {
+ eventType: "risk_control_no_certifiable_threshold",
+ actor: null,
+ targetKey: `riskcontrol:${scope}`,
+ outcome: "completed",
+ detail: `${scope}: ${result.totalPairs} labels available but no threshold achieves α=${alpha} (best upper bound ${Math.round(result.bestUpperBound * 1000) / 1000} at λ=${result.bestLambda}, n=${result.bestN})`,
+ metadata: { arm, ...result },
+ });
}
return result;
}
diff --git a/src/review/risk-control.ts b/src/review/risk-control.ts
index 89f378ddb..1cda722f7 100644
--- a/src/review/risk-control.ts
+++ b/src/review/risk-control.ts
@@ -1,17 +1,33 @@
// Distribution-free risk control for the act/hold threshold (#8835, epic #8828 Phase 3) — the mechanism that
// turns "99.5% accurate" from a hope into a guarantee ON THE DECISIONS ACTED.
//
-// METHOD: fixed-sequence testing, the Learn-then-Test special case (Angelopoulos et al.; applied to LLM
-// judges by Trust or Escalate, ICLR 2025). Sweep the confidence threshold λ from most conservative (1.0)
-// downward; at each λ compute the EMPIRICAL error rate over calibration pairs with confidence ≥ λ and take
-// an EXACT binomial (Clopper–Pearson) upper confidence bound on it. λ̂ is the smallest λ (largest coverage)
-// whose bound — and every more-conservative λ's bound — stays ≤ α. Then, with probability ≥ 1−δ over the
-// calibration draw, P(decision wrong | confidence ≥ λ̂) ≤ α. No distributional assumptions.
+// METHOD: Learn-then-Test (Angelopoulos et al.; applied to LLM judges by Trust or Escalate, ICLR 2025) over an
+// ASCENDING scan of the confidence threshold λ — permissive-first (the full calibration set) toward
+// conservative (the highest-confidence stratum alone), chosen over the reverse direction for POWER (see the
+// second doc comment below for why conservative-first is a trap). At each λ, take an EXACT binomial
+// (Clopper–Pearson) UPPER confidence bound on the empirical error rate over pairs with confidence ≥ λ. λ̂ is
+// the FIRST λ in that scan whose bound certifies α.
//
-// HONESTY GUARDS, both load-bearing:
+// MULTIPLICITY (#9066): reporting whichever of the K distinct-confidence candidates passes first is a
+// SELECTION over K tests, not a single test — at a raw per-test δ, the true false-certification rate could
+// rise to as much as K·δ, so "confidence 1−δ" would overstate what the scan actually delivers. This module
+// closes that gap with a Bonferroni split: each candidate is tested at δ/K, and a union bound over all K caps
+// the probability that ANY tested bound is wrong at δ — so whichever λ this function returns is certified at
+// the FULL, originally-requested δ regardless of which one it turns out to be. This needs only the COUNT K,
+// not a pre-registered grid, so it is valid even though the candidates are the observed confidences. (The
+// alternative — true fixed-sequence: a pre-registered λ grid, tested conservative-first, stopping at the first
+// non-rejection — is issue #9066's other offered fix; Bonferroni was chosen as the smaller, safer diff over
+// the existing ascending/bisection scan below, at the cost of needing more labels per certified λ.)
+//
+// HONESTY GUARDS, all load-bearing:
// • INSUFFICIENT LABELS IS A REFUSAL, never a degraded guess. Even a zero-error calibration set cannot
// certify α until n ≥ ln(δ)/ln(1−α) (the exact rule-of-three bound) — at α=0.005, δ=0.05 that is 598
// clean labels. Below it this module refuses and the caller keeps the static floor.
+// • NO CERTIFIABLE THRESHOLD IS ALSO A REFUSAL, distinct from insufficient labels (#9048): a repo can have
+// plenty of labels and still refuse when no λ's bound clears α — a fundamentally different shortfall
+// ("the error rate is too high", not "there aren't enough labels"). `no_certifiable_threshold` carries the
+// total label count and the best bound the scan reached, so a caller never reports a residual stratum
+// size as if it were the repo's real label supply.
// • `uncertain` adjudications are EXCLUDED from both numerator and denominator (the rubric's contract):
// a genuine judgment call is not evidence about the gate's correctness in either direction.
//
@@ -62,6 +78,12 @@ export type CalibrationPair = {
confidence: number;
/** The human adjudication: true = the decision was correct. (`uncertain` rows never reach this module.) */
correct: boolean;
+ /** Provenance (#9050): true for a label reconstructed by the 2026-07 calibration-corpus backfill (the
+ * historical config was unknowable at reconstruction time, so the record carries the `configDigest`
+ * sentinel `"backfill:unavailable"` instead of a real digest); false for a label the live pipeline
+ * produced. Surfaced so a published guarantee can say how much of its evidence is backfilled vs
+ * live-accruing, rather than presenting both as equally fresh. */
+ backfilled: boolean;
};
export type CalibrationResult =
@@ -78,21 +100,40 @@ export type CalibrationResult =
alpha: number;
delta: number;
totalPairs: number;
+ /** #9050: of `totalPairs`, how many are backfilled rather than live — the published guarantee should
+ * say when it rests mostly on reconstructed history instead of live-accruing labels. */
+ backfilledPairs: number;
}
- | { status: "insufficient_labels"; needed: number; have: number; alpha: number; delta: number };
+ | { status: "insufficient_labels"; needed: number; have: number; alpha: number; delta: number }
+ | {
+ /** #9048: distinct from `insufficient_labels` — this repo/arm has AMPLE labels (`totalPairs` cleared the
+ * floor), but no λ's Clopper–Pearson bound ever certified alpha. The remediation is completely
+ * different (investigate close/merge precision, or accept a weaker alpha), not "collect more labels",
+ * so this status must never be reported or messaged as if it were a label shortfall. */
+ status: "no_certifiable_threshold";
+ totalPairs: number;
+ /** The candidate (n, λ, upper bound) the scan came closest to certifying with — the largest n the scan
+ * actually tested a bound at, and that bound's value, for an actionable "how far off" message. */
+ bestN: number;
+ bestLambda: number;
+ bestUpperBound: number;
+ alpha: number;
+ delta: number;
+ };
/**
- * Fixed-sequence calibration, walked in DESCENDING-coverage order: candidates are the distinct observed
- * confidences ascending, so the FIRST test is the most permissive λ (full calibration set — maximum power,
- * maximum coverage) and each subsequent test drops the lowest-confidence stratum. The sweep stops at the
- * FIRST λ whose Clopper–Pearson bound certifies α — ordered stopping at the first rejection is what keeps
- * the selection valid at level δ without a multiplicity correction (Learn-then-Test, fixed-sequence
- * variant). Walking the other way (conservative-first) is a trap: early candidates fail on sample-size
- * POWER, not on errors, and monotone stopping would kill the sweep before it ever reached a certifiable λ.
+ * Ascending-λ scan (permissive → conservative): candidates are the distinct observed confidences ascending,
+ * so the FIRST test is the most permissive λ (full calibration set — maximum power, maximum coverage) and
+ * each subsequent test drops the lowest-confidence stratum. Returns the FIRST λ whose Clopper–Pearson bound
+ * certifies alpha, tested at the Bonferroni-split δ/K described in the module header (#9066) — this keeps
+ * the REPORTED λ̂ valid at the original δ without needing a fixed-sequence stopping rule. Walking the other
+ * way (conservative-first) is a trap: early candidates fail on sample-size POWER, not on errors, and monotone
+ * stopping would kill the sweep before it ever reached a certifiable λ.
*
- * Refuses (never guesses) when the total set is under the zero-error floor, or when no candidate certifies
- * — whether from real errors or from a passing-but-tiny high-confidence clique that cannot carry α on its
- * own. PURE and deterministic.
+ * Refuses (never guesses) when the total set is under the zero-error floor (`insufficient_labels`), or when
+ * ample labels exist but no candidate ever certifies (`no_certifiable_threshold`, #9048) — the latter carries
+ * the best bound the scan reached, so a caller can tell "collect more labels" apart from "the current error
+ * rate can't support this alpha". PURE and deterministic.
*/
export function calibrateActThreshold(pairs: CalibrationPair[], alpha: number, delta: number): CalibrationResult {
const needed = minimumCalibrationLabels(alpha, delta);
@@ -101,25 +142,42 @@ export function calibrateActThreshold(pairs: CalibrationPair[], alpha: number, d
const sorted = [...pairs].sort((a, b) => a.confidence - b.confidence); // ascending
const candidates = [...new Set(sorted.map((pair) => pair.confidence))]; // ascending λ = descending coverage
const totalErrors = sorted.filter((pair) => !pair.correct).length;
+ const backfilledPairs = sorted.filter((pair) => pair.backfilled).length;
+ // #9066: Bonferroni-split δ across the K candidates this scan actually tests — see the module header for
+ // why this, not literal fixed-sequence stopping, is the chosen fix.
+ const testDelta = delta / candidates.length;
let dropped = 0;
let droppedErrors = 0;
let index = 0;
- let lastTestedN = sorted.length;
+ // Tracks the closest-to-certifying candidate for `no_certifiable_threshold`'s message. The very first
+ // candidate always computes a bound (n = sorted.length ≥ needed, guaranteed by the floor check above), so
+ // these are always overwritten before any caller can observe the initial values.
+ let bestLambda = candidates[0]!;
+ let bestN = sorted.length;
+ let bestUpperBound = Infinity;
for (const lambda of candidates) {
const n = sorted.length - dropped;
const errors = totalErrors - droppedErrors;
- lastTestedN = n;
- if (n >= needed && clopperPearsonUpperBound(errors, n, delta) <= alpha) {
- return {
- status: "calibrated",
- lambda,
- coverageAtLambda: n / sorted.length,
- nAtLambda: n,
- errorsAtLambda: errors,
- alpha,
- delta,
- totalPairs: sorted.length,
- };
+ if (n >= needed) {
+ const bound = clopperPearsonUpperBound(errors, n, testDelta);
+ if (bound < bestUpperBound) {
+ bestUpperBound = bound;
+ bestLambda = lambda;
+ bestN = n;
+ }
+ if (bound <= alpha) {
+ return {
+ status: "calibrated",
+ lambda,
+ coverageAtLambda: n / sorted.length,
+ nAtLambda: n,
+ errorsAtLambda: errors,
+ alpha,
+ delta,
+ totalPairs: sorted.length,
+ backfilledPairs,
+ };
+ }
}
// Drop this stratum and test the next, more conservative λ.
while (index < sorted.length && sorted[index]!.confidence <= lambda) {
@@ -128,5 +186,26 @@ export function calibrateActThreshold(pairs: CalibrationPair[], alpha: number, d
index += 1;
}
}
- return { status: "insufficient_labels", needed, have: lastTestedN, alpha, delta };
+ return { status: "no_certifiable_threshold", totalPairs: sorted.length, bestN, bestLambda, bestUpperBound, alpha, delta };
+}
+
+/**
+ * Validates an untrusted, already-JSON-parsed calibration payload (an ingest body, or a stored flag) before
+ * it is allowed to reach storage or a public surface (#9068): requires `status === "calibrated"` (a refusal
+ * must never be published as a guarantee), range-checks alpha/lambda/coverageAtLambda, and enforces the SAME
+ * zero-error sample-size floor `calibrateActThreshold` itself refuses below — a payload claiming `nAtLambda`
+ * under that floor could not have been legitimately certified, whatever else it claims. PURE; returns the
+ * narrowed numeric fields on success, null otherwise.
+ */
+export function validateCalibrationPayload(value: unknown): { alpha: number; lambda: number; coverageAtLambda: number; nAtLambda: number; delta: number } | null {
+ if (typeof value !== "object" || value === null) return null;
+ const v = value as Record;
+ if (v.status !== "calibrated") return null;
+ if (typeof v.alpha !== "number" || !(v.alpha > 0 && v.alpha <= 0.05)) return null;
+ if (typeof v.lambda !== "number" || !(v.lambda >= 0 && v.lambda <= 1)) return null;
+ if (typeof v.coverageAtLambda !== "number" || !(v.coverageAtLambda >= 0 && v.coverageAtLambda <= 1)) return null;
+ if (typeof v.delta !== "number" || !(v.delta > 0 && v.delta <= 1)) return null;
+ if (typeof v.nAtLambda !== "number" || !Number.isFinite(v.nAtLambda)) return null;
+ if (v.nAtLambda < minimumCalibrationLabels(v.alpha, v.delta)) return null;
+ return { alpha: v.alpha, lambda: v.lambda, coverageAtLambda: v.coverageAtLambda, nAtLambda: v.nAtLambda, delta: v.delta };
}
diff --git a/test/integration/orb-ingest.test.ts b/test/integration/orb-ingest.test.ts
index 6262c1234..601be08e4 100644
--- a/test/integration/orb-ingest.test.ts
+++ b/test/integration/orb-ingest.test.ts
@@ -457,11 +457,13 @@ describe("GET /v1/internal/fleet/analytics route", () => {
.prepare("SELECT payload_json FROM orb_risk_control_arms WHERE instance_id='inst1' AND arm='close'")
.first<{ payload_json: string }>()
)?.payload_json;
+ // minimumCalibrationLabels(0.015, 0.05) = 199 -- 200 clears the floor with margin to spare.
+ const calibrated = { status: "calibrated", alpha: 0.015, lambda: 0.94, coverageAtLambda: 0.8, nAtLambda: 200, delta: 0.05 };
const send = (risk_control: unknown, instanceSecret?: string) =>
handleOrbIngest(JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "rh", pr_hash: `g${Math.random()}`, outcome: "merged" }], risk_control }), db, instanceSecret);
// Unregistered sender: the strongest homepage claim must not be plantable via open ingest.
- await send({ close: { alpha: 0.015, lambda: 0.94, coverageAtLambda: 0.8, nAtLambda: 200 } });
+ await send({ close: calibrated });
expect(await flag()).toBeUndefined();
// #9121: registering now mints a per-instance credential -- the real, HTTP registration route is used
@@ -471,12 +473,12 @@ describe("GET /v1/internal/fleet/analytics route", () => {
expect(instanceSecret).toMatch(/^orbis_[0-9a-f]{64}$/);
// Registered but WITHOUT the credential: the claim is refused, not silently accepted.
- const rejected = await send({ close: { alpha: 0.015, lambda: 0.94, coverageAtLambda: 0.8, nAtLambda: 200 } });
+ const rejected = await send({ close: calibrated });
expect(rejected).toEqual({ error: "instance_unauthenticated" });
expect(await flag()).toBeUndefined();
// Registered AND credential-authenticated: the claim is accepted.
- await send({ close: { alpha: 0.015, lambda: 0.94, coverageAtLambda: 0.8, nAtLambda: 200 } }, instanceSecret);
+ await send({ close: calibrated }, instanceSecret);
expect(JSON.parse((await flag())!)).toMatchObject({ lambda: 0.94 });
// An ABSENT arm is "no change", never an implicit retraction (#9121) -- the stored value survives.
@@ -488,8 +490,36 @@ describe("GET /v1/internal/fleet/analytics route", () => {
expect(await flag()).toBeUndefined();
// A WRONG credential is also refused, not silently accepted.
- const wrongSecret = await send({ close: { alpha: 0.015, lambda: 0.5, coverageAtLambda: 0.8, nAtLambda: 200 } }, "orbis_wrongwrongwrong");
+ const wrongSecret = await send({ close: { ...calibrated, lambda: 0.5 } }, "orbis_wrongwrongwrong");
expect(wrongSecret).toEqual({ error: "instance_unauthenticated" });
expect(await flag()).toBeUndefined();
});
+
+ it("#9068: rejects a malformed/uncertifiable risk_control payload before it ever reaches storage", async () => {
+ const env = createTestEnv();
+ const db = env.DB as unknown as D1Database;
+ const flag = async () =>
+ (
+ await (db as unknown as TestD1Database)
+ .prepare("SELECT payload_json FROM orb_risk_control_arms WHERE instance_id='inst2' AND arm='close'")
+ .first<{ payload_json: string }>()
+ )?.payload_json;
+ const reg = await app.request("/v1/internal/orb/instances/register", { method: "POST", headers: auth, body: JSON.stringify({ instanceId: "inst2" }) }, env);
+ const { instanceSecret } = (await reg.json()) as { instanceSecret: string };
+ const send = (risk_control: unknown) =>
+ handleOrbIngest(JSON.stringify({ instance_id: "inst2", events: [{ repo_hash: "rh", pr_hash: `g${Math.random()}`, outcome: "merged" }], risk_control }), db, instanceSecret);
+
+ // A refusal status must never publish as a guarantee.
+ await send({ close: { status: "insufficient_labels", alpha: 0.015, lambda: 0.94, coverageAtLambda: 0.8, nAtLambda: 200, delta: 0.05 } });
+ expect(await flag()).toBeUndefined();
+ // nAtLambda below the zero-error floor for its own alpha/delta (minimumCalibrationLabels(0.015,0.05)=199).
+ await send({ close: { status: "calibrated", alpha: 0.015, lambda: 0.94, coverageAtLambda: 0.8, nAtLambda: 198, delta: 0.05 } });
+ expect(await flag()).toBeUndefined();
+ // alpha out of the accepted (0, 0.05] range.
+ await send({ close: { status: "calibrated", alpha: 0.2, lambda: 0.94, coverageAtLambda: 0.8, nAtLambda: 200, delta: 0.05 } });
+ expect(await flag()).toBeUndefined();
+ // A subsequent well-formed payload still succeeds -- the guard rejects only the bad rows.
+ await send({ close: { status: "calibrated", alpha: 0.015, lambda: 0.94, coverageAtLambda: 0.8, nAtLambda: 200, delta: 0.05 } });
+ expect(JSON.parse((await flag())!)).toMatchObject({ lambda: 0.94 });
+ });
});
diff --git a/test/unit/orb-analytics.test.ts b/test/unit/orb-analytics.test.ts
index 65f2aee86..f78eff8c5 100644
--- a/test/unit/orb-analytics.test.ts
+++ b/test/unit/orb-analytics.test.ts
@@ -151,20 +151,40 @@ describe("computeFleetAnalytics()", () => {
expect(inst.policyActions).toBe(0);
});
- it("decisionAccuracy is null for a holds-only instance (no decision to score) and drives the fleet median", async () => {
+ it("decisionAccuracy is null for a holds-only instance (no decision to score)", async () => {
const env = createTestEnv();
await signals(env, "holds-only", 6, { verdict: "hold", outcome: "merged" });
const holdsOnly = (await computeFleetAnalytics(env)).instances[0]!;
expect(holdsOnly.decisionAccuracy).toBeNull();
+ });
- const env2 = createTestEnv();
- await signals(env2, "a", 8, { verdict: "merge", outcome: "merged" });
- await signals(env2, "a", 2, { verdict: "merge", outcome: "closed" }); // 0.8
- await signals(env2, "b", 9, { verdict: "close", outcome: "closed" });
- await signals(env2, "b", 1, { verdict: "close", outcome: "merged" }); // 0.9
- await env2.DB.prepare(`INSERT INTO orb_instances (instance_id, registered) VALUES ('a', 1), ('b', 1)`).run();
- const fleet = await computeFleetAnalytics(env2);
- expect(fleet.fleet.decisionAccuracy).toBeCloseTo(0.85); // median of 0.8 and 0.9
+ it("#9068: fleet.decisionAccuracy is the POOLED proportion, not the per-instance median — the two coincide only at equal per-instance volumes", async () => {
+ const env = createTestEnv();
+ await signals(env, "a", 8, { verdict: "merge", outcome: "merged" });
+ await signals(env, "a", 2, { verdict: "merge", outcome: "closed" }); // instance accuracy 0.8, weight 10
+ await signals(env, "b", 9, { verdict: "close", outcome: "closed" });
+ await signals(env, "b", 1, { verdict: "close", outcome: "merged" }); // instance accuracy 0.9, weight 10
+ await register(env, "a", "b");
+ const fleet = await computeFleetAnalytics(env);
+ // Equal weights (10 verdicts each): pooled (8+9)/(10+10)=0.85 and median (0.8+0.9)/2=0.85 coincide here.
+ expect(fleet.fleet.decisionAccuracy).toBeCloseTo(0.85);
+ expect(fleet.fleet.decisionAccuracyMedian).toBeCloseTo(0.85);
+ });
+
+ it("#9068 REGRESSION: pooled decisionAccuracy diverges from the per-instance median once weights differ — publishing the median would misrepresent the pooled population", async () => {
+ const env = createTestEnv();
+ // Instance "a": 90 merge verdicts, 81 confirmed -> instance accuracy 0.9 (heavy weight).
+ await signals(env, "a", 81, { verdict: "merge", outcome: "merged" });
+ await signals(env, "a", 9, { verdict: "merge", outcome: "closed" });
+ // Instance "b": 10 close verdicts, 5 confirmed -> instance accuracy 0.5 (light weight).
+ await signals(env, "b", 5, { verdict: "close", outcome: "closed" });
+ await signals(env, "b", 5, { verdict: "close", outcome: "merged" });
+ await register(env, "a", "b");
+ const fleet = await computeFleetAnalytics(env);
+ // Median of [0.9, 0.5] = 0.7. Pooled (81+5)/(90+10) = 0.86 -- a real, material divergence.
+ expect(fleet.fleet.decisionAccuracyMedian).toBeCloseTo(0.7);
+ expect(fleet.fleet.decisionAccuracy).toBeCloseTo(0.86);
+ expect(fleet.fleet.decisionAccuracy).not.toBeCloseTo(fleet.fleet.decisionAccuracyMedian!, 1);
});
it("a superseded close (#8820) disconfirms closePrecision and counts toward reversalRate, exactly like a reopen", async () => {
@@ -414,12 +434,60 @@ describe("gamingPatternFlags — anti-farming detection (#2350)", () => {
const env = createTestEnv();
const result = await computeFleetAnalytics(env);
expect(result.gamingPatternFlags).toEqual([]);
+ expect(result.gamingDetectionEligible).toBe(false);
});
it("fail-safe on a DB error -> empty gamingPatternFlags", async () => {
const broken = { DB: { prepare: () => ({ bind: () => ({ all: () => Promise.reject(new Error("boom")) }) }) } } as unknown as Env;
const result = await computeFleetAnalytics(broken);
expect(result.gamingPatternFlags).toEqual([]);
+ expect(result.gamingDetectionEligible).toBe(false);
+ });
+
+ it("#9068 REGRESSION: with fewer than GAMING_MIN_ELIGIBLE (3) eligible instances, detection does not run at all — gamingDetectionEligible is false, not a vacuous true", async () => {
+ const env = createTestEnv();
+ // Exactly 2 eligible instances, an extreme farming-shaped pattern on one of them: with only 1 comparable
+ // peer, "this far above the fleet median" is unsatisfiable by construction (see the module doc comment),
+ // so the detector must not even run — a structural zero must never be mistaken for "checked, found none".
+ await normalInstance(env, "normal1");
+ await signals(env, "farmer", 30, { verdict: "merge", outcome: "merged", reversal: "none" });
+ await register(env, "normal1", "farmer");
+
+ const result = await computeFleetAnalytics(env);
+ expect(result.instanceCount).toBe(2);
+ expect(result.gamingDetectionEligible).toBe(false);
+ expect(result.gamingPatternFlags).toEqual([]);
+ });
+
+ it("#9068 REGRESSION: with GAMING_MIN_ELIGIBLE (3) eligible instances, detection runs and gamingDetectionEligible is true", async () => {
+ const env = createTestEnv();
+ await normalInstance(env, "a");
+ await normalInstance(env, "b");
+ await normalInstance(env, "c");
+ await register(env, "a", "b", "c");
+
+ const result = await computeFleetAnalytics(env);
+ expect(result.instanceCount).toBe(3);
+ expect(result.gamingDetectionEligible).toBe(true);
+ });
+
+ it("#9068 REGRESSION: flags a farming pattern even when the fleet's own reversal rate is uniformly zero — a fraction of zero can never fire", async () => {
+ const env = createTestEnv();
+ // 3 normal instances: 7 confirmed merges + 3 OUTRIGHT-WRONG merges (merge -> closed, no reversal marker)
+ // each -- mergePrecision 0.7, but reversalRate 0 (nothing here carries an explicit reversal flag).
+ for (const id of ["normal1", "normal2", "normal3"]) {
+ await signals(env, id, 7, { verdict: "merge", outcome: "merged", reversal: "none" });
+ await signals(env, id, 3, { verdict: "merge", outcome: "closed", reversal: "none" });
+ }
+ // Farmer: 30 decided (> 2x the fleet median of 10), precision 1.0 (> 0.7 + 0.25), reversalRate ALSO 0 --
+ // identical to the fleet's own median. Under the old fraction-of-median check this could never flag
+ // (0 < 0 * 0.5 is always false); the absolute floor makes "low reversal" meaningful even at a zero median.
+ await signals(env, "farmer", 30, { verdict: "merge", outcome: "merged", reversal: "none" });
+ await register(env, "normal1", "normal2", "normal3", "farmer");
+
+ const result = await computeFleetAnalytics(env);
+ expect(result.fleet.reversalRate).toBe(0); // confirms the fleet median really is zero here
+ expect(result.gamingPatternFlags.map((f) => f.instanceId)).toEqual(["farmer"]);
});
});
diff --git a/test/unit/public-stats.test.ts b/test/unit/public-stats.test.ts
index d1fde747a..5179d16b6 100644
--- a/test/unit/public-stats.test.ts
+++ b/test/unit/public-stats.test.ts
@@ -218,7 +218,7 @@ describe("getPublicStats — live aggregate over the review ledger", () => {
decidedCount: 0,
instanceCount: 0,
windowDays: 90,
- gamingFlagsCaught: 0,
+ gamingFlagsCaught: null, // #9068: fewer than GAMING_MIN_ELIGIBLE eligible instances -- detector didn't run
guaranteed: { close: null, merge: null },
});
});
@@ -509,7 +509,8 @@ describe("getPublicStats — live aggregate over the review ledger", () => {
const out = await getPublicStats(env, NOW);
// 5 merge verdicts, 4 confirmed (the 5th was reverted) -> decisionAccuracy 4/5 -> 80%.
- expect(out.fleetAccuracy).toMatchObject({ accuracyPct: 80, instanceCount: 1, windowDays: 90, gamingFlagsCaught: 0 });
+ // Only 1 eligible instance -- below GAMING_MIN_ELIGIBLE (3), so gamingFlagsCaught is null (#9068).
+ expect(out.fleetAccuracy).toMatchObject({ accuracyPct: 80, instanceCount: 1, windowDays: 90, gamingFlagsCaught: null });
// #8829: per-arm split, coverage, sample size, and a Wilson interval ride every published figure.
expect(out.fleetAccuracy.mergePrecisionPct).toBe(80);
expect(out.fleetAccuracy.closePrecisionPct).toBeNull(); // no close verdicts in this fixture
@@ -831,16 +832,27 @@ describe("getPublicStats — live aggregate over the review ledger", () => {
});
});
-describe("fleetAccuracy.guaranteed (#8835/#9121)", () => {
+describe("fleetAccuracy.guaranteed (#8835/#9121/#9050/#9068)", () => {
+ // minimumCalibrationLabels(0.015, 0.05) = 199 -- every valid fixture below clears it with margin.
+ const calibrated = (overrides: Record = {}) => ({
+ status: "calibrated",
+ alpha: 0.015,
+ lambda: 0.94,
+ coverageAtLambda: 0.82,
+ nAtLambda: 240,
+ delta: 0.05,
+ ...overrides,
+ });
+
it("publishes a live per-arm guarantee from a REGISTERED instance's orb_risk_control_arms row; malformed, unregistered, or absent rows read null (fail-open)", async () => {
const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" });
await env.DB.prepare(`INSERT INTO orb_instances (instance_id, registered) VALUES ('inst-a', 1)`).run();
await env.DB.prepare(`INSERT INTO orb_risk_control_arms (instance_id, arm, payload_json) VALUES ('inst-a', 'close', ?)`)
- .bind(JSON.stringify({ alpha: 0.015, lambda: 0.94, coverageAtLambda: 0.82, nAtLambda: 240 }))
+ .bind(JSON.stringify(calibrated()))
.run();
await env.DB.prepare(`INSERT INTO orb_risk_control_arms (instance_id, arm, payload_json) VALUES ('inst-a', 'merge', '{broken')`).run();
const out = await getPublicStats(env, NOW);
- expect(out.fleetAccuracy.guaranteed.close).toEqual({ alpha: 0.015, lambda: 0.94, coveragePct: 82, n: 240 });
+ expect(out.fleetAccuracy.guaranteed.close).toEqual({ alpha: 0.015, lambda: 0.94, aiJudgedCoveragePct: 82, n: 240, backfilledPct: null });
expect(out.fleetAccuracy.guaranteed.merge).toBeNull();
// A structurally-wrong row (missing fields) also reads null rather than publishing garbage.
await env.DB.prepare(`UPDATE orb_risk_control_arms SET payload_json = '{"alpha":"high"}' WHERE instance_id = 'inst-a' AND arm = 'close'`).run();
@@ -848,11 +860,31 @@ describe("fleetAccuracy.guaranteed (#8835/#9121)", () => {
expect(again.fleetAccuracy.guaranteed.close).toBeNull();
});
+ it("#9050: surfaces the backfilled-vs-live split when the stored calibration carries totalPairs/backfilledPairs", async () => {
+ const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" });
+ await env.DB.prepare(`INSERT INTO orb_instances (instance_id, registered) VALUES ('inst-a', 1)`).run();
+ await env.DB.prepare(`INSERT INTO orb_risk_control_arms (instance_id, arm, payload_json) VALUES ('inst-a', 'close', ?)`)
+ .bind(JSON.stringify(calibrated({ totalPairs: 234, backfilledPairs: 231 })))
+ .run();
+ const out = await getPublicStats(env, NOW);
+ expect(out.fleetAccuracy.guaranteed.close).toEqual({ alpha: 0.015, lambda: 0.94, aiJudgedCoveragePct: 82, n: 240, backfilledPct: 98.7 });
+ });
+
+ it("#9068: rejects a stored row whose status is not 'calibrated' (defense in depth against a pre-#9068 write)", async () => {
+ const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" });
+ await env.DB.prepare(`INSERT INTO orb_instances (instance_id, registered) VALUES ('inst-a', 1)`).run();
+ await env.DB.prepare(`INSERT INTO orb_risk_control_arms (instance_id, arm, payload_json) VALUES ('inst-a', 'close', ?)`)
+ .bind(JSON.stringify(calibrated({ status: "insufficient_labels" })))
+ .run();
+ const out = await getPublicStats(env, NOW);
+ expect(out.fleetAccuracy.guaranteed.close).toBeNull();
+ });
+
it("#9121: an UNREGISTERED instance's row never publishes (the same open-ingest-can't-plant-a-guarantee invariant, now enforced at read time too)", async () => {
const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" });
await env.DB.prepare(`INSERT INTO orb_instances (instance_id, registered) VALUES ('inst-b', 0)`).run();
await env.DB.prepare(`INSERT INTO orb_risk_control_arms (instance_id, arm, payload_json) VALUES ('inst-b', 'close', ?)`)
- .bind(JSON.stringify({ alpha: 0.015, lambda: 0.94, coverageAtLambda: 0.82, nAtLambda: 240 }))
+ .bind(JSON.stringify(calibrated()))
.run();
const out = await getPublicStats(env, NOW);
expect(out.fleetAccuracy.guaranteed.close).toBeNull();
@@ -862,12 +894,26 @@ describe("fleetAccuracy.guaranteed (#8835/#9121)", () => {
const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" });
await env.DB.prepare(`INSERT INTO orb_instances (instance_id, registered) VALUES ('small-n', 1), ('big-n', 1)`).run();
await env.DB.prepare(`INSERT INTO orb_risk_control_arms (instance_id, arm, payload_json) VALUES ('small-n', 'close', ?)`)
- .bind(JSON.stringify({ alpha: 0.015, lambda: 0.9, coverageAtLambda: 0.7, nAtLambda: 30 }))
+ .bind(JSON.stringify(calibrated({ lambda: 0.9, coverageAtLambda: 0.7, nAtLambda: 210 })))
.run();
await env.DB.prepare(`INSERT INTO orb_risk_control_arms (instance_id, arm, payload_json) VALUES ('big-n', 'close', ?)`)
- .bind(JSON.stringify({ alpha: 0.015, lambda: 0.97, coverageAtLambda: 0.9, nAtLambda: 5000 }))
+ .bind(JSON.stringify(calibrated({ lambda: 0.97, coverageAtLambda: 0.9, nAtLambda: 5000 })))
+ .run();
+ const out = await getPublicStats(env, NOW);
+ expect(out.fleetAccuracy.guaranteed.close).toEqual({ alpha: 0.015, lambda: 0.97, aiJudgedCoveragePct: 90, n: 5000, backfilledPct: null });
+ });
+
+ it("#9068: a top (largest-nAtLambda) row that fails validation no longer hides a smaller VALID row behind it", async () => {
+ const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" });
+ await env.DB.prepare(`INSERT INTO orb_instances (instance_id, registered) VALUES ('bad-n', 1), ('good-n', 1)`).run();
+ // Largest nAtLambda, but alpha is out of range -- must be skipped, not returned as null outright.
+ await env.DB.prepare(`INSERT INTO orb_risk_control_arms (instance_id, arm, payload_json) VALUES ('bad-n', 'close', ?)`)
+ .bind(JSON.stringify(calibrated({ alpha: 0.2, nAtLambda: 9000 })))
+ .run();
+ await env.DB.prepare(`INSERT INTO orb_risk_control_arms (instance_id, arm, payload_json) VALUES ('good-n', 'close', ?)`)
+ .bind(JSON.stringify(calibrated({ nAtLambda: 300 })))
.run();
const out = await getPublicStats(env, NOW);
- expect(out.fleetAccuracy.guaranteed.close).toEqual({ alpha: 0.015, lambda: 0.97, coveragePct: 90, n: 5000 });
+ expect(out.fleetAccuracy.guaranteed.close).toEqual({ alpha: 0.015, lambda: 0.94, aiJudgedCoveragePct: 82, n: 300, backfilledPct: null });
});
});
diff --git a/test/unit/risk-control-wire.test.ts b/test/unit/risk-control-wire.test.ts
index 438e42659..d732a4b06 100644
--- a/test/unit/risk-control-wire.test.ts
+++ b/test/unit/risk-control-wire.test.ts
@@ -5,7 +5,14 @@ import { createTestEnv } from "../helpers/d1";
// #8835: the IO around the calibration math. What must never drift: uncertain labels excluded, rule-only
// (no-confidence) decisions skipped, per-arm scoping, publish-on-certify, RETRACT-on-insufficient.
-async function seedLabeledDecision(env: Env, n: number, verdict: "close" | "merge", adjudication: "correct" | "incorrect" | "uncertain", aiConfidence: number | null): Promise {
+async function seedLabeledDecision(
+ env: Env,
+ n: number,
+ verdict: "close" | "merge",
+ adjudication: "correct" | "incorrect" | "uncertain",
+ aiConfidence: number | null,
+ configDigest?: string,
+): Promise {
await env.DB.prepare(
`INSERT INTO decision_audit_labels (id, project, target_id, verdict, outcome, stratum, rubric_version, sampled_at, status, adjudication, adjudicated_at)
VALUES (?, 'o/r', ?, ?, 'closed', 'close_arm', '1', ?, 'adjudicated', ?, ?)`,
@@ -16,7 +23,7 @@ async function seedLabeledDecision(env: Env, n: number, verdict: "close" | "merg
`INSERT INTO decision_records (id, repo_full_name, pull_number, head_sha, action, reason_code, record_digest, record_json, created_at)
VALUES (?, 'o/r', ?, 'sha', ?, 'r', 'd', ?, ?)`,
)
- .bind(`record:o/r#${n}@sha`, n, verdict, JSON.stringify({ aiConfidence }), new Date().toISOString())
+ .bind(`record:o/r#${n}@sha`, n, verdict, JSON.stringify({ aiConfidence, ...(configDigest !== undefined ? { configDigest } : {}) }), new Date().toISOString())
.run();
}
@@ -37,8 +44,8 @@ describe("loadCalibrationPairs", () => {
await seedLabeledDecision(env, 5, "merge", "correct", 0.99); // other arm
const pairs = await loadCalibrationPairs(env, "close");
expect(pairs.sort((a, b) => a.confidence - b.confidence)).toEqual([
- { confidence: 0.6, correct: false },
- { confidence: 0.95, correct: true },
+ { confidence: 0.6, correct: false, backfilled: false },
+ { confidence: 0.95, correct: true, backfilled: false },
]);
});
@@ -55,7 +62,33 @@ describe("loadCalibrationPairs", () => {
.bind(`record:o/r#1@${sha}`, sha, JSON.stringify({ aiConfidence: confidence }), new Date(Date.now() + offsetMs).toISOString())
.run();
}
- expect(await loadCalibrationPairs(env, "close")).toEqual([{ confidence: 0.9, correct: true }]);
+ expect(await loadCalibrationPairs(env, "close")).toEqual([{ confidence: 0.9, correct: true, backfilled: false }]);
+ });
+
+ it("#9050: tags provenance from the record's configDigest — the backfill sentinel marks a pair backfilled", async () => {
+ const env = createTestEnv();
+ await seedLabeledDecision(env, 1, "close", "correct", 0.9); // live: no configDigest sentinel
+ await seedLabeledDecision(env, 2, "close", "correct", 0.95, "backfill:unavailable"); // backfilled
+ const pairs = await loadCalibrationPairs(env, "close");
+ expect(pairs.sort((a, b) => a.confidence - b.confidence)).toEqual([
+ { confidence: 0.9, correct: true, backfilled: false },
+ { confidence: 0.95, correct: true, backfilled: true },
+ ]);
+ });
+
+ it("#9050 (latent-risk hardening): only joins a record whose action matches the label's verdict — a later HOLD record must not shadow the acted CLOSE", async () => {
+ const env = createTestEnv();
+ await seedLabeledDecision(env, 1, "close", "correct", 0.9); // the acted close, confidence 0.9
+ // A LATER record for the SAME PR (a later push whose review cycle ended in a hold instead), at a
+ // different confidence. Without `AND dr2.action = dal.verdict`, the "latest record" subquery would pick
+ // this one purely on recency and join the close label to the WRONG decision's confidence.
+ await env.DB.prepare(
+ `INSERT INTO decision_records (id, repo_full_name, pull_number, head_sha, action, reason_code, record_digest, record_json, created_at)
+ VALUES (?, 'o/r', 1, 'sha2', 'hold', 'r', 'd', ?, ?)`,
+ )
+ .bind(`record:o/r#1@sha2`, JSON.stringify({ aiConfidence: 0.4 }), new Date(Date.now() + 60_000).toISOString())
+ .run();
+ expect(await loadCalibrationPairs(env, "close")).toEqual([{ confidence: 0.9, correct: true, backfilled: false }]);
});
});
@@ -85,6 +118,28 @@ describe("runRiskControlRecalibration", () => {
expect(stored.alpha).toBe(0.05); // 210 pairs < 350 → the schedule's first tier
const audit = await env.DB.prepare(`SELECT detail FROM audit_events WHERE event_type = 'risk_control_calibrated'`).first<{ detail: string }>();
expect(audit!.detail).toContain("P(wrong | acted) ≤ 0.05 guaranteed at 100% coverage");
+ expect(stored).toMatchObject({ backfilledPairs: 0 }); // #9050: none of these seeded pairs are backfilled
+ });
+
+ it("REGRESSION (#9048): NO CERTIFIABLE THRESHOLD — ample labels but no λ clears α reports the true total, not a residual stratum, under a distinct event_type", async () => {
+ const env = createTestEnv({ LOOPOVER_RISK_CONTROL_CLOSE_ALPHA: "0.005" }); // fixed alpha; floor = 598
+ // 600 dirty pairs (all wrong) at 0.5, then 50 clean pairs at 0.99 — 650 total, well over the 598 floor,
+ // but no candidate certifies: 0.5 has ruinous error, 0.99 falls below the floor once 0.5 is dropped.
+ for (let i = 1; i <= 600; i += 1) await seedLabeledDecision(env, i, "close", "incorrect", 0.5);
+ for (let i = 601; i <= 650; i += 1) await seedLabeledDecision(env, i, "close", "correct", 0.99);
+ const summary = await runRiskControlRecalibration(env);
+ expect(summary.close).toBe("no_certifiable_threshold");
+ const flag = await env.DB.prepare(`SELECT value FROM system_flags WHERE key = ?`).bind(riskControlFlagKey("close")).first();
+ expect(flag).toBeFalsy(); // no certifiable guarantee — retracted, same as insufficient_labels
+ const audit = await env.DB.prepare(
+ `SELECT detail FROM audit_events WHERE event_type = 'risk_control_no_certifiable_threshold' AND target_key = 'riskcontrol:close'`,
+ ).first<{ detail: string }>();
+ expect(audit!.detail).toContain("650 labels available but no threshold achieves α=0.005");
+ // The label-shortfall event must NOT also fire for this scope — it is a distinct outcome (#9048).
+ const shortfall = await env.DB.prepare(
+ `SELECT detail FROM audit_events WHERE event_type = 'risk_control_insufficient' AND target_key = 'riskcontrol:close'`,
+ ).first();
+ expect(shortfall).toBeFalsy();
});
});
@@ -97,7 +152,7 @@ describe("fail-safe arms", () => {
await env.DB.prepare("UPDATE decision_records SET record_json = '{broken' WHERE pull_number = 1").run();
await seedLabeledDecision(env, 2, "close", "incorrect", 0.6);
const pairs = await loadCalibrationPairs(env, "close");
- expect(pairs).toEqual([{ confidence: 0.6, correct: false }]);
+ expect(pairs).toEqual([{ confidence: 0.6, correct: false, backfilled: false }]);
expect(warn).toHaveBeenCalled();
vi.restoreAllMocks();
});
diff --git a/test/unit/risk-control.test.ts b/test/unit/risk-control.test.ts
index 9e7de936d..ced85b892 100644
--- a/test/unit/risk-control.test.ts
+++ b/test/unit/risk-control.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
-import { calibrateActThreshold, clopperPearsonUpperBound, minimumCalibrationLabels, type CalibrationPair } from "../../src/review/risk-control";
+import { calibrateActThreshold, clopperPearsonUpperBound, minimumCalibrationLabels, validateCalibrationPayload, type CalibrationPair } from "../../src/review/risk-control";
// #8835: the math that turns "99.5%" into a guarantee is pinned exactly — an anti-conservative bound here
// would publish a certainty the labels cannot support, which is the epic's original sin.
@@ -28,8 +28,8 @@ describe("minimumCalibrationLabels", () => {
});
});
-describe("calibrateActThreshold (fixed-sequence)", () => {
- const pair = (confidence: number, correct: boolean): CalibrationPair => ({ confidence, correct });
+describe("calibrateActThreshold", () => {
+ const pair = (confidence: number, correct: boolean, backfilled = false): CalibrationPair => ({ confidence, correct, backfilled });
it("REFUSES below the sample-size floor — insufficient labels is never a degraded guess", () => {
const clean = Array.from({ length: 100 }, () => pair(0.99, true));
@@ -37,39 +37,65 @@ describe("calibrateActThreshold (fixed-sequence)", () => {
expect(result).toMatchObject({ status: "insufficient_labels", needed: 598, have: 100 });
});
- it("certifies a clean, large set at full coverage", () => {
- const clean = Array.from({ length: 700 }, (_, i) => pair(0.9 + (i % 10) / 100, true));
+ it("certifies a clean, large set at full coverage — the least conservative candidate still certifies", () => {
+ // Sized to still certify under the #9066 Bonferroni-split delta (3 distinct confidences here → delta/3):
+ // the zero-error bound at n=900, delta/3≈0.01667 is ≈0.0045, comfortably under alpha=0.005.
+ const clean = Array.from({ length: 900 }, (_, i) => pair(0.9 + (i % 3) / 100, true));
const result = calibrateActThreshold(clean, 0.005, 0.05);
expect(result.status).toBe("calibrated");
if (result.status === "calibrated") {
- expect(result.lambda).toBe(0.9); // the least conservative candidate still certifies
+ expect(result.lambda).toBe(0.9);
expect(result.coverageAtLambda).toBe(1);
expect(result.errorsAtLambda).toBe(0);
+ expect(result.nAtLambda).toBe(900);
+ expect(result.backfilledPairs).toBe(0); // #9050: none of these pairs are backfilled
}
});
- it("stops the sweep at the first failing candidate — errors clustered at low confidence RAISE lambda and cut coverage", () => {
- // 650 clean pairs at 0.97, then a dirty low-confidence band: the sweep must stop before absorbing it.
+ it("stops the sweep at the first certifying candidate — errors clustered at low confidence RAISE lambda and cut coverage", () => {
+ // 800 clean pairs at 0.97, then a dirty low-confidence band: the sweep must drop the dirty band before
+ // certifying. 800 (not 650) leaves headroom for the #9066 Bonferroni split across these 2 candidates
+ // (delta/2=0.025): the zero-error bound at n=800 is ≈0.0046, just under alpha=0.005.
const pairs = [
- ...Array.from({ length: 650 }, () => pair(0.97, true)),
+ ...Array.from({ length: 800 }, () => pair(0.97, true)),
...Array.from({ length: 100 }, (_, i) => pair(0.6, i % 3 !== 0)), // ~33% wrong below the band
];
const result = calibrateActThreshold(pairs, 0.005, 0.05);
expect(result.status).toBe("calibrated");
if (result.status === "calibrated") {
expect(result.lambda).toBe(0.97);
- expect(result.nAtLambda).toBe(650);
- expect(result.coverageAtLambda).toBeCloseTo(650 / 750, 5);
+ expect(result.nAtLambda).toBe(800);
+ expect(result.coverageAtLambda).toBeCloseTo(800 / 900, 5);
expect(result.errorsAtLambda).toBe(0);
}
});
- it("a passing-but-tiny high-confidence clique cannot certify — the prefix itself must clear the floor", () => {
- // 50 pristine pairs at 0.99, then errors immediately: the 0.99 prefix passes its bound test... but 50
- // labels cannot certify alpha=0.005, and pretending otherwise is the exact dishonesty this refuses.
- const pairs = [...Array.from({ length: 50 }, () => pair(0.99, true)), ...Array.from({ length: 600 }, () => pair(0.5, false))];
+ it("REGRESSION (#9048): a repo with AMPLE labels that cannot certify reports its true total, not a residual stratum size", () => {
+ // 650 total labels (well over the 598-label floor for alpha=0.005) — but every candidate either has a
+ // ruinous error rate (the 600 pairs at 0.5, all wrong) or falls below the sample-size floor once that
+ // dirty stratum is dropped (only 50 remain at 0.99). Before #9048 this reported `have: 50` under
+ // `insufficient_labels` — exactly the "N usable labels of 59/598 needed" bug: a residual stratum size
+ // misreported as the repo's total label supply.
+ const pairs = [...Array.from({ length: 600 }, () => pair(0.5, false)), ...Array.from({ length: 50 }, () => pair(0.99, true))];
const result = calibrateActThreshold(pairs, 0.005, 0.05);
- expect(result).toMatchObject({ status: "insufficient_labels", needed: 598, have: 50 });
+ expect(result.status).toBe("no_certifiable_threshold");
+ if (result.status === "no_certifiable_threshold") {
+ expect(result.totalPairs).toBe(650); // the TRUE total, never a residual stratum size
+ expect(result.bestN).toBe(650); // the only candidate that cleared the sample-size floor
+ expect(result.bestLambda).toBe(0.5);
+ expect(result.bestUpperBound).toBeGreaterThan(0.005); // could not certify alpha
+ expect(result.bestUpperBound).toBeLessThanOrEqual(1);
+ }
+ });
+
+ it("REGRESSION (#9066): does not over-certify across many observed confidence candidates — Bonferroni-splits delta across the K distinct thresholds actually tested", () => {
+ // Same shape the pre-fix code certified at lambda=0.9 (700 clean pairs, 10 distinct confidences, zero
+ // errors, alpha=0.005): the RAW zero-error bound at n=700, delta=0.05 is ≈0.0043 (<=0.005, would certify),
+ // but split across K=10 candidates (delta/10=0.005) the bound is ≈0.0075 (>0.005) — correctly refuses,
+ // because reporting whichever of 10 tested candidates passes first is a selection, not a single test.
+ const pairs = Array.from({ length: 700 }, (_, i) => pair(0.9 + (i % 10) / 100, true));
+ const result = calibrateActThreshold(pairs, 0.005, 0.05);
+ expect(result.status).toBe("no_certifiable_threshold");
});
it("is deterministic and input-order independent", () => {
@@ -83,4 +109,65 @@ describe("calibrateActThreshold (fixed-sequence)", () => {
expect(calibrateActThreshold(pairs, 0.015, 0.05).status).toBe("calibrated"); // close-arm alpha
expect(calibrateActThreshold(pairs, 0.002, 0.05).status).toBe("insufficient_labels"); // merge-arm alpha
});
+
+ it("#9050: tags backfilledPairs from each pair's provenance", () => {
+ const pairs = [
+ ...Array.from({ length: 300 }, () => pair(0.9, true, true)), // backfilled
+ ...Array.from({ length: 600 }, () => pair(0.95, true, false)), // live
+ ];
+ const result = calibrateActThreshold(pairs, 0.015, 0.05);
+ expect(result.status).toBe("calibrated");
+ if (result.status === "calibrated") {
+ expect(result.totalPairs).toBe(900);
+ expect(result.backfilledPairs).toBe(300);
+ }
+ });
+});
+
+describe("validateCalibrationPayload (#9068)", () => {
+ const valid = { status: "calibrated", alpha: 0.015, lambda: 0.9, coverageAtLambda: 0.8, nAtLambda: 200, delta: 0.05 };
+
+ it("accepts a well-formed calibrated payload that clears the sample-size floor", () => {
+ expect(validateCalibrationPayload(valid)).toEqual({ alpha: 0.015, lambda: 0.9, coverageAtLambda: 0.8, nAtLambda: 200, delta: 0.05 });
+ });
+
+ it("rejects a non-object / null payload", () => {
+ expect(validateCalibrationPayload(null)).toBeNull();
+ expect(validateCalibrationPayload("nope")).toBeNull();
+ expect(validateCalibrationPayload(42)).toBeNull();
+ });
+
+ it("rejects anything whose status is not 'calibrated' — a refusal must never publish as a guarantee", () => {
+ expect(validateCalibrationPayload({ ...valid, status: "insufficient_labels" })).toBeNull();
+ expect(validateCalibrationPayload({ ...valid, status: undefined })).toBeNull();
+ });
+
+ it("range-checks alpha (0, 0.05]", () => {
+ expect(validateCalibrationPayload({ ...valid, alpha: 0 })).toBeNull();
+ expect(validateCalibrationPayload({ ...valid, alpha: 0.06 })).toBeNull();
+ expect(validateCalibrationPayload({ ...valid, alpha: "0.015" })).toBeNull();
+ expect(validateCalibrationPayload({ ...valid, alpha: 0.05 })).not.toBeNull(); // inclusive upper bound
+ });
+
+ it("range-checks lambda [0, 1]", () => {
+ expect(validateCalibrationPayload({ ...valid, lambda: -0.01 })).toBeNull();
+ expect(validateCalibrationPayload({ ...valid, lambda: 1.01 })).toBeNull();
+ expect(validateCalibrationPayload({ ...valid, lambda: 1 })).not.toBeNull();
+ });
+
+ it("range-checks coverageAtLambda [0, 1]", () => {
+ expect(validateCalibrationPayload({ ...valid, coverageAtLambda: -0.01 })).toBeNull();
+ expect(validateCalibrationPayload({ ...valid, coverageAtLambda: 1.5 })).toBeNull();
+ });
+
+ it("enforces the zero-error sample-size floor on nAtLambda given the payload's own alpha/delta", () => {
+ // minimumCalibrationLabels(0.015, 0.05) = 199 — 198 could not have been legitimately certified.
+ expect(validateCalibrationPayload({ ...valid, nAtLambda: 198 })).toBeNull();
+ expect(validateCalibrationPayload({ ...valid, nAtLambda: 199 })).not.toBeNull();
+ });
+
+ it("rejects a missing/non-numeric delta (needed to evaluate the sample-size floor at all)", () => {
+ expect(validateCalibrationPayload({ ...valid, delta: undefined })).toBeNull();
+ expect(validateCalibrationPayload({ ...valid, delta: "0.05" })).toBeNull();
+ });
});