feat(orb): publish per-arm precision, coverage, and Wilson intervals with the fleet accuracy

apps/loopover-ui/public/openapi.json

@@ -568,10 +568,80 @@
               },
               "gamingFlagsCaught": {
                 "type": "number"
+              },
+              "accuracyCiPct": {
+                "type": "object",
+                "nullable": true,
+                "properties": {
+                  "lo": {
+                    "type": "number"
+                  },
+                  "hi": {
+                    "type": "number"
+                  }
+                },
+                "required": [
+                  "lo",
+                  "hi"
+                ]
+              },
+              "mergePrecisionPct": {
+                "type": "number",
+                "nullable": true
+              },
+              "mergePrecisionCiPct": {
+                "type": "object",
+                "nullable": true,
+                "properties": {
+                  "lo": {
+                    "type": "number"
+                  },
+                  "hi": {
+                    "type": "number"
+                  }
+                },
+                "required": [
+                  "lo",
+                  "hi"
+                ]
+              },
+              "closePrecisionPct": {
+                "type": "number",
+                "nullable": true
+              },
+              "closePrecisionCiPct": {
+                "type": "object",
+                "nullable": true,
+                "properties": {
+                  "lo": {
+                    "type": "number"
+                  },
+                  "hi": {
+                    "type": "number"
+                  }
+                },
+                "required": [
+                  "lo",
+                  "hi"
+                ]
+              },
+              "coveragePct": {
+                "type": "number",
+                "nullable": true
+              },
+              "decidedCount": {
+                "type": "number"
               }
             },
             "required": [
               "accuracyPct",
+              "accuracyCiPct",
+              "mergePrecisionPct",
+              "mergePrecisionCiPct",
+              "closePrecisionPct",
+              "closePrecisionCiPct",
+              "coveragePct",
+              "decidedCount",
               "instanceCount",
               "windowDays",
               "gamingFlagsCaught"

apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts

@@ -30,6 +30,15 @@ export type PublicStats = {
    *  (a frozen own-ledger snapshot) whenever instanceCount > 0. See public-stats.ts's PublicStatsPayload. */
   fleetAccuracy: {
     accuracyPct: number | null;
+    /** #8829 fields -- optional-chained at the render site: an older backend simply omits them and the tile
+     *  degrades to the bare figure rather than throwing. */
+    accuracyCiPct?: { lo: number; hi: number } | null;
+    mergePrecisionPct?: number | null;
+    mergePrecisionCiPct?: { lo: number; hi: number } | null;
+    closePrecisionPct?: number | null;
+    closePrecisionCiPct?: { lo: number; hi: number } | null;
+    coveragePct?: number | null;
+    decidedCount?: number;
     instanceCount: number;
     windowDays: number;
     gamingFlagsCaught: number;

apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx

@@ -206,7 +206,7 @@
       durationMs: 1,
       data: {
         ...PAYLOAD,
-        fleetAccuracy: { accuracyPct: 80, instanceCount: 3, windowDays: 90, gamingFlagsCaught: 2 },
+        fleetAccuracy: { accuracyPct: 80, coveragePct: 64.3, instanceCount: 3, windowDays: 90, gamingFlagsCaught: 2 },
       },
     });
     renderWithClient(<ProofOfPowerStats />);
@@ -217,7 +217,7 @@
     // rather than the retired "reversal-grounded" formula the surface no longer publishes.
     expect(
       screen.getByText(
-        "merge/close calls confirmed by outcome · 3 self-hosted instances · 2 gaming patterns flagged",
+        "merge/close calls confirmed by outcome · at 64.3% coverage · 3 self-hosted instances · 2 gaming patterns flagged",
       ),
     ).toBeTruthy();
     // No fleet-accuracy trend exists yet -- the tile's sparkline is omitted rather than showing a mismatched one.

apps/loopover-ui/src/components/site/proof-of-power-stats.tsx

@@ -166,7 +166,9 @@ export function ProofOfPowerStats({ className }: { className?: string }) {
               // (holds excluded -- they're deferrals, not decisions), so say that rather than the old
               // "reversal-grounded" wording, which described a formula the surface no longer publishes.
               fleetEligible
-                ? `merge/close calls confirmed by outcome · ${intFmt.format(data.fleetAccuracy.instanceCount)} self-hosted instance${data.fleetAccuracy.instanceCount === 1 ? "" : "s"}${data.fleetAccuracy.gamingFlagsCaught > 0 ? ` · ${intFmt.format(data.fleetAccuracy.gamingFlagsCaught)} gaming pattern${data.fleetAccuracy.gamingFlagsCaught === 1 ? "" : "s"} flagged` : ""}`
+                ? // #8829: a bare accuracy scalar at unstated coverage is gameable (holding more raises it), so
+                  // the tile names the coverage it was earned at whenever the backend supplies it.
+                  `merge/close calls confirmed by outcome${data.fleetAccuracy.coveragePct != null ? ` · at ${data.fleetAccuracy.coveragePct}% coverage` : ""} · ${intFmt.format(data.fleetAccuracy.instanceCount)} self-hosted instance${data.fleetAccuracy.instanceCount === 1 ? "" : "s"}${data.fleetAccuracy.gamingFlagsCaught > 0 ? ` · ${intFmt.format(data.fleetAccuracy.gamingFlagsCaught)} gaming pattern${data.fleetAccuracy.gamingFlagsCaught === 1 ? "" : "s"} flagged` : ""}`
                 : totals.reversed > 0
                   ? `${intFmt.format(totals.reversed)} human-reversed`
                   : "reversal-grounded"

src/openapi/schemas.ts

@@ -134,6 +134,13 @@ export const PublicStatsSchema = z
      *  accuracyPct is null until at least one registered instance clears the fleet's own minimum-volume bar. */
     fleetAccuracy: z.object({
       accuracyPct: z.number().nullable(),
+      accuracyCiPct: z.object({ lo: z.number(), hi: z.number() }).nullable(),
+      mergePrecisionPct: z.number().nullable(),
+      mergePrecisionCiPct: z.object({ lo: z.number(), hi: z.number() }).nullable(),
+      closePrecisionPct: z.number().nullable(),
+      closePrecisionCiPct: z.object({ lo: z.number(), hi: z.number() }).nullable(),
+      coveragePct: z.number().nullable(),
+      decidedCount: z.number(),
       instanceCount: z.number(),
       windowDays: z.number(),
       gamingFlagsCaught: z.number(),

src/orb/analytics.ts

@@ -76,6 +76,9 @@ export interface InstanceMetrics {
    *  copycat, review-nag, screenshot-table, linked-issue hard rule). Reported so the volume of policy actions
    *  stays visible instead of vanishing from every metric. */
   policyActions: number;
+  /** #8829: the raw confusion counts behind the ratios above, so callers can POOL across instances and put a
+   *  real interval on a published proportion. A ratio alone cannot be pooled (Simpson) or intervalled. */
+  counts: { mergeVerdicts: number; mergeConfirmed: number; closeVerdicts: number; closeConfirmed: number; holds: number };
 }
 
 /** #2350: one self-hosted instance whose combined volume/precision/reversal-rate pattern looks like it is
@@ -106,6 +109,11 @@ export interface FleetAnalytics {
     decisionAccuracy: number | null;
     cycleP50Ms: number | null;
     cycleP95Ms: number | null;
+    /** #8829: confusion counts POOLED over eligible instances. Medians are robust to a bad contributor but
+     *  cannot carry a sample size or an interval; the pooled counts can. With one registered instance (the
+     *  fleet today) pooled and median views coincide. Coverage = verdicts / (verdicts + holds): the share of
+     *  quality-scorable signals the gate actually decided — policy actions are enforcement, outside both. */
+    pooled: { mergeVerdicts: number; mergeConfirmed: number; closeVerdicts: number; closeConfirmed: number; holds: number; policyActions: number; coverage: number | null };
   };
   instances: InstanceMetrics[];
   outliers: Array<{ instanceId: string; metric: string; value: number; fleetMedian: number }>;
@@ -129,6 +137,21 @@ export function percentile(sorted: number[], p: number): number | null {
   return sorted[idx]!;
 }
 
+/** #8829: Wilson score interval for a binomial proportion — the interval every published accuracy/precision
+ *  figure must carry. Wilson, never Wald: the Wald interval degenerates near p→0/1 (exactly where a gate
+ *  metric lives — at 59/60 confirmed Wald claims impossible certainty, Wilson stays honest), and Wilson never
+ *  leaves [0,1]. z defaults to 1.96 (95%). Returns null for zero trials — "no data" must render as no claim,
+ *  never as a fabricated interval. PURE. */
+export function wilsonInterval(successes: number, trials: number, z = 1.96): { lo: number; hi: number } | null {
+  if (trials <= 0) return null;
+  const p = successes / trials;
+  const z2 = z * z;
+  const denom = 1 + z2 / trials;
+  const center = (p + z2 / (2 * trials)) / denom;
+  const half = (z * Math.sqrt((p * (1 - p)) / trials + z2 / (4 * trials * trials))) / denom;
+  return { lo: Math.max(0, center - half), hi: Math.min(1, center + half) };
+}
+
 /** Fold the confusion-matrix cells for one instance into accuracy metrics (reversals count as the gate
  *  being wrong: a reverted merge is a false positive; a reopened OR superseded close is a false negative —
  *  `superseded` (#8166) is the one-shot culture's dominant "bot was wrong" shape: the closed PR's work later
@@ -174,10 +197,15 @@ export function foldInstance(instanceId: string, cells: Cell[]): InstanceMetrics
     reversalRate: reversals / decided, // decided ≥ 1 (the instance has at least one cell)
     decisionAccuracy: verdicts > 0 ? (mergeConfirmed + closeConfirmed) / verdicts : null,
     policyActions,
+    counts: { mergeVerdicts: wouldMerge, mergeConfirmed, closeVerdicts: wouldClose, closeConfirmed, holds: decided - verdicts - policyActions },
   };
 }
 
 /** Compute fleet calibration analytics over the collected orb_signals within the window. Fail-safe → empty. */
+function emptyPooled(): FleetAnalytics["fleet"]["pooled"] {
+  return { mergeVerdicts: 0, mergeConfirmed: 0, closeVerdicts: 0, closeConfirmed: 0, holds: 0, policyActions: 0, coverage: null };
+}
+
 export async function computeFleetAnalytics(env: Env, opts: { windowDays?: number } = {}): Promise<FleetAnalytics> {
   const windowDays = Number.isFinite(opts.windowDays) && (opts.windowDays as number) > 0 ? Math.min(opts.windowDays as number, 365) : 90;
   // Date-only cutoff (like computeGateEval) so it compares correctly whether received_at is ISO ('…T…Z')
@@ -213,7 +241,7 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
     const reg = await env.DB.prepare(`SELECT instance_id FROM orb_instances WHERE registered = 1`).all<{ instance_id: string }>();
     registered = new Set((reg.results ?? []).map((r) => r.instance_id));
   } catch {
-    return { windowDays, instanceCount: 0, fleet: { mergePrecision: null, closePrecision: null, fpRate: null, reversalRate: null, decisionAccuracy: null, cycleP50Ms: null, cycleP95Ms: null }, instances: [], outliers: [], gamingPatternFlags: [] };
+    return { windowDays, instanceCount: 0, fleet: { mergePrecision: null, closePrecision: null, fpRate: null, reversalRate: null, decisionAccuracy: null, cycleP50Ms: null, cycleP95Ms: null, pooled: emptyPooled() }, instances: [], outliers: [], gamingPatternFlags: [] };
   }
 
   // Group cells by instance, fold each.
@@ -268,6 +296,18 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
     }
   }
 
+  const pooled = emptyPooled();
+  for (const i of eligible) {
+    pooled.mergeVerdicts += i.counts.mergeVerdicts;
+    pooled.mergeConfirmed += i.counts.mergeConfirmed;
+    pooled.closeVerdicts += i.counts.closeVerdicts;
+    pooled.closeConfirmed += i.counts.closeConfirmed;
+    pooled.holds += i.counts.holds;
+    pooled.policyActions += i.policyActions;
+  }
+  const pooledVerdicts = pooled.mergeVerdicts + pooled.closeVerdicts;
+  pooled.coverage = pooledVerdicts + pooled.holds > 0 ? pooledVerdicts / (pooledVerdicts + pooled.holds) : null;
+
   return {
     windowDays,
     instanceCount: eligible.length,
@@ -279,6 +319,7 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
       decisionAccuracy: median(nums((i) => i.decisionAccuracy)),
       cycleP50Ms: percentile(cycle, 50),
       cycleP95Ms: percentile(cycle, 95),
+      pooled,
     },
     instances,
     outliers,

src/review/public-stats.ts

@@ -50,7 +50,7 @@
 // `github_app.pr_public_surface_published` audit events this file's own disposition query reads, so the two sums
 // are over disjoint PR sets and can be added directly.
 import { getOrbGlobalStats } from "../orb/outcomes";
-import { computeFleetAnalytics } from "../orb/analytics";
+import { computeFleetAnalytics, wilsonInterval } from "../orb/analytics";
 import { loadRepoFocusManifest } from "../signals/focus-manifest-loader";
 import { resolveLoopOverSelfRepoFullName } from "../config/loopover-repo-focus-manifest";
 import { errorMessage } from "../utils/json";
@@ -242,6 +242,20 @@ export interface PublicStatsPayload {
    *  computeFleetAnalytics's own minimum-volume bar -- the caller falls back to totals.accuracyPct then. */
   fleetAccuracy: {
     accuracyPct: number | null;
+    /** #8829: a bare scalar at unstated coverage is gameable (raising the hold rate mechanically raises it)
+     *  and, at small samples, unverifiable. Every published figure therefore carries: the per-arm split (a
+     *  wrong merge costs more than a wrong close, so the arms must be separable), the coverage it was earned
+     *  at, its Wilson 95% interval, and the sample size behind it. All null/0 when the fleet has no signal. */
+    accuracyCiPct: { lo: number; hi: number } | null;
+    mergePrecisionPct: number | null;
+    mergePrecisionCiPct: { lo: number; hi: number } | null;
+    closePrecisionPct: number | null;
+    closePrecisionCiPct: { lo: number; hi: number } | null;
+    /** Share of quality-scorable signals the gate decided (verdicts / (verdicts + holds)); policy actions are
+     *  enforcement and sit outside both sides. */
+    coveragePct: number | null;
+    /** merge + close verdicts behind accuracyPct — the denominator a reader needs to judge the claim. */
+    decidedCount: number;
     instanceCount: number;
     windowDays: number;
     /** Self-hosted instances currently flagged by computeFleetAnalytics's anti-farming detector
@@ -434,6 +448,15 @@ export async function getPublicStats(
   // for an empty fleet.
   const fleetAccuracyPct =
     fleet.fleet.decisionAccuracy === null ? null : Math.round(fleet.fleet.decisionAccuracy * 1000) / 10;
+  // #8829: intervals/coverage come from the POOLED counts (a median cannot carry a sample size); with one
+  // registered instance — the fleet today — pooled and median views coincide exactly.
+  const pooled = fleet.fleet.pooled;
+  const pooledVerdicts = pooled.mergeVerdicts + pooled.closeVerdicts;
+  const pct = (x: number): number => Math.round(x * 1000) / 10;
+  const ciPct = (successes: number, trials: number): { lo: number; hi: number } | null => {
+    const ci = wilsonInterval(successes, trials);
+    return ci === null ? null : { lo: pct(ci.lo), hi: pct(ci.hi) };
+  };
 
   const reviewed = reviewedOf(totals);
   const w = weeklyRows[0] ?? { reviewed: 0, merged: 0 };
@@ -461,6 +484,13 @@ export async function getPublicStats(
     byProject,
     fleetAccuracy: {
       accuracyPct: fleetAccuracyPct,
+      accuracyCiPct: ciPct(pooled.mergeConfirmed + pooled.closeConfirmed, pooledVerdicts),
+      mergePrecisionPct: pooled.mergeVerdicts > 0 ? pct(pooled.mergeConfirmed / pooled.mergeVerdicts) : null,
+      mergePrecisionCiPct: ciPct(pooled.mergeConfirmed, pooled.mergeVerdicts),
+      closePrecisionPct: pooled.closeVerdicts > 0 ? pct(pooled.closeConfirmed / pooled.closeVerdicts) : null,
+      closePrecisionCiPct: ciPct(pooled.closeConfirmed, pooled.closeVerdicts),
+      coveragePct: pooled.coverage === null ? null : pct(pooled.coverage),
+      decidedCount: pooledVerdicts,
       instanceCount: fleet.instanceCount,
       windowDays: fleet.windowDays,
       gamingFlagsCaught: fleet.gamingPatternFlags.length,

test/unit/orb-analytics.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { computeFleetAnalytics, getFleetHealthSummary, HEALTH_STALE_HOURS } from "../../src/orb/analytics";
+import { computeFleetAnalytics, getFleetHealthSummary, HEALTH_STALE_HOURS, wilsonInterval } from "../../src/orb/analytics";
 import { createTestEnv, TestD1Database } from "../helpers/d1";
 
 let seq = 0;
@@ -28,6 +28,32 @@ async function register(env: Env, ...ids: string[]): Promise<void> {
   }
 }
 
+describe("wilsonInterval (#8829)", () => {
+  it("stays inside [0,1] and stays honest at the p->1 edge where Wald degenerates", () => {
+    const ci = wilsonInterval(59, 60)!;
+    expect(ci.hi).toBeLessThanOrEqual(1);
+    expect(ci.lo).toBeLessThan(59 / 60); // a real lower bound, not a point mass
+    expect(ci.hi).toBeGreaterThan(59 / 60);
+    const perfect = wilsonInterval(60, 60)!;
+    expect(perfect.hi).toBeCloseTo(1, 10);
+    expect(perfect.lo).toBeGreaterThan(0.9);
+    expect(perfect.lo).toBeLessThan(1); // 60/60 does NOT prove p=1 — that's the whole point
+  });
+
+  it("p->0 edge mirrors p->1, and more trials tighten the interval", () => {
+    const zero = wilsonInterval(0, 20)!;
+    expect(zero.lo).toBe(0);
+    expect(zero.hi).toBeGreaterThan(0); // 0/20 does not prove p=0
+    const small = wilsonInterval(8, 10)!;
+    const large = wilsonInterval(800, 1000)!;
+    expect(large.hi - large.lo).toBeLessThan(small.hi - small.lo);
+  });
+
+  it("zero trials -> null: no data is no claim, never a fabricated interval", () => {
+    expect(wilsonInterval(0, 0)).toBeNull();
+  });
+});
+
 describe("computeFleetAnalytics()", () => {
   it("empty store → zeroed report (and a custom/clamped window)", async () => {
     const env = createTestEnv();
@@ -152,6 +178,27 @@ describe("computeFleetAnalytics()", () => {
     expect(inst.reversalRate).toBeCloseTo(2 / 5);
   });
 
+  it("#8829: pooled counts sum across ELIGIBLE instances and coverage counts holds but not policy actions", async () => {
+    const env = createTestEnv();
+    await signals(env, "a", 6, { verdict: "merge", outcome: "merged" });
+    await signals(env, "a", 4, { verdict: "hold", outcome: "merged" });
+    await signals(env, "b", 4, { verdict: "close", outcome: "closed" });
+    await signals(env, "b", 1, { verdict: "close", outcome: "merged" });
+    await signals(env, "b", 5, { verdict: "close", outcome: "closed", bucket: "policy_action" });
+    await signals(env, "stranger", 9, { verdict: "merge", outcome: "closed" }); // unregistered — never pooled
+    await register(env, "a", "b");
+    const fleet = (await computeFleetAnalytics(env)).fleet;
+    expect(fleet.pooled).toEqual({
+      mergeVerdicts: 6,
+      mergeConfirmed: 6,
+      closeVerdicts: 5,
+      closeConfirmed: 4,
+      holds: 4,
+      policyActions: 5,
+      coverage: 11 / 15, // (6+5) verdicts over verdicts+holds; the 5 policy actions sit outside both
+    });
+  });
+
   it("null precision when an instance made no merge verdicts", async () => {
     const env = createTestEnv();
     await signals(env, "inst1", 5, { verdict: "close", outcome: "closed" });