From 745fbf26d5e382ea82ed81a5e7c9c3355712eb43 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 26 Jul 2026 03:46:25 -0700 Subject: [PATCH] feat(orb): publish per-arm precision, coverage, and Wilson intervals with the fleet accuracy A bare accuracy scalar at unstated coverage is gameable -- because holds are (correctly) excluded from the denominator, raising the hold rate mechanically raises the published number -- and at small samples it is unverifiable: the interval around it is wider than the difference the tuning work is trying to demonstrate. Closes #8829. Part of epic #8828 (Phase 1 -- measurement precedes labels precedes mechanism). - analytics: InstanceMetrics carries the raw confusion counts (ratios cannot be pooled or intervalled); FleetAnalytics.fleet gains counts POOLED over eligible instances plus coverage = verdicts / (verdicts + holds), with policy actions outside both sides; wilsonInterval() as the one shared interval primitive -- Wilson, never Wald, which degenerates exactly where a gate metric lives - public-stats: fleetAccuracy publishes the per-arm split (a wrong merge costs more than a wrong close, so the arms must be separable), Wilson 95% intervals on all three proportions, coveragePct, and decidedCount -- the denominator a reader needs to judge the claim - UI: the hero hint names the coverage the figure was earned at; older backends omit the field and the tile degrades gracefully - openapi regenerated --- apps/loopover-ui/public/openapi.json | 70 +++++++++++++++++++ .../site/proof-of-power-stats-model.ts | 9 +++ .../site/proof-of-power-stats.test.tsx | 4 +- .../components/site/proof-of-power-stats.tsx | 4 +- src/openapi/schemas.ts | 7 ++ src/orb/analytics.ts | 43 +++++++++++- src/review/public-stats.ts | 32 ++++++++- test/unit/orb-analytics.test.ts | 49 ++++++++++++- test/unit/public-stats.test.ts | 37 +++++++++- 9 files changed, 246 insertions(+), 9 deletions(-) diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json index 300decb9ed..3e9e63129f 100644 --- a/apps/loopover-ui/public/openapi.json +++ b/apps/loopover-ui/public/openapi.json @@ -568,10 +568,80 @@ }, "gamingFlagsCaught": { "type": "number" + }, + "accuracyCiPct": { + "type": "object", + "nullable": true, + "properties": { + "lo": { + "type": "number" + }, + "hi": { + "type": "number" + } + }, + "required": [ + "lo", + "hi" + ] + }, + "mergePrecisionPct": { + "type": "number", + "nullable": true + }, + "mergePrecisionCiPct": { + "type": "object", + "nullable": true, + "properties": { + "lo": { + "type": "number" + }, + "hi": { + "type": "number" + } + }, + "required": [ + "lo", + "hi" + ] + }, + "closePrecisionPct": { + "type": "number", + "nullable": true + }, + "closePrecisionCiPct": { + "type": "object", + "nullable": true, + "properties": { + "lo": { + "type": "number" + }, + "hi": { + "type": "number" + } + }, + "required": [ + "lo", + "hi" + ] + }, + "coveragePct": { + "type": "number", + "nullable": true + }, + "decidedCount": { + "type": "number" } }, "required": [ "accuracyPct", + "accuracyCiPct", + "mergePrecisionPct", + "mergePrecisionCiPct", + "closePrecisionPct", + "closePrecisionCiPct", + "coveragePct", + "decidedCount", "instanceCount", "windowDays", "gamingFlagsCaught" diff --git a/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts b/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts index 6b75fa41d7..f2004785da 100644 --- a/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts +++ b/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts @@ -30,6 +30,15 @@ export type PublicStats = { * (a frozen own-ledger snapshot) whenever instanceCount > 0. See public-stats.ts's PublicStatsPayload. */ fleetAccuracy: { accuracyPct: number | null; + /** #8829 fields -- optional-chained at the render site: an older backend simply omits them and the tile + * degrades to the bare figure rather than throwing. */ + accuracyCiPct?: { lo: number; hi: number } | null; + mergePrecisionPct?: number | null; + mergePrecisionCiPct?: { lo: number; hi: number } | null; + closePrecisionPct?: number | null; + closePrecisionCiPct?: { lo: number; hi: number } | null; + coveragePct?: number | null; + decidedCount?: number; instanceCount: number; windowDays: number; gamingFlagsCaught: number; diff --git a/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx b/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx index 74fc5a9f9a..45d760b709 100644 --- a/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx +++ b/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx @@ -206,7 +206,7 @@ describe("ProofOfPowerStats", () => { durationMs: 1, data: { ...PAYLOAD, - fleetAccuracy: { accuracyPct: 80, instanceCount: 3, windowDays: 90, gamingFlagsCaught: 2 }, + fleetAccuracy: { accuracyPct: 80, coveragePct: 64.3, instanceCount: 3, windowDays: 90, gamingFlagsCaught: 2 }, }, }); renderWithClient(); @@ -217,7 +217,7 @@ describe("ProofOfPowerStats", () => { // rather than the retired "reversal-grounded" formula the surface no longer publishes. expect( screen.getByText( - "merge/close calls confirmed by outcome · 3 self-hosted instances · 2 gaming patterns flagged", + "merge/close calls confirmed by outcome · at 64.3% coverage · 3 self-hosted instances · 2 gaming patterns flagged", ), ).toBeTruthy(); // No fleet-accuracy trend exists yet -- the tile's sparkline is omitted rather than showing a mismatched one. diff --git a/apps/loopover-ui/src/components/site/proof-of-power-stats.tsx b/apps/loopover-ui/src/components/site/proof-of-power-stats.tsx index 868d0f60ff..3813dc9bef 100644 --- a/apps/loopover-ui/src/components/site/proof-of-power-stats.tsx +++ b/apps/loopover-ui/src/components/site/proof-of-power-stats.tsx @@ -166,7 +166,9 @@ export function ProofOfPowerStats({ className }: { className?: string }) { // (holds excluded -- they're deferrals, not decisions), so say that rather than the old // "reversal-grounded" wording, which described a formula the surface no longer publishes. fleetEligible - ? `merge/close calls confirmed by outcome · ${intFmt.format(data.fleetAccuracy.instanceCount)} self-hosted instance${data.fleetAccuracy.instanceCount === 1 ? "" : "s"}${data.fleetAccuracy.gamingFlagsCaught > 0 ? ` · ${intFmt.format(data.fleetAccuracy.gamingFlagsCaught)} gaming pattern${data.fleetAccuracy.gamingFlagsCaught === 1 ? "" : "s"} flagged` : ""}` + ? // #8829: a bare accuracy scalar at unstated coverage is gameable (holding more raises it), so + // the tile names the coverage it was earned at whenever the backend supplies it. + `merge/close calls confirmed by outcome${data.fleetAccuracy.coveragePct != null ? ` · at ${data.fleetAccuracy.coveragePct}% coverage` : ""} · ${intFmt.format(data.fleetAccuracy.instanceCount)} self-hosted instance${data.fleetAccuracy.instanceCount === 1 ? "" : "s"}${data.fleetAccuracy.gamingFlagsCaught > 0 ? ` · ${intFmt.format(data.fleetAccuracy.gamingFlagsCaught)} gaming pattern${data.fleetAccuracy.gamingFlagsCaught === 1 ? "" : "s"} flagged` : ""}` : totals.reversed > 0 ? `${intFmt.format(totals.reversed)} human-reversed` : "reversal-grounded" diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index 8ed4a4dd89..00884a63d2 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -134,6 +134,13 @@ export const PublicStatsSchema = z * accuracyPct is null until at least one registered instance clears the fleet's own minimum-volume bar. */ fleetAccuracy: z.object({ accuracyPct: z.number().nullable(), + accuracyCiPct: z.object({ lo: z.number(), hi: z.number() }).nullable(), + mergePrecisionPct: z.number().nullable(), + mergePrecisionCiPct: z.object({ lo: z.number(), hi: z.number() }).nullable(), + closePrecisionPct: z.number().nullable(), + closePrecisionCiPct: z.object({ lo: z.number(), hi: z.number() }).nullable(), + coveragePct: z.number().nullable(), + decidedCount: z.number(), instanceCount: z.number(), windowDays: z.number(), gamingFlagsCaught: z.number(), diff --git a/src/orb/analytics.ts b/src/orb/analytics.ts index cb5568e9d8..761a721af8 100644 --- a/src/orb/analytics.ts +++ b/src/orb/analytics.ts @@ -76,6 +76,9 @@ export interface InstanceMetrics { * copycat, review-nag, screenshot-table, linked-issue hard rule). Reported so the volume of policy actions * stays visible instead of vanishing from every metric. */ policyActions: number; + /** #8829: the raw confusion counts behind the ratios above, so callers can POOL across instances and put a + * real interval on a published proportion. A ratio alone cannot be pooled (Simpson) or intervalled. */ + counts: { mergeVerdicts: number; mergeConfirmed: number; closeVerdicts: number; closeConfirmed: number; holds: number }; } /** #2350: one self-hosted instance whose combined volume/precision/reversal-rate pattern looks like it is @@ -106,6 +109,11 @@ export interface FleetAnalytics { decisionAccuracy: number | null; cycleP50Ms: number | null; cycleP95Ms: number | null; + /** #8829: confusion counts POOLED over eligible instances. Medians are robust to a bad contributor but + * cannot carry a sample size or an interval; the pooled counts can. With one registered instance (the + * fleet today) pooled and median views coincide. Coverage = verdicts / (verdicts + holds): the share of + * quality-scorable signals the gate actually decided — policy actions are enforcement, outside both. */ + pooled: { mergeVerdicts: number; mergeConfirmed: number; closeVerdicts: number; closeConfirmed: number; holds: number; policyActions: number; coverage: number | null }; }; instances: InstanceMetrics[]; outliers: Array<{ instanceId: string; metric: string; value: number; fleetMedian: number }>; @@ -129,6 +137,21 @@ export function percentile(sorted: number[], p: number): number | null { return sorted[idx]!; } +/** #8829: Wilson score interval for a binomial proportion — the interval every published accuracy/precision + * figure must carry. Wilson, never Wald: the Wald interval degenerates near p→0/1 (exactly where a gate + * metric lives — at 59/60 confirmed Wald claims impossible certainty, Wilson stays honest), and Wilson never + * leaves [0,1]. z defaults to 1.96 (95%). Returns null for zero trials — "no data" must render as no claim, + * never as a fabricated interval. PURE. */ +export function wilsonInterval(successes: number, trials: number, z = 1.96): { lo: number; hi: number } | null { + if (trials <= 0) return null; + const p = successes / trials; + const z2 = z * z; + const denom = 1 + z2 / trials; + const center = (p + z2 / (2 * trials)) / denom; + const half = (z * Math.sqrt((p * (1 - p)) / trials + z2 / (4 * trials * trials))) / denom; + return { lo: Math.max(0, center - half), hi: Math.min(1, center + half) }; +} + /** Fold the confusion-matrix cells for one instance into accuracy metrics (reversals count as the gate * being wrong: a reverted merge is a false positive; a reopened OR superseded close is a false negative — * `superseded` (#8166) is the one-shot culture's dominant "bot was wrong" shape: the closed PR's work later @@ -174,10 +197,15 @@ export function foldInstance(instanceId: string, cells: Cell[]): InstanceMetrics reversalRate: reversals / decided, // decided ≥ 1 (the instance has at least one cell) decisionAccuracy: verdicts > 0 ? (mergeConfirmed + closeConfirmed) / verdicts : null, policyActions, + counts: { mergeVerdicts: wouldMerge, mergeConfirmed, closeVerdicts: wouldClose, closeConfirmed, holds: decided - verdicts - policyActions }, }; } /** Compute fleet calibration analytics over the collected orb_signals within the window. Fail-safe → empty. */ +function emptyPooled(): FleetAnalytics["fleet"]["pooled"] { + return { mergeVerdicts: 0, mergeConfirmed: 0, closeVerdicts: 0, closeConfirmed: 0, holds: 0, policyActions: 0, coverage: null }; +} + export async function computeFleetAnalytics(env: Env, opts: { windowDays?: number } = {}): Promise { const windowDays = Number.isFinite(opts.windowDays) && (opts.windowDays as number) > 0 ? Math.min(opts.windowDays as number, 365) : 90; // Date-only cutoff (like computeGateEval) so it compares correctly whether received_at is ISO ('…T…Z') @@ -213,7 +241,7 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe const reg = await env.DB.prepare(`SELECT instance_id FROM orb_instances WHERE registered = 1`).all<{ instance_id: string }>(); registered = new Set((reg.results ?? []).map((r) => r.instance_id)); } catch { - return { windowDays, instanceCount: 0, fleet: { mergePrecision: null, closePrecision: null, fpRate: null, reversalRate: null, decisionAccuracy: null, cycleP50Ms: null, cycleP95Ms: null }, instances: [], outliers: [], gamingPatternFlags: [] }; + return { windowDays, instanceCount: 0, fleet: { mergePrecision: null, closePrecision: null, fpRate: null, reversalRate: null, decisionAccuracy: null, cycleP50Ms: null, cycleP95Ms: null, pooled: emptyPooled() }, instances: [], outliers: [], gamingPatternFlags: [] }; } // Group cells by instance, fold each. @@ -268,6 +296,18 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe } } + const pooled = emptyPooled(); + for (const i of eligible) { + pooled.mergeVerdicts += i.counts.mergeVerdicts; + pooled.mergeConfirmed += i.counts.mergeConfirmed; + pooled.closeVerdicts += i.counts.closeVerdicts; + pooled.closeConfirmed += i.counts.closeConfirmed; + pooled.holds += i.counts.holds; + pooled.policyActions += i.policyActions; + } + const pooledVerdicts = pooled.mergeVerdicts + pooled.closeVerdicts; + pooled.coverage = pooledVerdicts + pooled.holds > 0 ? pooledVerdicts / (pooledVerdicts + pooled.holds) : null; + return { windowDays, instanceCount: eligible.length, @@ -279,6 +319,7 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe decisionAccuracy: median(nums((i) => i.decisionAccuracy)), cycleP50Ms: percentile(cycle, 50), cycleP95Ms: percentile(cycle, 95), + pooled, }, instances, outliers, diff --git a/src/review/public-stats.ts b/src/review/public-stats.ts index d3ecf73ff2..5d7fec1c5b 100644 --- a/src/review/public-stats.ts +++ b/src/review/public-stats.ts @@ -50,7 +50,7 @@ // `github_app.pr_public_surface_published` audit events this file's own disposition query reads, so the two sums // are over disjoint PR sets and can be added directly. import { getOrbGlobalStats } from "../orb/outcomes"; -import { computeFleetAnalytics } from "../orb/analytics"; +import { computeFleetAnalytics, wilsonInterval } from "../orb/analytics"; import { loadRepoFocusManifest } from "../signals/focus-manifest-loader"; import { resolveLoopOverSelfRepoFullName } from "../config/loopover-repo-focus-manifest"; import { errorMessage } from "../utils/json"; @@ -242,6 +242,20 @@ export interface PublicStatsPayload { * computeFleetAnalytics's own minimum-volume bar -- the caller falls back to totals.accuracyPct then. */ fleetAccuracy: { accuracyPct: number | null; + /** #8829: a bare scalar at unstated coverage is gameable (raising the hold rate mechanically raises it) + * and, at small samples, unverifiable. Every published figure therefore carries: the per-arm split (a + * wrong merge costs more than a wrong close, so the arms must be separable), the coverage it was earned + * at, its Wilson 95% interval, and the sample size behind it. All null/0 when the fleet has no signal. */ + accuracyCiPct: { lo: number; hi: number } | null; + mergePrecisionPct: number | null; + mergePrecisionCiPct: { lo: number; hi: number } | null; + closePrecisionPct: number | null; + closePrecisionCiPct: { lo: number; hi: number } | null; + /** Share of quality-scorable signals the gate decided (verdicts / (verdicts + holds)); policy actions are + * enforcement and sit outside both sides. */ + coveragePct: number | null; + /** merge + close verdicts behind accuracyPct — the denominator a reader needs to judge the claim. */ + decidedCount: number; instanceCount: number; windowDays: number; /** Self-hosted instances currently flagged by computeFleetAnalytics's anti-farming detector @@ -434,6 +448,15 @@ export async function getPublicStats( // for an empty fleet. const fleetAccuracyPct = fleet.fleet.decisionAccuracy === null ? null : Math.round(fleet.fleet.decisionAccuracy * 1000) / 10; + // #8829: intervals/coverage come from the POOLED counts (a median cannot carry a sample size); with one + // registered instance — the fleet today — pooled and median views coincide exactly. + const pooled = fleet.fleet.pooled; + const pooledVerdicts = pooled.mergeVerdicts + pooled.closeVerdicts; + const pct = (x: number): number => Math.round(x * 1000) / 10; + const ciPct = (successes: number, trials: number): { lo: number; hi: number } | null => { + const ci = wilsonInterval(successes, trials); + return ci === null ? null : { lo: pct(ci.lo), hi: pct(ci.hi) }; + }; const reviewed = reviewedOf(totals); const w = weeklyRows[0] ?? { reviewed: 0, merged: 0 }; @@ -461,6 +484,13 @@ export async function getPublicStats( byProject, fleetAccuracy: { accuracyPct: fleetAccuracyPct, + accuracyCiPct: ciPct(pooled.mergeConfirmed + pooled.closeConfirmed, pooledVerdicts), + mergePrecisionPct: pooled.mergeVerdicts > 0 ? pct(pooled.mergeConfirmed / pooled.mergeVerdicts) : null, + mergePrecisionCiPct: ciPct(pooled.mergeConfirmed, pooled.mergeVerdicts), + closePrecisionPct: pooled.closeVerdicts > 0 ? pct(pooled.closeConfirmed / pooled.closeVerdicts) : null, + closePrecisionCiPct: ciPct(pooled.closeConfirmed, pooled.closeVerdicts), + coveragePct: pooled.coverage === null ? null : pct(pooled.coverage), + decidedCount: pooledVerdicts, instanceCount: fleet.instanceCount, windowDays: fleet.windowDays, gamingFlagsCaught: fleet.gamingPatternFlags.length, diff --git a/test/unit/orb-analytics.test.ts b/test/unit/orb-analytics.test.ts index e434bd9a0e..65f2aee866 100644 --- a/test/unit/orb-analytics.test.ts +++ b/test/unit/orb-analytics.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { computeFleetAnalytics, getFleetHealthSummary, HEALTH_STALE_HOURS } from "../../src/orb/analytics"; +import { computeFleetAnalytics, getFleetHealthSummary, HEALTH_STALE_HOURS, wilsonInterval } from "../../src/orb/analytics"; import { createTestEnv, TestD1Database } from "../helpers/d1"; let seq = 0; @@ -28,6 +28,32 @@ async function register(env: Env, ...ids: string[]): Promise { } } +describe("wilsonInterval (#8829)", () => { + it("stays inside [0,1] and stays honest at the p->1 edge where Wald degenerates", () => { + const ci = wilsonInterval(59, 60)!; + expect(ci.hi).toBeLessThanOrEqual(1); + expect(ci.lo).toBeLessThan(59 / 60); // a real lower bound, not a point mass + expect(ci.hi).toBeGreaterThan(59 / 60); + const perfect = wilsonInterval(60, 60)!; + expect(perfect.hi).toBeCloseTo(1, 10); + expect(perfect.lo).toBeGreaterThan(0.9); + expect(perfect.lo).toBeLessThan(1); // 60/60 does NOT prove p=1 — that's the whole point + }); + + it("p->0 edge mirrors p->1, and more trials tighten the interval", () => { + const zero = wilsonInterval(0, 20)!; + expect(zero.lo).toBe(0); + expect(zero.hi).toBeGreaterThan(0); // 0/20 does not prove p=0 + const small = wilsonInterval(8, 10)!; + const large = wilsonInterval(800, 1000)!; + expect(large.hi - large.lo).toBeLessThan(small.hi - small.lo); + }); + + it("zero trials -> null: no data is no claim, never a fabricated interval", () => { + expect(wilsonInterval(0, 0)).toBeNull(); + }); +}); + describe("computeFleetAnalytics()", () => { it("empty store → zeroed report (and a custom/clamped window)", async () => { const env = createTestEnv(); @@ -152,6 +178,27 @@ describe("computeFleetAnalytics()", () => { expect(inst.reversalRate).toBeCloseTo(2 / 5); }); + it("#8829: pooled counts sum across ELIGIBLE instances and coverage counts holds but not policy actions", async () => { + const env = createTestEnv(); + await signals(env, "a", 6, { verdict: "merge", outcome: "merged" }); + await signals(env, "a", 4, { verdict: "hold", outcome: "merged" }); + await signals(env, "b", 4, { verdict: "close", outcome: "closed" }); + await signals(env, "b", 1, { verdict: "close", outcome: "merged" }); + await signals(env, "b", 5, { verdict: "close", outcome: "closed", bucket: "policy_action" }); + await signals(env, "stranger", 9, { verdict: "merge", outcome: "closed" }); // unregistered — never pooled + await register(env, "a", "b"); + const fleet = (await computeFleetAnalytics(env)).fleet; + expect(fleet.pooled).toEqual({ + mergeVerdicts: 6, + mergeConfirmed: 6, + closeVerdicts: 5, + closeConfirmed: 4, + holds: 4, + policyActions: 5, + coverage: 11 / 15, // (6+5) verdicts over verdicts+holds; the 5 policy actions sit outside both + }); + }); + it("null precision when an instance made no merge verdicts", async () => { const env = createTestEnv(); await signals(env, "inst1", 5, { verdict: "close", outcome: "closed" }); diff --git a/test/unit/public-stats.test.ts b/test/unit/public-stats.test.ts index 7a8281dcee..1923722ab1 100644 --- a/test/unit/public-stats.test.ts +++ b/test/unit/public-stats.test.ts @@ -207,7 +207,19 @@ describe("getPublicStats — live aggregate over the review ledger", () => { ]); expect(out.updatedAt).toBe(out.generatedAt); // No registered self-hosted instances in this fixture -- fleetAccuracy degrades to the "not eligible yet" shape. - expect(out.fleetAccuracy).toEqual({ accuracyPct: null, instanceCount: 0, windowDays: 90, gamingFlagsCaught: 0 }); + expect(out.fleetAccuracy).toEqual({ + accuracyPct: null, + accuracyCiPct: null, + mergePrecisionPct: null, + mergePrecisionCiPct: null, + closePrecisionPct: null, + closePrecisionCiPct: null, + coveragePct: null, + decidedCount: 0, + instanceCount: 0, + windowDays: 90, + gamingFlagsCaught: 0, + }); }); // #1955/#2070: minutesSaved sums per-PR estimates (with MINUTES_SAVED_PER_PR fallback for missing rows) @@ -496,7 +508,19 @@ describe("getPublicStats — live aggregate over the review ledger", () => { const out = await getPublicStats(env, NOW); // 5 merge verdicts, 4 confirmed (the 5th was reverted) -> decisionAccuracy 4/5 -> 80%. - expect(out.fleetAccuracy).toEqual({ accuracyPct: 80, instanceCount: 1, windowDays: 90, gamingFlagsCaught: 0 }); + expect(out.fleetAccuracy).toMatchObject({ accuracyPct: 80, instanceCount: 1, windowDays: 90, gamingFlagsCaught: 0 }); + // #8829: per-arm split, coverage, sample size, and a Wilson interval ride every published figure. + expect(out.fleetAccuracy.mergePrecisionPct).toBe(80); + expect(out.fleetAccuracy.closePrecisionPct).toBeNull(); // no close verdicts in this fixture + expect(out.fleetAccuracy.closePrecisionCiPct).toBeNull(); + expect(out.fleetAccuracy.coveragePct).toBe(100); // no holds in this fixture + expect(out.fleetAccuracy.decidedCount).toBe(5); + const ci = out.fleetAccuracy.accuracyCiPct!; + // Wilson at 4/5 is WIDE (n=5) -- the interval is the honesty the bare 80 lacks. + expect(ci.lo).toBeGreaterThan(30); + expect(ci.lo).toBeLessThan(80); + expect(ci.hi).toBeGreaterThan(80); + expect(ci.hi).toBeLessThanOrEqual(100); }); it("REGRESSION (#8820): the published fleet accuracy scores DECISIONS — holds are excluded and marker-less mispredictions count", async () => { @@ -513,12 +537,19 @@ describe("getPublicStats — live aggregate over the review ledger", () => { }; await signal(6, "merge", "merged", "ok"); // confirmed await signal(2, "merge", "closed", "bad"); // WRONG, and carries no reversal marker + await signal(3, "close", "closed", "cok"); // confirmed closes + await signal(1, "close", "merged", "cbad"); // wrong close await signal(40, "hold", "merged", "hold"); // deferrals — must not enter the denominator const out = await getPublicStats(env, NOW); - // 8 real decisions, 6 confirmed -> 75%. The retired `1 - reversalRate` formula would have published + // 12 real decisions, 9 confirmed -> 75%. The retired `1 - reversalRate` formula would have published // 100% here: zero reversal markers, and 40 holds swamping its denominator. expect(out.fleetAccuracy.accuracyPct).toBe(75); + // #8829: per-arm split + the coverage this figure was earned at (12 verdicts over 52 scorable signals). + expect(out.fleetAccuracy.mergePrecisionPct).toBe(75); + expect(out.fleetAccuracy.closePrecisionPct).toBe(75); + expect(out.fleetAccuracy.coveragePct).toBeCloseTo(23.1, 1); + expect(out.fleetAccuracy.decidedCount).toBe(12); }); it("REGRESSION (#fairness-analytics): surfaces gamingFlagsCaught from computeFleetAnalytics's anti-farming detector", async () => {