diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json index d69108685..47177df11 100644 --- a/apps/loopover-ui/public/openapi.json +++ b/apps/loopover-ui/public/openapi.json @@ -609,7 +609,8 @@ "nullable": true }, "decidedCount": { - "type": "number" + "type": "number", + "nullable": true }, "guaranteed": { "type": "object", @@ -687,6 +688,13 @@ "gamingFlagsCaught": { "type": "number", "nullable": true + }, + "basis": { + "type": "string", + "enum": [ + "fleet", + "single_instance_self_report" + ] } }, "required": [ @@ -700,6 +708,7 @@ "decidedCount", "guaranteed", "instanceCount", + "basis", "windowDays", "gamingFlagsCaught" ] diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index 0a2d7af7a..ed3d7263a 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -149,12 +149,20 @@ export const PublicStatsSchema = z closePrecisionPct: z.number().nullable(), closePrecisionCiPct: z.object({ lo: z.number(), hi: z.number() }).nullable(), coveragePct: z.number().nullable(), - decidedCount: z.number(), + // #9168: nullable because the pooled COUNT is withheld at 1 < instanceCount < FLEET_FRAMING_MIN_INSTANCES + // — at exactly two instances it isolates the other participant's decision volume by subtraction, since + // this deployment's own volume is already public. Rates stay published at every n. + decidedCount: z.number().nullable(), guaranteed: z.object({ close: z.object({ alpha: z.number(), lambda: z.number(), aiJudgedCoveragePct: z.number(), n: z.number(), backfilledPct: z.number().nullable() }).nullable(), merge: z.object({ alpha: z.number(), lambda: z.number(), aiJudgedCoveragePct: z.number(), n: z.number(), backfilledPct: z.number().nullable() }).nullable(), }), instanceCount: z.number(), + // #9168: whether these figures are a fleet aggregate or one operator's self-report. Below + // FLEET_FRAMING_MIN_INSTANCES a median is not robust to a single bad contributor and the anti-farming + // detector cannot fire, so "fleet" would overclaim — the numbers are real either way, the label is what + // stops a reader treating one party's self-report as corroboration of that party's own guarantee. + basis: z.enum(["fleet", "single_instance_self_report"]), windowDays: z.number(), gamingFlagsCaught: z.number().nullable(), }), diff --git a/src/orb/analytics.ts b/src/orb/analytics.ts index 0ebe7488c..9e6fb7961 100644 --- a/src/orb/analytics.ts +++ b/src/orb/analytics.ts @@ -40,6 +40,13 @@ const GAMING_REVERSAL_RATIO = 0.5; // reversalRate below this fraction of the fl // data point" below this floor, so it does not run at all rather than publish a structurally-guaranteed zero // as if it were a clean bill of health. export const GAMING_MIN_ELIGIBLE = 3; +// #9168: the same floor, for the same reason, applied to the FRAMING rather than the detector. Below this +// many eligible instances a "fleet" aggregate is not one: at n=1 the pooled counts ARE the sole operator's +// own counts, and a median of one is that one value, so the robustness the median is chosen for does no work +// (at n=2 a median is just the mean of two). Publishing those numbers under fleet framing invites a reader to +// treat one party's self-report as independent corroboration of that same party's guarantee. The numbers stay +// published — they are real — but `basis` says which of the two they are. +export const FLEET_FRAMING_MIN_INSTANCES = GAMING_MIN_ELIGIBLE; // #9068: an instance's reversalRate is a fraction of ALL its decided signals, so a genuinely well-run fleet // commonly has a fleet-median reversalRate of exactly 0 — `reversalRate < 0 * GAMING_REVERSAL_RATIO` can never // be true, so the "low reversal" conjunct (and therefore the whole flag) was structurally unfireable whenever @@ -149,6 +156,12 @@ export interface FleetAnalytics { * run yet" — this disambiguates, so a public surface can publish null ("not enough instances to compare") * instead of a structurally-guaranteed zero presented as a positive safety signal. */ gamingDetectionEligible: boolean; + /** #9168: whether there are enough eligible instances for "fleet" to mean anything + * (eligible.length >= FLEET_FRAMING_MIN_INSTANCES). Separate from the numbers themselves, which stay + * published either way — this says whether they are a fleet aggregate or one operator's self-report, so a + * public surface can label them honestly instead of letting fleet framing imply corroboration that a + * single-instance sample cannot provide. */ + fleetFramingEligible: boolean; } function median(xs: number[]): number | null { @@ -280,6 +293,10 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe outliers: [], gamingPatternFlags: [], gamingDetectionEligible: false, + // Fails closed with the rest of this fallback: if the fleet tables cannot be read, we certainly cannot + // claim a fleet aggregate. instanceCount is 0 here, so the public surface labels it a self-report and + // publishes the nulls it already had. + fleetFramingEligible: false, }; } @@ -292,8 +309,15 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe } const instances = [...byInstance.entries()].map(([id, cs]) => foldInstance(id, cs)).sort((a, b) => a.instanceId.localeCompare(b.instanceId)); - // Fleet = median across REGISTERED instances with enough volume (robust to a single bad contributor and - // to unregistered/untrusted senders — registration is the fleet's trust anchor). + // Fleet = median across REGISTERED instances with enough volume. Registration is the fleet's trust anchor + // (open ingest stores everyone's signals; a stranger cannot move calibration until a human opts them in). + // + // #9168, on the OTHER property this median is often credited with: robustness to a single bad contributor + // requires n >= 3, and this comment used to claim it unconditionally. At n=1 the median IS that instance's + // value; at n=2 it is the mean of the two, which a single bad contributor moves by half its error. The + // eligible count is therefore surfaced as `instanceCount` and gates both the gaming detector + // (GAMING_MIN_ELIGIBLE) and the published framing (FLEET_FRAMING_MIN_INSTANCES) rather than being left for + // a reader to infer. const eligible = instances.filter((i) => i.decided >= MIN_DECIDED && registered.has(i.instanceId)); const eligibleIds = new Set(eligible.map((i) => i.instanceId)); const cycle = cycleRows.filter((r) => eligibleIds.has(r.instance_id)).map((r) => r.ms); @@ -375,6 +399,7 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe outliers, gamingPatternFlags, gamingDetectionEligible, + fleetFramingEligible: eligible.length >= FLEET_FRAMING_MIN_INSTANCES, }; } diff --git a/src/review/public-stats.ts b/src/review/public-stats.ts index 49a749beb..117752cdd 100644 --- a/src/review/public-stats.ts +++ b/src/review/public-stats.ts @@ -256,7 +256,9 @@ export interface PublicStatsPayload { * enforcement and sit outside both sides. */ coveragePct: number | null; /** merge + close verdicts behind accuracyPct — the denominator a reader needs to judge the claim. */ - decidedCount: number; + /** null when 1 < instanceCount < FLEET_FRAMING_MIN_INSTANCES — see the k-anonymity note at the assignment + * site. A rate is publishable at any n; a pooled COUNT isolates one participant by subtraction at n=2. */ + decidedCount: number | null; /** #8835: the live finite-sample guarantees, per arm, when a registered instance publishes one — * "P(wrong | acted) ≤ alpha at aiJudgedCoveragePct" with the certification's own sample size. Null arms * mean no guarantee is currently live (insufficient labels, or the instrument retracted it). @@ -273,6 +275,13 @@ export interface PublicStatsPayload { merge: { alpha: number; lambda: number; aiJudgedCoveragePct: number; n: number; backfilledPct: number | null } | null; }; instanceCount: number; + /** #9168: which of two things these figures are. `"fleet"` means instanceCount >= FLEET_FRAMING_MIN_INSTANCES, + * the point at which a median is robust to a single bad contributor and the anti-farming detector can + * fire. `"single_instance_self_report"` means they are ONE operator's own outcomes — real, and disclosed + * as such, but not independent corroboration of anything that operator also publishes (the risk-control + * guarantee in `guaranteed` is calibrated by the same instance, so fleet framing would invite a reader + * to double-count one source as two). The numbers are published either way; only the claim changes. */ + basis: "fleet" | "single_instance_self_report"; windowDays: number; /** #9068: self-hosted instances flagged by computeFleetAnalytics's anti-farming detector * (gamingPatternFlags, src/orb/analytics.ts). null (not 0) when the fleet has fewer than @@ -577,9 +586,17 @@ export async function getPublicStats( closePrecisionPct: pooled.closeVerdicts > 0 ? pct(pooled.closeConfirmed / pooled.closeVerdicts) : null, closePrecisionCiPct: ciPct(pooled.closeConfirmed, pooled.closeVerdicts), coveragePct: pooled.coverage === null ? null : pct(pooled.coverage), - decidedCount: pooledVerdicts, + // #9168 (k-anonymity): the pooled count is a plain SUM over eligible instances, and this deployment's own + // volume is already public via byProject/own-ledger totals. So at exactly 2 instances a reader recovers + // the OTHER instance's decision volume by subtraction — and for a hosted tenant that volume is a business + // metric (how many PRs they ship, how many get closed), not ours to publish. Suppressed in that window. + // At n=1 there is nothing to subtract (the sum IS this deployment's own, already-public figure), and at + // n >= FLEET_FRAMING_MIN_INSTANCES the sum no longer isolates any single participant. RATES are safe at + // every n — a proportion carries no volume — so only the count is withheld. + decidedCount: fleet.instanceCount > 1 && !fleet.fleetFramingEligible ? null : pooledVerdicts, guaranteed, instanceCount: fleet.instanceCount, + basis: fleet.fleetFramingEligible ? "fleet" : "single_instance_self_report", windowDays: fleet.windowDays, gamingFlagsCaught: fleet.gamingDetectionEligible ? fleet.gamingPatternFlags.length : null, }, diff --git a/test/unit/orb-analytics.test.ts b/test/unit/orb-analytics.test.ts index f78eff8c5..cce322270 100644 --- a/test/unit/orb-analytics.test.ts +++ b/test/unit/orb-analytics.test.ts @@ -543,3 +543,69 @@ describe("getFleetHealthSummary() (#4933)", () => { expect(await getFleetHealthSummary(env)).toEqual({ healthyCount: 1, unhealthyCount: 0, unknownCount: 0, totalCount: 1 }); }); }); + +// #9168: `fleet` is a CLAIM, and below a few instances it is not a true one — a median of one is that one +// value, a median of two is their mean, and the anti-farming detector cannot fire against a median an +// instance is itself half of. The numbers stay published (they are real); `fleetFramingEligible` is what lets +// the public surface say which of the two things they are, instead of letting fleet framing imply +// corroboration a single-instance sample cannot provide. +describe("fleetFramingEligible (#9168)", () => { + it("REGRESSION: one registered instance is NOT fleet-framed — the n=1 shape live in production today", async () => { + // The live payload at the time of writing: instanceCount 1, real numbers, published under fleet framing + // alongside a risk-control guarantee calibrated by that very same instance. + const env = createTestEnv(); + await register(env, "solo"); + await signals(env, "solo", 20, { verdict: "merge", outcome: "merged" }); + const a = await computeFleetAnalytics(env); + expect(a.instanceCount).toBe(1); + expect(a.fleetFramingEligible).toBe(false); + // ...and the figures themselves are untouched: this gates the LABEL, never the measurement. + expect(a.fleet.mergePrecision).not.toBeNull(); + expect(a.fleet.pooled.mergeVerdicts).toBe(20); + }); + + it("two instances are still not fleet-framed — a median of two is their mean, with no robustness", async () => { + const env = createTestEnv(); + await register(env, "a", "b"); + await signals(env, "a", 10, { verdict: "merge", outcome: "merged" }); + await signals(env, "b", 10, { verdict: "merge", outcome: "merged" }); + const a = await computeFleetAnalytics(env); + expect(a.instanceCount).toBe(2); + expect(a.fleetFramingEligible).toBe(false); + }); + + it("INVARIANT: three eligible instances clear the floor, and it moves in lockstep with the gaming detector", async () => { + const env = createTestEnv(); + await register(env, "a", "b", "c"); + for (const id of ["a", "b", "c"]) await signals(env, id, 10, { verdict: "merge", outcome: "merged" }); + const a = await computeFleetAnalytics(env); + expect(a.instanceCount).toBe(3); + expect(a.fleetFramingEligible).toBe(true); + // Deliberately the SAME floor: the n at which a median becomes robust is the n at which "this far above + // the median" becomes satisfiable. Pinning them together stops one being moved without the other. + expect(a.gamingDetectionEligible).toBe(true); + }); + + it("INVARIANT: the floor counts ELIGIBLE instances — volume and registration both gate it", async () => { + const env = createTestEnv(); + await register(env, "a", "b", "c"); + await signals(env, "a", 10); + await signals(env, "b", 10); + await signals(env, "c", 1); // under MIN_DECIDED, so ineligible despite being registered + await signals(env, "unregistered", 10); // volume, but never opted in + const a = await computeFleetAnalytics(env); + expect(a.instanceCount).toBe(2); + expect(a.fleetFramingEligible).toBe(false); + }); + + it("INVARIANT: an unreadable fleet fails CLOSED — no fleet claim when the tables cannot be read", async () => { + const env = createTestEnv(); + (env.DB as unknown as { prepare: () => never }).prepare = () => { + throw new Error("d1 unavailable"); + }; + const a = await computeFleetAnalytics(env); + expect(a.instanceCount).toBe(0); + expect(a.fleetFramingEligible).toBe(false); + expect(a.gamingDetectionEligible).toBe(false); + }); +}); diff --git a/test/unit/public-stats.test.ts b/test/unit/public-stats.test.ts index 97fc270a2..ac36f6b3f 100644 --- a/test/unit/public-stats.test.ts +++ b/test/unit/public-stats.test.ts @@ -217,6 +217,10 @@ describe("getPublicStats — live aggregate over the review ledger", () => { coveragePct: null, decidedCount: 0, instanceCount: 0, + // #9168: zero registered instances is not a fleet, so the block never claims to be one. Note this is + // NOT the k-anonymity case -- decidedCount stays 0 (a real, non-identifying figure) because at n<=1 + // there is no second participant whose volume subtraction could isolate. + basis: "single_instance_self_report", windowDays: 90, gamingFlagsCaught: null, // #9068: fewer than GAMING_MIN_ELIGIBLE eligible instances -- detector didn't run guaranteed: { close: null, merge: null }, @@ -484,6 +488,75 @@ describe("getPublicStats — live aggregate over the review ledger", () => { expect(out.totals.accuracyPct).toBe(0); // 1 - 1/1 }); + // #9168: the block is published under "fleet" framing next to a risk-control guarantee calibrated by the + // same instance, which invites a reader to read one party's self-report as two independent sources. The + // numbers are real and stay published; `basis` is what stops the overclaim, and the pooled COUNT is + // withheld in the one window where it isolates another participant. + describe("fleetAccuracy basis and pooled-count k-anonymity (#9168)", () => { + /** N registered instances, each with `per` confirmed merge verdicts (well clear of MIN_DECIDED). */ + async function fleetOf(env: Env, instanceIds: string[], per = 8): Promise { + for (const id of instanceIds) { + await env.DB.prepare("INSERT INTO orb_instances (instance_id, registered) VALUES (?, 1)").bind(id).run(); + for (let i = 0; i < per; i += 1) { + await env.DB + .prepare( + `INSERT INTO orb_signals (instance_id, repo_hash, pr_hash, gate_verdict, outcome, reversal_flag) + VALUES (?, ?, ?, 'merge', 'merged', 'none')`, + ) + .bind(id, "repo-hash", `${id}-pr-${i}`) + .run(); + } + } + } + + it("REGRESSION: at one instance it is labelled a self-report, NOT a fleet — the shape live in production", async () => { + const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" }); + await fleetOf(env, ["inst-1"]); + const out = await getPublicStats(env, NOW); + expect(out.fleetAccuracy.instanceCount).toBe(1); + expect(out.fleetAccuracy.basis).toBe("single_instance_self_report"); + // The count is NOT withheld at n=1: the pooled sum IS this deployment's own volume, which is already + // public via byProject, so there is no second party for subtraction to isolate. + expect(out.fleetAccuracy.decidedCount).toBe(8); + expect(out.fleetAccuracy.mergePrecisionPct).toBe(100); + }); + + it("REGRESSION: at exactly two instances the pooled COUNT is withheld — it isolates the other party by subtraction", async () => { + const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" }); + await fleetOf(env, ["inst-1", "inst-2"]); + const out = await getPublicStats(env, NOW); + expect(out.fleetAccuracy.instanceCount).toBe(2); + expect(out.fleetAccuracy.basis).toBe("single_instance_self_report"); + // The whole point: our own volume is public, so `pooled - ours` would hand a reader the other + // instance's decision volume exactly — a hosted tenant's business metric, not ours to publish. + expect(out.fleetAccuracy.decidedCount).toBeNull(); + // RATES stay published at every n: a proportion carries no volume. + expect(out.fleetAccuracy.mergePrecisionPct).toBe(100); + expect(out.fleetAccuracy.accuracyPct).toBe(100); + }); + + it("at three instances it is a genuine fleet: framing, pooled count, and the gaming detector all switch on together", async () => { + const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" }); + await fleetOf(env, ["inst-1", "inst-2", "inst-3"]); + const out = await getPublicStats(env, NOW); + expect(out.fleetAccuracy.instanceCount).toBe(3); + expect(out.fleetAccuracy.basis).toBe("fleet"); + // Restored once the sum no longer isolates any single participant. + expect(out.fleetAccuracy.decidedCount).toBe(24); + // #9068's detector shares the same floor, so a "fleet" label always comes with a real (0, not null) + // gaming count -- a reader never sees fleet framing next to "the detector could not run". + expect(out.fleetAccuracy.gamingFlagsCaught).toBe(0); + }); + + it("INVARIANT: an empty fleet is a self-report with a 0 count, never a fleet claim", async () => { + const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" }); + const out = await getPublicStats(env, NOW); + expect(out.fleetAccuracy.instanceCount).toBe(0); + expect(out.fleetAccuracy.basis).toBe("single_instance_self_report"); + expect(out.fleetAccuracy.decidedCount).toBe(0); + }); + }); + it("REGRESSION (#fairness-analytics): the headline accuracy prefers live fleet data once a registered instance clears the volume bar", async () => { const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" }); const db = env.DB;