fix(public-stats): stop calling a one-instance sample a fleet, and withhold the pooled count at n=2 (#9168)

apps/loopover-ui/public/openapi.json

@@ -609,7 +609,8 @@
                 "nullable": true
               },
               "decidedCount": {
-                "type": "number"
+                "type": "number",
+                "nullable": true
               },
               "guaranteed": {
                 "type": "object",
@@ -687,6 +688,13 @@
               "gamingFlagsCaught": {
                 "type": "number",
                 "nullable": true
+              },
+              "basis": {
+                "type": "string",
+                "enum": [
+                  "fleet",
+                  "single_instance_self_report"
+                ]
               }
             },
             "required": [
@@ -700,6 +708,7 @@
               "decidedCount",
               "guaranteed",
               "instanceCount",
+              "basis",
               "windowDays",
               "gamingFlagsCaught"
             ]

src/openapi/schemas.ts

@@ -149,12 +149,20 @@ export const PublicStatsSchema = z
       closePrecisionPct: z.number().nullable(),
       closePrecisionCiPct: z.object({ lo: z.number(), hi: z.number() }).nullable(),
       coveragePct: z.number().nullable(),
-      decidedCount: z.number(),
+      // #9168: nullable because the pooled COUNT is withheld at 1 < instanceCount < FLEET_FRAMING_MIN_INSTANCES
+      // — at exactly two instances it isolates the other participant's decision volume by subtraction, since
+      // this deployment's own volume is already public. Rates stay published at every n.
+      decidedCount: z.number().nullable(),
       guaranteed: z.object({
         close: z.object({ alpha: z.number(), lambda: z.number(), aiJudgedCoveragePct: z.number(), n: z.number(), backfilledPct: z.number().nullable() }).nullable(),
         merge: z.object({ alpha: z.number(), lambda: z.number(), aiJudgedCoveragePct: z.number(), n: z.number(), backfilledPct: z.number().nullable() }).nullable(),
       }),
       instanceCount: z.number(),
+      // #9168: whether these figures are a fleet aggregate or one operator's self-report. Below
+      // FLEET_FRAMING_MIN_INSTANCES a median is not robust to a single bad contributor and the anti-farming
+      // detector cannot fire, so "fleet" would overclaim — the numbers are real either way, the label is what
+      // stops a reader treating one party's self-report as corroboration of that party's own guarantee.
+      basis: z.enum(["fleet", "single_instance_self_report"]),
       windowDays: z.number(),
       gamingFlagsCaught: z.number().nullable(),
     }),

src/orb/analytics.ts

@@ -40,6 +40,13 @@ const GAMING_REVERSAL_RATIO = 0.5; // reversalRate below this fraction of the fl
 // data point" below this floor, so it does not run at all rather than publish a structurally-guaranteed zero
 // as if it were a clean bill of health.
 export const GAMING_MIN_ELIGIBLE = 3;
+// #9168: the same floor, for the same reason, applied to the FRAMING rather than the detector. Below this
+// many eligible instances a "fleet" aggregate is not one: at n=1 the pooled counts ARE the sole operator's
+// own counts, and a median of one is that one value, so the robustness the median is chosen for does no work
+// (at n=2 a median is just the mean of two). Publishing those numbers under fleet framing invites a reader to
+// treat one party's self-report as independent corroboration of that same party's guarantee. The numbers stay
+// published — they are real — but `basis` says which of the two they are.
+export const FLEET_FRAMING_MIN_INSTANCES = GAMING_MIN_ELIGIBLE;
 // #9068: an instance's reversalRate is a fraction of ALL its decided signals, so a genuinely well-run fleet
 // commonly has a fleet-median reversalRate of exactly 0 — `reversalRate < 0 * GAMING_REVERSAL_RATIO` can never
 // be true, so the "low reversal" conjunct (and therefore the whole flag) was structurally unfireable whenever
@@ -149,6 +156,12 @@ export interface FleetAnalytics {
    *  run yet" — this disambiguates, so a public surface can publish null ("not enough instances to compare")
    *  instead of a structurally-guaranteed zero presented as a positive safety signal. */
   gamingDetectionEligible: boolean;
+  /** #9168: whether there are enough eligible instances for "fleet" to mean anything
+   *  (eligible.length >= FLEET_FRAMING_MIN_INSTANCES). Separate from the numbers themselves, which stay
+   *  published either way — this says whether they are a fleet aggregate or one operator's self-report, so a
+   *  public surface can label them honestly instead of letting fleet framing imply corroboration that a
+   *  single-instance sample cannot provide. */
+  fleetFramingEligible: boolean;
 }
 
 function median(xs: number[]): number | null {
@@ -280,6 +293,10 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
       outliers: [],
       gamingPatternFlags: [],
       gamingDetectionEligible: false,
+      // Fails closed with the rest of this fallback: if the fleet tables cannot be read, we certainly cannot
+      // claim a fleet aggregate. instanceCount is 0 here, so the public surface labels it a self-report and
+      // publishes the nulls it already had.
+      fleetFramingEligible: false,
     };
   }
 
@@ -292,8 +309,15 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
   }
   const instances = [...byInstance.entries()].map(([id, cs]) => foldInstance(id, cs)).sort((a, b) => a.instanceId.localeCompare(b.instanceId));
 
-  // Fleet = median across REGISTERED instances with enough volume (robust to a single bad contributor and
-  // to unregistered/untrusted senders — registration is the fleet's trust anchor).
+  // Fleet = median across REGISTERED instances with enough volume. Registration is the fleet's trust anchor
+  // (open ingest stores everyone's signals; a stranger cannot move calibration until a human opts them in).
+  //
+  // #9168, on the OTHER property this median is often credited with: robustness to a single bad contributor
+  // requires n >= 3, and this comment used to claim it unconditionally. At n=1 the median IS that instance's
+  // value; at n=2 it is the mean of the two, which a single bad contributor moves by half its error. The
+  // eligible count is therefore surfaced as `instanceCount` and gates both the gaming detector
+  // (GAMING_MIN_ELIGIBLE) and the published framing (FLEET_FRAMING_MIN_INSTANCES) rather than being left for
+  // a reader to infer.
   const eligible = instances.filter((i) => i.decided >= MIN_DECIDED && registered.has(i.instanceId));
   const eligibleIds = new Set(eligible.map((i) => i.instanceId));
   const cycle = cycleRows.filter((r) => eligibleIds.has(r.instance_id)).map((r) => r.ms);
@@ -375,6 +399,7 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
     outliers,
     gamingPatternFlags,
     gamingDetectionEligible,
+    fleetFramingEligible: eligible.length >= FLEET_FRAMING_MIN_INSTANCES,
   };
 }
 

src/review/public-stats.ts

@@ -256,7 +256,9 @@ export interface PublicStatsPayload {
      *  enforcement and sit outside both sides. */
     coveragePct: number | null;
     /** merge + close verdicts behind accuracyPct — the denominator a reader needs to judge the claim. */
-    decidedCount: number;
+    /** null when 1 < instanceCount < FLEET_FRAMING_MIN_INSTANCES — see the k-anonymity note at the assignment
+     *  site. A rate is publishable at any n; a pooled COUNT isolates one participant by subtraction at n=2. */
+    decidedCount: number | null;
     /** #8835: the live finite-sample guarantees, per arm, when a registered instance publishes one —
      *  "P(wrong | acted) ≤ alpha at aiJudgedCoveragePct" with the certification's own sample size. Null arms
      *  mean no guarantee is currently live (insufficient labels, or the instrument retracted it).
@@ -273,6 +275,13 @@ export interface PublicStatsPayload {
       merge: { alpha: number; lambda: number; aiJudgedCoveragePct: number; n: number; backfilledPct: number | null } | null;
     };
     instanceCount: number;
+    /** #9168: which of two things these figures are. `"fleet"` means instanceCount >= FLEET_FRAMING_MIN_INSTANCES,
+     *  the point at which a median is robust to a single bad contributor and the anti-farming detector can
+     *  fire. `"single_instance_self_report"` means they are ONE operator's own outcomes — real, and disclosed
+     *  as such, but not independent corroboration of anything that operator also publishes (the risk-control
+     *  guarantee in `guaranteed` is calibrated by the same instance, so fleet framing would invite a reader
+     *  to double-count one source as two). The numbers are published either way; only the claim changes. */
+    basis: "fleet" | "single_instance_self_report";
     windowDays: number;
     /** #9068: self-hosted instances flagged by computeFleetAnalytics's anti-farming detector
      *  (gamingPatternFlags, src/orb/analytics.ts). null (not 0) when the fleet has fewer than
@@ -577,9 +586,17 @@ export async function getPublicStats(
       closePrecisionPct: pooled.closeVerdicts > 0 ? pct(pooled.closeConfirmed / pooled.closeVerdicts) : null,
       closePrecisionCiPct: ciPct(pooled.closeConfirmed, pooled.closeVerdicts),
       coveragePct: pooled.coverage === null ? null : pct(pooled.coverage),
-      decidedCount: pooledVerdicts,
+      // #9168 (k-anonymity): the pooled count is a plain SUM over eligible instances, and this deployment's own
+      // volume is already public via byProject/own-ledger totals. So at exactly 2 instances a reader recovers
+      // the OTHER instance's decision volume by subtraction — and for a hosted tenant that volume is a business
+      // metric (how many PRs they ship, how many get closed), not ours to publish. Suppressed in that window.
+      // At n=1 there is nothing to subtract (the sum IS this deployment's own, already-public figure), and at
+      // n >= FLEET_FRAMING_MIN_INSTANCES the sum no longer isolates any single participant. RATES are safe at
+      // every n — a proportion carries no volume — so only the count is withheld.
+      decidedCount: fleet.instanceCount > 1 && !fleet.fleetFramingEligible ? null : pooledVerdicts,
       guaranteed,
       instanceCount: fleet.instanceCount,
+      basis: fleet.fleetFramingEligible ? "fleet" : "single_instance_self_report",
       windowDays: fleet.windowDays,
       gamingFlagsCaught: fleet.gamingDetectionEligible ? fleet.gamingPatternFlags.length : null,
     },

test/unit/orb-analytics.test.ts

@@ -543,3 +543,69 @@ describe("getFleetHealthSummary() (#4933)", () => {
     expect(await getFleetHealthSummary(env)).toEqual({ healthyCount: 1, unhealthyCount: 0, unknownCount: 0, totalCount: 1 });
   });
 });
+
+// #9168: `fleet` is a CLAIM, and below a few instances it is not a true one — a median of one is that one
+// value, a median of two is their mean, and the anti-farming detector cannot fire against a median an
+// instance is itself half of. The numbers stay published (they are real); `fleetFramingEligible` is what lets
+// the public surface say which of the two things they are, instead of letting fleet framing imply
+// corroboration a single-instance sample cannot provide.
+describe("fleetFramingEligible (#9168)", () => {
+  it("REGRESSION: one registered instance is NOT fleet-framed — the n=1 shape live in production today", async () => {
+    // The live payload at the time of writing: instanceCount 1, real numbers, published under fleet framing
+    // alongside a risk-control guarantee calibrated by that very same instance.
+    const env = createTestEnv();
+    await register(env, "solo");
+    await signals(env, "solo", 20, { verdict: "merge", outcome: "merged" });
+    const a = await computeFleetAnalytics(env);
+    expect(a.instanceCount).toBe(1);
+    expect(a.fleetFramingEligible).toBe(false);
+    // ...and the figures themselves are untouched: this gates the LABEL, never the measurement.
+    expect(a.fleet.mergePrecision).not.toBeNull();
+    expect(a.fleet.pooled.mergeVerdicts).toBe(20);
+  });
+
+  it("two instances are still not fleet-framed — a median of two is their mean, with no robustness", async () => {
+    const env = createTestEnv();
+    await register(env, "a", "b");
+    await signals(env, "a", 10, { verdict: "merge", outcome: "merged" });
+    await signals(env, "b", 10, { verdict: "merge", outcome: "merged" });
+    const a = await computeFleetAnalytics(env);
+    expect(a.instanceCount).toBe(2);
+    expect(a.fleetFramingEligible).toBe(false);
+  });
+
+  it("INVARIANT: three eligible instances clear the floor, and it moves in lockstep with the gaming detector", async () => {
+    const env = createTestEnv();
+    await register(env, "a", "b", "c");
+    for (const id of ["a", "b", "c"]) await signals(env, id, 10, { verdict: "merge", outcome: "merged" });
+    const a = await computeFleetAnalytics(env);
+    expect(a.instanceCount).toBe(3);
+    expect(a.fleetFramingEligible).toBe(true);
+    // Deliberately the SAME floor: the n at which a median becomes robust is the n at which "this far above
+    // the median" becomes satisfiable. Pinning them together stops one being moved without the other.
+    expect(a.gamingDetectionEligible).toBe(true);
+  });
+
+  it("INVARIANT: the floor counts ELIGIBLE instances — volume and registration both gate it", async () => {
+    const env = createTestEnv();
+    await register(env, "a", "b", "c");
+    await signals(env, "a", 10);
+    await signals(env, "b", 10);
+    await signals(env, "c", 1); // under MIN_DECIDED, so ineligible despite being registered
+    await signals(env, "unregistered", 10); // volume, but never opted in
+    const a = await computeFleetAnalytics(env);
+    expect(a.instanceCount).toBe(2);
+    expect(a.fleetFramingEligible).toBe(false);
+  });
+
+  it("INVARIANT: an unreadable fleet fails CLOSED — no fleet claim when the tables cannot be read", async () => {
+    const env = createTestEnv();
+    (env.DB as unknown as { prepare: () => never }).prepare = () => {
+      throw new Error("d1 unavailable");
+    };
+    const a = await computeFleetAnalytics(env);
+    expect(a.instanceCount).toBe(0);
+    expect(a.fleetFramingEligible).toBe(false);
+    expect(a.gamingDetectionEligible).toBe(false);
+  });
+});

test/unit/public-stats.test.ts

@@ -217,6 +217,10 @@ describe("getPublicStats — live aggregate over the review ledger", () => {
       coveragePct: null,
       decidedCount: 0,
       instanceCount: 0,
+      // #9168: zero registered instances is not a fleet, so the block never claims to be one. Note this is
+      // NOT the k-anonymity case -- decidedCount stays 0 (a real, non-identifying figure) because at n<=1
+      // there is no second participant whose volume subtraction could isolate.
+      basis: "single_instance_self_report",
       windowDays: 90,
       gamingFlagsCaught: null, // #9068: fewer than GAMING_MIN_ELIGIBLE eligible instances -- detector didn't run
       guaranteed: { close: null, merge: null },
@@ -484,6 +488,75 @@ describe("getPublicStats — live aggregate over the review ledger", () => {
     expect(out.totals.accuracyPct).toBe(0); // 1 - 1/1
   });
 
+  // #9168: the block is published under "fleet" framing next to a risk-control guarantee calibrated by the
+  // same instance, which invites a reader to read one party's self-report as two independent sources. The
+  // numbers are real and stay published; `basis` is what stops the overclaim, and the pooled COUNT is
+  // withheld in the one window where it isolates another participant.
+  describe("fleetAccuracy basis and pooled-count k-anonymity (#9168)", () => {
+    /** N registered instances, each with `per` confirmed merge verdicts (well clear of MIN_DECIDED). */
+    async function fleetOf(env: Env, instanceIds: string[], per = 8): Promise<void> {
+      for (const id of instanceIds) {
+        await env.DB.prepare("INSERT INTO orb_instances (instance_id, registered) VALUES (?, 1)").bind(id).run();
+        for (let i = 0; i < per; i += 1) {
+          await env.DB
+            .prepare(
+              `INSERT INTO orb_signals (instance_id, repo_hash, pr_hash, gate_verdict, outcome, reversal_flag)
+               VALUES (?, ?, ?, 'merge', 'merged', 'none')`,
+            )
+            .bind(id, "repo-hash", `${id}-pr-${i}`)
+            .run();
+        }
+      }
+    }
+
+    it("REGRESSION: at one instance it is labelled a self-report, NOT a fleet — the shape live in production", async () => {
+      const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" });
+      await fleetOf(env, ["inst-1"]);
+      const out = await getPublicStats(env, NOW);
+      expect(out.fleetAccuracy.instanceCount).toBe(1);
+      expect(out.fleetAccuracy.basis).toBe("single_instance_self_report");
+      // The count is NOT withheld at n=1: the pooled sum IS this deployment's own volume, which is already
+      // public via byProject, so there is no second party for subtraction to isolate.
+      expect(out.fleetAccuracy.decidedCount).toBe(8);
+      expect(out.fleetAccuracy.mergePrecisionPct).toBe(100);
+    });
+
+    it("REGRESSION: at exactly two instances the pooled COUNT is withheld — it isolates the other party by subtraction", async () => {
+      const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" });
+      await fleetOf(env, ["inst-1", "inst-2"]);
+      const out = await getPublicStats(env, NOW);
+      expect(out.fleetAccuracy.instanceCount).toBe(2);
+      expect(out.fleetAccuracy.basis).toBe("single_instance_self_report");
+      // The whole point: our own volume is public, so `pooled - ours` would hand a reader the other
+      // instance's decision volume exactly — a hosted tenant's business metric, not ours to publish.
+      expect(out.fleetAccuracy.decidedCount).toBeNull();
+      // RATES stay published at every n: a proportion carries no volume.
+      expect(out.fleetAccuracy.mergePrecisionPct).toBe(100);
+      expect(out.fleetAccuracy.accuracyPct).toBe(100);
+    });
+
+    it("at three instances it is a genuine fleet: framing, pooled count, and the gaming detector all switch on together", async () => {
+      const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" });
+      await fleetOf(env, ["inst-1", "inst-2", "inst-3"]);
+      const out = await getPublicStats(env, NOW);
+      expect(out.fleetAccuracy.instanceCount).toBe(3);
+      expect(out.fleetAccuracy.basis).toBe("fleet");
+      // Restored once the sum no longer isolates any single participant.
+      expect(out.fleetAccuracy.decidedCount).toBe(24);
+      // #9068's detector shares the same floor, so a "fleet" label always comes with a real (0, not null)
+      // gaming count -- a reader never sees fleet framing next to "the detector could not run".
+      expect(out.fleetAccuracy.gamingFlagsCaught).toBe(0);
+    });
+
+    it("INVARIANT: an empty fleet is a self-report with a 0 count, never a fleet claim", async () => {
+      const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" });
+      const out = await getPublicStats(env, NOW);
+      expect(out.fleetAccuracy.instanceCount).toBe(0);
+      expect(out.fleetAccuracy.basis).toBe("single_instance_self_report");
+      expect(out.fleetAccuracy.decidedCount).toBe(0);
+    });
+  });
+
   it("REGRESSION (#fairness-analytics): the headline accuracy prefers live fleet data once a registered instance clears the volume bar", async () => {
     const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" });
     const db = env.DB;