From f7777b16ca8ff1fca499e55ae517e4b78227c899 Mon Sep 17 00:00:00 2001
From: JSONbored <49853598+JSONbored@users.noreply.github.com>
Date: Wed, 29 Jul 2026 00:53:05 -0700
Subject: [PATCH] feat(stats): publish a fleet accuracy trend, so the weekly
table means something again
After #9718 and #9768 the own-ledger weekly trend is correct and, for recent
weeks, correctly null: the hosted Worker does not execute reviews, so that
ledger is frozen while Orb volume keeps growing. A column of nulls beside a
rising volume column is honest and tells a reader nothing about how the gate
behaves today.
orb_signals already carries everything needed, with no new ingest, no new column
and no new secret: self-host runtimes export gate_verdict/outcome/reversal_flag
per PR, the hosted side validates and stores them, and computeFleetAnalytics
already turns them into the 90-day headline. This buckets the SAME rows weekly.
Two deliberate non-choices, both of which would have reintroduced the bug class
this surface is being corrected for:
- It is a SEPARATE series, never blended with the own-ledger one. orb_signals
is keyed by per-instance HMACs of repo/PR; audit_events and orb_pr_outcomes
use raw owner/repo#number. The populations cannot be joined, so a reversal
from one can never be attributed to a decided row in the other, and one
number over a mixed denominator would be exactly the defect #9676 reports.
- It does NOT use 1 - reversalRate. #8820 established decisionAccuracy as the
fleet estimand because the reversal formula divides by holds -- deferrals
that cannot be right or wrong -- and misses mispredictions carrying no
reversal marker. A weekly series on a different estimand than the headline
directly above it would make the page disagree with itself.
The weekly fold calls foldInstance rather than reimplementing the confusion
matrix. That function is already exported for the federated bundle with a doc
comment warning that a second copy makes comparisons apples-to-oranges, and the
accounting is subtle: policy_action rows are excluded from scoring but still
count as activity, and a superseded close is disconfirmed exactly like a reopen.
Only registered instances count, matching computeFleetAnalytics' own trust gate
-- the ingest is open, so a stranger's signals must not move a published number
until a human opts them in. There is a test pinning that.
Refs #9676
---
apps/loopover-ui/public/openapi.json | 25 +++
.../site/fairness-report-page.test.tsx | 28 +++
.../components/site/fairness-report-page.tsx | 51 +++++
.../site/proof-of-power-stats.test.tsx | 1 +
packages/loopover-contract/src/public-api.ts | 14 ++
src/api/routes.ts | 9 +-
src/services/public-fleet-accuracy-trend.ts | 135 ++++++++++++
test/unit/public-fleet-accuracy-trend.test.ts | 205 ++++++++++++++++++
8 files changed, 466 insertions(+), 2 deletions(-)
create mode 100644 src/services/public-fleet-accuracy-trend.ts
create mode 100644 test/unit/public-fleet-accuracy-trend.test.ts
diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json
index 281e2017db..22e9a360e3 100644
--- a/apps/loopover-ui/public/openapi.json
+++ b/apps/loopover-ui/public/openapi.json
@@ -800,6 +800,30 @@
"filteredPct"
]
}
+ },
+ "fleetAccuracyTrend": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "weekStart": {
+ "type": "string"
+ },
+ "verdicts": {
+ "type": "number",
+ "nullable": true
+ },
+ "accuracyPct": {
+ "type": "number",
+ "nullable": true
+ }
+ },
+ "required": [
+ "weekStart",
+ "verdicts",
+ "accuracyPct"
+ ]
+ }
}
},
"required": [
@@ -811,6 +835,7 @@
"byProject",
"fleetAccuracy",
"accuracyTrend",
+ "fleetAccuracyTrend",
"reuseRateTrend",
"reviewVolumeTrend"
]
diff --git a/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx b/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx
index 81631909be..8c3624a5bf 100644
--- a/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx
+++ b/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx
@@ -81,6 +81,7 @@ const FIXTURE: PublicStats = {
windowDays: 90,
gamingFlagsCaught: 1,
},
+ fleetAccuracyTrend: [{ weekStart: "2026-07-13", verdicts: 40, accuracyPct: 92.5 }],
accuracyTrend: [
{ weekStart: "2026-07-13", merged: 30, closed: 15, reversed: 1, accuracyPct: 97.8 },
],
@@ -244,6 +245,33 @@ describe("FairnessReportPage (#fairness-analytics)", () => {
expect(notes.length).toBe(2); // one under each affected table
});
+ it("#9676: renders the fleet trend as its own section, never merged into the own-ledger table", async () => {
+ apiFetch.mockResolvedValue({ ok: true, data: FIXTURE, status: 200, durationMs: 10 });
+ renderWithClient();
+
+ await waitFor(() => expect(screen.getByText("Weekly trend — self-hosted fleet")).toBeTruthy());
+ // Two distinct sections, so a reader can never read one population's number off the other's row.
+ expect(screen.getByText("Weekly trend")).toBeTruthy();
+ expect(screen.getByText("Decisions scored")).toBeTruthy();
+ expect(screen.getByText("92.5%")).toBeTruthy();
+ });
+
+ it("#9676: hides the fleet trend entirely when no week has a scored verdict", async () => {
+ apiFetch.mockResolvedValue({
+ ok: true,
+ data: {
+ ...FIXTURE,
+ fleetAccuracyTrend: [{ weekStart: "2026-07-13", verdicts: null, accuracyPct: null }],
+ },
+ status: 200,
+ durationMs: 10,
+ });
+ renderWithClient();
+
+ await waitFor(() => expect(screen.getByText("Weekly trend")).toBeTruthy());
+ expect(screen.queryByText("Weekly trend — self-hosted fleet")).toBeNull();
+ });
+
it("does not show the unmeasurable-accuracy note when every accuracy is real", async () => {
apiFetch.mockResolvedValue({ ok: true, data: FIXTURE, status: 200, durationMs: 10 });
renderWithClient();
diff --git a/apps/loopover-ui/src/components/site/fairness-report-page.tsx b/apps/loopover-ui/src/components/site/fairness-report-page.tsx
index 7a5646a8e6..f11b583f0f 100644
--- a/apps/loopover-ui/src/components/site/fairness-report-page.tsx
+++ b/apps/loopover-ui/src/components/site/fairness-report-page.tsx
@@ -300,6 +300,57 @@ export function FairnessReportPage() {
) : null}
+ {/* #9676: the fleet-population sibling of the table above. Rendered as its own section, never
+ merged into it: the two measure different populations on different estimands, and blending
+ them into one column is the bug this whole surface is being corrected for. */}
+ {data.fleetAccuracyTrend &&
+ data.fleetAccuracyTrend.some((week) => week.verdicts != null) ? (
+
+
Weekly trend — self-hosted fleet
+
+ The same weeks, measured over the live self-hosted fleet instead of this
+ site's own frozen review history. This scores the gate's merge/close{" "}
+ decisions — the share the realized outcome confirmed — so it matches the
+ headline above rather than the reversal-grounded table beside it. Holds are
+ excluded: a deferral to a human is not a decision that can be right or wrong. Only
+ registered instances count.
+
+
+
+
+ Weekly scored fleet verdicts and the share the realized outcome confirmed.
+
+
+
+ |
+ Week
+ |
+
+ Decisions scored
+ |
+
+ Accuracy
+ |
+
+
+
+ {data.fleetAccuracyTrend.map((week) => (
+
+ | {week.weekStart} |
+
+ {week.verdicts != null ? intFmt.format(week.verdicts) : "—"}
+ |
+
+ {week.accuracyPct != null ? `${pctFmt.format(week.accuracyPct)}%` : "—"}
+ |
+
+ ))}
+
+
+
+
+ ) : null}
+
{data.rulePrecision && data.rulePrecision.rules.length > 0 ? (
Measured accuracy per rule
diff --git a/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx b/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx
index 6c31f744bb..775f3d01f5 100644
--- a/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx
+++ b/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx
@@ -98,6 +98,7 @@ const PAYLOAD: PublicStats = {
windowDays: 90,
gamingFlagsCaught: 0,
},
+ fleetAccuracyTrend: [{ weekStart: "2026-07-13", verdicts: 40, accuracyPct: 92.5 }],
accuracyTrend: [
{ weekStart: "2026-05-04", merged: 40, closed: 10, reversed: 2, accuracyPct: 96 },
{ weekStart: "2026-05-11", merged: 42, closed: 9, reversed: 1, accuracyPct: 98 },
diff --git a/packages/loopover-contract/src/public-api.ts b/packages/loopover-contract/src/public-api.ts
index 5bdf1a79af..f3716097e9 100644
--- a/packages/loopover-contract/src/public-api.ts
+++ b/packages/loopover-contract/src/public-api.ts
@@ -95,6 +95,20 @@ export const PublicStatsSchema = z.object({
accuracyPct: z.number().nullable(),
}),
),
+ /** Trailing weekly FLEET accuracy (#9676). A SEPARATE series from `accuracyTrend`, never blended with it:
+ * that one is reversal-grounded over the own-ledger population (raw `owner/repo#number` keys), this one is
+ * `decisionAccuracy` over registered self-host instances' `orb_signals` (per-instance HMAC'd keys). The two
+ * populations cannot be joined, and #8820 established `decisionAccuracy` -- not `1 - reversalRate` -- as the
+ * fleet estimand, so this matches the headline it sits under rather than the table beside it. `verdicts`
+ * counts scored merge/close decisions only; holds and policy actions are excluded. Null on a week means too
+ * few scored verdicts to publish. */
+ fleetAccuracyTrend: z.array(
+ z.object({
+ weekStart: z.string(),
+ verdicts: z.number().nullable(),
+ accuracyPct: z.number().nullable(),
+ }),
+ ),
/** Trailing weekly "how often we avoid redoing AI work" trend (#4448) -- a competence signal, not a cost
* claim. Counts cache hits/misses across every instrumented AI-touching capability (grounding,
* review-memory, impact-map, repo-culture-profile, ai_review, ai_slop, linked_issue_satisfaction,
diff --git a/src/api/routes.ts b/src/api/routes.ts
index 57aff8ebe8..718db374aa 100644
--- a/src/api/routes.ts
+++ b/src/api/routes.ts
@@ -313,6 +313,7 @@ import { ingestBittensorAnchorReport, parseBittensorAnchorReport } from "../revi
import { loadPublicLedgerAnchors } from "../review/ledger-anchor-persistence";
import { getPublicStats, isPublicStatsEnabled, resolvePublicStatsManifestOverride } from "../review/public-stats";
import { loadPublicAccuracyTrend } from "../services/public-accuracy-trend";
+import { loadPublicFleetAccuracyTrend } from "../services/public-fleet-accuracy-trend";
import { loadPublicRulePrecision } from "../review/public-rule-precision";
import { loadCalibrationTrend } from "../services/rule-calibration-trend";
import { isSatisfactionFloorAutotuneEnabled, loadSatisfactionFloorStatus, runSatisfactionFloorLoosening } from "../services/satisfaction-floor-loosening-run";
@@ -1272,9 +1273,13 @@ export function createApp() {
const publicStatsManifestOverride = await resolvePublicStatsManifestOverride(c.env);
if (!isPublicStatsEnabled(c.env, publicStatsManifestOverride)) return c.json({ error: "not_found" }, 404);
try {
- const [stats, accuracyTrend, reuseRateTrend, reviewVolumeTrend, rulePrecision] = await Promise.all([
+ const [stats, accuracyTrend, fleetAccuracyTrend, reuseRateTrend, reviewVolumeTrend, rulePrecision] = await Promise.all([
getPublicStats(c.env),
loadPublicAccuracyTrend(c.env),
+ // #9676: the fleet-population sibling of the series above. Deliberately a SECOND series rather than a
+ // merged one -- see public-fleet-accuracy-trend.ts's header for why the two populations cannot be
+ // joined, and why this one matches the headline's estimand instead of the table's.
+ loadPublicFleetAccuracyTrend(c.env),
loadPublicReuseRateTrend(c.env),
loadPublicReviewVolumeTrend(c.env),
// #8230: measured per-rule precision + the reproducibility freeze point. Same flag, same cache,
@@ -1282,7 +1287,7 @@ export function createApp() {
loadPublicRulePrecision(c.env),
]);
c.header("Cache-Control", "public, max-age=60, stale-while-revalidate=300");
- return c.json({ ...stats, accuracyTrend, reuseRateTrend, reviewVolumeTrend, rulePrecision });
+ return c.json({ ...stats, accuracyTrend, fleetAccuracyTrend, reuseRateTrend, reviewVolumeTrend, rulePrecision });
} catch {
return c.json({ error: "public_stats_unavailable" }, 503);
}
diff --git a/src/services/public-fleet-accuracy-trend.ts b/src/services/public-fleet-accuracy-trend.ts
new file mode 100644
index 0000000000..db9d272d89
--- /dev/null
+++ b/src/services/public-fleet-accuracy-trend.ts
@@ -0,0 +1,135 @@
+// Public fleet accuracy trend (#9676). The own-ledger weekly trend (public-accuracy-trend.ts) is
+// reversal-grounded over `audit_events`, and that ledger is frozen: the hosted Worker does not execute
+// reviews, so every recent week correctly reports `null` while Orb volume keeps growing. A table of nulls
+// beside a rising volume column is honest but tells a reader nothing about how the gate behaves TODAY.
+//
+// `orb_signals` already carries what is needed, with no new ingest, no new column and no new secret:
+// self-host runtimes export `gate_verdict`, `outcome` and `reversal_flag` per PR (src/selfhost/orb-collector.ts),
+// the hosted side validates and stores them (src/orb/ingest.ts), and computeFleetAnalytics already turns them
+// into the 90-day headline. This module buckets the SAME rows weekly.
+//
+// TWO THINGS THIS DELIBERATELY DOES NOT DO:
+//
+// 1. It never blends with the own-ledger series. `orb_signals` is keyed by per-instance HMACs of repo/PR;
+// `orb_pr_outcomes` and `audit_events` are keyed by raw `owner/repo#number`. The two count different
+// populations and cannot be joined, so a reversal from one can never be attributed to a decided row in
+// the other. Publishing one number over a mixed denominator is exactly the class of bug #9676 exists to
+// fix -- so this is a SEPARATE series with its own numerator and denominator.
+//
+// 2. It does not use `1 - reversalRate`. #8820 established that the fleet number is `decisionAccuracy` --
+// the share of the gate's own merge/close verdicts the realized outcome confirmed, holds excluded --
+// because the reversal formula both divides by deferrals that cannot be right or wrong and misses
+// outright mispredictions that carry no reversal marker. Publishing a weekly series on a different
+// estimand than the headline it sits under would make the page disagree with itself.
+//
+// The weekly fold calls `foldInstance` rather than reimplementing the confusion-matrix accounting. That
+// function's own doc comment already warns that a second copy silently makes comparisons
+// apples-to-oranges (it was exported for the federated bundle for the same reason), and the accounting is
+// subtle: `policy_action` rows are excluded from scoring but still count as activity, and a `superseded`
+// close is disconfirmed exactly like a literal reopen.
+import { foldInstance, type Cell } from "../orb/analytics";
+import { safeAll } from "../review/public-stats";
+import { isoWeekStart } from "./public-quality-metrics";
+
+export const PUBLIC_FLEET_TREND_WEEKS = 8;
+
+/** Below this many scored verdicts in a week, that week's accuracy is too noisy to publish. Mirrors
+ * MIN_ACCURACY_TREND_SAMPLE's role on the own-ledger series -- a public percentage over one or two
+ * decisions is a coin flip wearing a number. */
+export const MIN_FLEET_TREND_VERDICTS = 5;
+
+export type PublicFleetAccuracyTrendWeek = {
+ /** UTC Monday (YYYY-MM-DD) that starts the bucket. */
+ weekStart: string;
+ /** Scored merge/close verdicts in the week -- holds and policy actions excluded, matching the estimand. */
+ verdicts: number | null;
+ /** Share of those verdicts the realized outcome confirmed; null below the sample floor. */
+ accuracyPct: number | null;
+};
+
+/** One `orb_signals` confusion-matrix cell, bucketed to a week. */
+export type FleetTrendCell = Cell & { weekStart: string };
+
+/** The raw query shape: the same cell dimensions, bucketed to a UTC day by SQL. */
+type FleetTrendDayCell = Cell & { day: string };
+
+const MS_PER_WEEK = 7 * 86_400_000;
+
+function roundPct(value: number): number {
+ return Math.round(value * 1000) / 10;
+}
+
+/**
+ * PURE. Fold week-bucketed cells into `weeks` trailing UTC-Monday buckets ending in the week containing
+ * `nowMs`. Mirrors buildPublicAccuracyTrend's bucketing shape so the two series line up row-for-row on the
+ * page even though they measure different populations.
+ */
+export function buildPublicFleetAccuracyTrend(
+ cells: readonly FleetTrendCell[],
+ nowMs: number,
+ weeks: number = PUBLIC_FLEET_TREND_WEEKS,
+): PublicFleetAccuracyTrendWeek[] {
+ const currentStartMs = Date.parse(isoWeekStart(nowMs));
+ const oldestStartMs = currentStartMs - (weeks - 1) * MS_PER_WEEK;
+ const buckets: Cell[][] = Array.from({ length: weeks }, () => []);
+
+ for (const cell of cells) {
+ const weekMs = Date.parse(`${cell.weekStart}T00:00:00.000Z`);
+ if (!Number.isFinite(weekMs)) continue;
+ const offset = Math.round((weekMs - oldestStartMs) / MS_PER_WEEK);
+ if (offset < 0 || offset >= weeks) continue;
+ buckets[offset]!.push(cell);
+ }
+
+ return buckets.map((bucketCells, offset) => {
+ const weekStart = isoWeekStart(oldestStartMs + offset * MS_PER_WEEK);
+ if (bucketCells.length === 0) return { weekStart, verdicts: null, accuracyPct: null };
+ // Pooled across every registered instance in the week, matching how the headline pools (#9068): one
+ // synthetic id, because this series is a fleet aggregate and not a per-instance figure.
+ const folded = foldInstance("fleet", bucketCells);
+ const verdicts = folded.counts.mergeVerdicts + folded.counts.closeVerdicts;
+ if (verdicts < MIN_FLEET_TREND_VERDICTS || folded.decisionAccuracy === null) {
+ return { weekStart, verdicts: null, accuracyPct: null };
+ }
+ return { weekStart, verdicts, accuracyPct: roundPct(folded.decisionAccuracy) };
+ });
+}
+
+/**
+ * Load the weekly fleet accuracy series. Fail-safe: `safeAll` swallows a read error into an empty result, so
+ * a bad query yields all-null weeks rather than throwing the whole public stats payload.
+ *
+ * Only REGISTERED instances count, matching computeFleetAnalytics' own trust gate: the ingest is open, so a
+ * stranger's signals must not move a published number until a human opts them in.
+ */
+export async function loadPublicFleetAccuracyTrend(env: Env, nowMs: number = Date.now()): Promise
{
+ const sinceIso = new Date(Date.parse(isoWeekStart(nowMs)) - (PUBLIC_FLEET_TREND_WEEKS - 1) * MS_PER_WEEK).toISOString();
+ // Bucket by when the gate DECIDED, not when the row arrived: a batch exported late would otherwise pile
+ // weeks of decisions into the week it was received. `received_at` is the fallback only because
+ // `decision_timestamp` is nullable in the schema.
+ const cells = await safeAll(
+ env,
+ // instance_id is selected and grouped even though this series is a fleet aggregate: it keeps the rows a
+ // genuine `Cell`, which is what foldInstance consumes, and summing per-instance cells within a week
+ // yields exactly the pooled figure anyway. Narrowing the projection would mean hand-rolling a
+ // near-Cell type and re-deriving the accounting -- the duplication foldInstance is exported to prevent.
+ `SELECT substr(COALESCE(s.decision_timestamp, s.received_at), 1, 10) AS day, s.instance_id,
+ s.gate_verdict AS verdict, s.outcome, s.reversal_flag, s.gate_reasoncode_bucket, COUNT(*) AS n
+ FROM orb_signals s
+ JOIN orb_instances i ON i.instance_id = s.instance_id AND i.registered = 1
+ WHERE COALESCE(s.decision_timestamp, s.received_at) >= ?
+ GROUP BY day, s.instance_id, s.gate_verdict, s.outcome, s.reversal_flag, s.gate_reasoncode_bucket`,
+ sinceIso,
+ );
+ // isoWeekStart is applied here rather than in SQL: SQLite's strftime('%W') is not ISO-8601 week numbering,
+ // and the own-ledger trend already owns this exact conversion in JS.
+ const weekly: FleetTrendCell[] = [];
+ for (const cell of cells) {
+ const dayMs = Date.parse(`${cell.day}T00:00:00.000Z`);
+ // A row whose timestamp column is unparseable is dropped rather than bucketed to the epoch, which would
+ // silently land it outside the window anyway -- explicit is better than accidentally-correct.
+ if (!Number.isFinite(dayMs)) continue;
+ weekly.push({ ...cell, weekStart: isoWeekStart(dayMs) });
+ }
+ return buildPublicFleetAccuracyTrend(weekly, nowMs);
+}
diff --git a/test/unit/public-fleet-accuracy-trend.test.ts b/test/unit/public-fleet-accuracy-trend.test.ts
new file mode 100644
index 0000000000..08cca008aa
--- /dev/null
+++ b/test/unit/public-fleet-accuracy-trend.test.ts
@@ -0,0 +1,205 @@
+import { describe, expect, it } from "vitest";
+import {
+ buildPublicFleetAccuracyTrend,
+ loadPublicFleetAccuracyTrend,
+ MIN_FLEET_TREND_VERDICTS,
+ PUBLIC_FLEET_TREND_WEEKS,
+ type FleetTrendCell,
+} from "../../src/services/public-fleet-accuracy-trend";
+import { isoWeekStart } from "../../src/services/public-quality-metrics";
+import { createTestEnv } from "../helpers/d1";
+
+// #9676: the fleet-population weekly series. Load-bearing properties: it uses decisionAccuracy (the
+// headline's estimand, #8820) rather than 1 - reversalRate, it counts only scored merge/close verdicts, and
+// it only ever sees REGISTERED instances.
+
+const NOW = Date.parse("2026-06-22T12:00:00.000Z");
+
+const cell = (over: Partial = {}): FleetTrendCell => ({
+ weekStart: isoWeekStart(NOW),
+ instance_id: "inst",
+ verdict: "merge",
+ outcome: "merged",
+ reversal_flag: "none",
+ gate_reasoncode_bucket: "quality",
+ n: 1,
+ ...over,
+});
+
+describe("buildPublicFleetAccuracyTrend (#9676)", () => {
+ it("scores decisionAccuracy: confirmed verdicts over scored verdicts", () => {
+ const week = isoWeekStart(NOW);
+ const trend = buildPublicFleetAccuracyTrend(
+ [cell({ n: 8 }), cell({ outcome: "closed", n: 2 })], // 8 confirmed merges, 2 merges that closed instead
+ NOW,
+ 1,
+ );
+ expect(trend[0]).toEqual({ weekStart: week, verdicts: 10, accuracyPct: 80 });
+ });
+
+ it("INVARIANT: uses decisionAccuracy, NOT 1 - reversalRate -- holds must not dilute it", () => {
+ // 5 confirmed merges plus 95 holds. The reversal formula would divide by all 100 and report ~100%
+ // regardless of gate quality; decisionAccuracy scores only the 5 decisions the gate actually made.
+ const trend = buildPublicFleetAccuracyTrend([cell({ n: 5 }), cell({ verdict: "hold", n: 95 })], NOW, 1);
+ expect(trend[0]?.verdicts).toBe(5);
+ expect(trend[0]?.accuracyPct).toBe(100);
+ });
+
+ it("counts a reverted merge and a reopened/superseded close as the gate being wrong", () => {
+ const reverted = buildPublicFleetAccuracyTrend([cell({ n: 5 }), cell({ reversal_flag: "reverted", n: 5 })], NOW, 1);
+ expect(reverted[0]?.accuracyPct).toBe(50);
+ for (const flag of ["reopened", "superseded"] as const) {
+ const closes = buildPublicFleetAccuracyTrend(
+ [cell({ verdict: "close", outcome: "closed", n: 5 }), cell({ verdict: "close", outcome: "closed", reversal_flag: flag, n: 5 })],
+ NOW,
+ 1,
+ );
+ expect(closes[0]?.accuracyPct).toBe(50);
+ }
+ });
+
+ it("excludes policy_action rows from scoring entirely (#8825), rather than inflating either side", () => {
+ const trend = buildPublicFleetAccuracyTrend(
+ [cell({ n: 5 }), cell({ gate_reasoncode_bucket: "policy_action", verdict: "close", outcome: "closed", n: 50 })],
+ NOW,
+ 1,
+ );
+ expect(trend[0]).toEqual({ weekStart: isoWeekStart(NOW), verdicts: 5, accuracyPct: 100 });
+ });
+
+ it("redacts a week below the verdict floor rather than publishing a coin flip", () => {
+ const trend = buildPublicFleetAccuracyTrend([cell({ n: MIN_FLEET_TREND_VERDICTS - 1 })], NOW, 1);
+ expect(trend[0]).toMatchObject({ verdicts: null, accuracyPct: null });
+ });
+
+ it("publishes at exactly the floor", () => {
+ expect(buildPublicFleetAccuracyTrend([cell({ n: MIN_FLEET_TREND_VERDICTS })], NOW, 1)[0]?.accuracyPct).toBe(100);
+ });
+
+ it("redacts a week of holds only -- activity without a single scored decision", () => {
+ const trend = buildPublicFleetAccuracyTrend([cell({ verdict: "hold", n: 40 })], NOW, 1);
+ expect(trend[0]).toMatchObject({ verdicts: null, accuracyPct: null });
+ });
+
+ it("buckets into trailing UTC-Monday weeks and ignores rows outside the window", () => {
+ const current = isoWeekStart(NOW);
+ const prior = isoWeekStart(NOW - 7 * 86_400_000);
+ const ancient = isoWeekStart(NOW - 60 * 86_400_000);
+ const trend = buildPublicFleetAccuracyTrend(
+ [cell({ weekStart: prior, n: 10 }), cell({ weekStart: current, n: 5 }), cell({ weekStart: ancient, n: 999 })],
+ NOW,
+ 2,
+ );
+ expect(trend.map((w) => w.weekStart)).toEqual([prior, current]);
+ expect(trend.map((w) => w.verdicts)).toEqual([10, 5]);
+ });
+
+ it("ignores an unparseable weekStart rather than corrupting a bucket", () => {
+ const trend = buildPublicFleetAccuracyTrend([cell({ weekStart: "not-a-date", n: 99 }), cell({ n: 5 })], NOW, 1);
+ expect(trend[0]?.verdicts).toBe(5);
+ });
+
+ it("returns all-null weeks for empty input, and defaults to PUBLIC_FLEET_TREND_WEEKS", () => {
+ const trend = buildPublicFleetAccuracyTrend([], NOW);
+ expect(trend).toHaveLength(PUBLIC_FLEET_TREND_WEEKS);
+ for (const week of trend) expect(week).toMatchObject({ verdicts: null, accuracyPct: null });
+ });
+});
+
+describe("loadPublicFleetAccuracyTrend — end to end over orb_signals", () => {
+ async function seed(env: Env, input: { instance: string; registered: boolean; rows: Array<{ verdict: string; outcome: string; reversal?: string; decidedAt: string }> }): Promise {
+ await env.DB.prepare("INSERT OR IGNORE INTO orb_instances (instance_id, registered) VALUES (?, ?)").bind(input.instance, input.registered ? 1 : 0).run();
+ let i = 0;
+ for (const row of input.rows) {
+ i += 1;
+ await env.DB
+ .prepare(
+ `INSERT INTO orb_signals (instance_id, repo_hash, pr_hash, gate_verdict, outcome, reversal_flag, gate_reasoncode_bucket, decision_timestamp, received_at)
+ VALUES (?, ?, ?, ?, ?, ?, 'quality', ?, ?)`,
+ )
+ .bind(input.instance, `repo-${input.instance}`, `pr-${input.instance}-${i}`, row.verdict, row.outcome, row.reversal ?? "none", row.decidedAt, row.decidedAt)
+ .run();
+ }
+ }
+
+ const thisWeek = `${isoWeekStart(NOW)}T09:00:00.000Z`;
+
+ it("scores a registered instance's week from real rows", async () => {
+ const env = createTestEnv();
+ await seed(env, {
+ instance: "inst-a",
+ registered: true,
+ rows: [
+ ...Array.from({ length: 8 }, () => ({ verdict: "merge", outcome: "merged", decidedAt: thisWeek })),
+ ...Array.from({ length: 2 }, () => ({ verdict: "merge", outcome: "closed", decidedAt: thisWeek })),
+ ],
+ });
+ const trend = await loadPublicFleetAccuracyTrend(env, NOW);
+ expect(trend[trend.length - 1]).toEqual({ weekStart: isoWeekStart(NOW), verdicts: 10, accuracyPct: 80 });
+ });
+
+ it("INVARIANT: an UNREGISTERED instance never moves a published number", async () => {
+ // The ingest is open, so a stranger's signals must not reach the public surface until a human opts them
+ // in -- the same trust gate computeFleetAnalytics applies to the headline.
+ const env = createTestEnv();
+ await seed(env, {
+ instance: "stranger",
+ registered: false,
+ rows: Array.from({ length: 40 }, () => ({ verdict: "merge", outcome: "closed", decidedAt: thisWeek })),
+ });
+ const trend = await loadPublicFleetAccuracyTrend(env, NOW);
+ for (const week of trend) expect(week).toMatchObject({ verdicts: null, accuracyPct: null });
+ });
+
+ it("pools registered instances within a week", async () => {
+ const env = createTestEnv();
+ await seed(env, { instance: "a", registered: true, rows: Array.from({ length: 5 }, () => ({ verdict: "merge", outcome: "merged", decidedAt: thisWeek })) });
+ await seed(env, { instance: "b", registered: true, rows: Array.from({ length: 5 }, () => ({ verdict: "merge", outcome: "closed", decidedAt: thisWeek })) });
+ expect((await loadPublicFleetAccuracyTrend(env, NOW))[PUBLIC_FLEET_TREND_WEEKS - 1]).toMatchObject({ verdicts: 10, accuracyPct: 50 });
+ });
+
+ it("buckets by when the gate DECIDED, not when the row was received", async () => {
+ // A late export must not pile older decisions into the week it arrived.
+ const env = createTestEnv();
+ const priorWeek = `${isoWeekStart(NOW - 7 * 86_400_000)}T09:00:00.000Z`;
+ await env.DB.prepare("INSERT OR IGNORE INTO orb_instances (instance_id, registered) VALUES ('late', 1)").run();
+ for (let i = 0; i < 6; i += 1) {
+ await env.DB
+ .prepare(
+ `INSERT INTO orb_signals (instance_id, repo_hash, pr_hash, gate_verdict, outcome, reversal_flag, gate_reasoncode_bucket, decision_timestamp, received_at)
+ VALUES ('late', 'r', ?, 'merge', 'merged', 'none', 'quality', ?, ?)`,
+ )
+ .bind(`pr-${i}`, priorWeek, thisWeek)
+ .run();
+ }
+ const trend = await loadPublicFleetAccuracyTrend(env, NOW);
+ expect(trend[trend.length - 2]).toMatchObject({ verdicts: 6 });
+ expect(trend[trend.length - 1]).toMatchObject({ verdicts: null });
+ });
+
+ it("drops a row whose timestamp column is unparseable rather than bucketing it to the epoch", async () => {
+ // decision_timestamp is TEXT with no format constraint, so a malformed value is reachable in practice.
+ // It must be dropped explicitly, not silently land somewhere via Date.parse -> NaN arithmetic.
+ const env = createTestEnv();
+ await env.DB.prepare("INSERT OR IGNORE INTO orb_instances (instance_id, registered) VALUES ('bad', 1)").run();
+ for (let i = 0; i < 6; i += 1) {
+ await env.DB
+ .prepare(
+ `INSERT INTO orb_signals (instance_id, repo_hash, pr_hash, gate_verdict, outcome, reversal_flag, gate_reasoncode_bucket, decision_timestamp, received_at)
+ VALUES ('bad', 'r', ?, 'merge', 'merged', 'none', 'quality', 'not-a-timestamp', 'not-a-timestamp')`,
+ )
+ .bind(`pr-${i}`)
+ .run();
+ }
+ const trend = await loadPublicFleetAccuracyTrend(env, NOW);
+ for (const week of trend) expect(week).toMatchObject({ verdicts: null, accuracyPct: null });
+ });
+
+ it("degrades to all-null weeks (never throws) when the read fails", async () => {
+ const broken = createTestEnv();
+ broken.DB = { prepare: () => { throw new Error("boom"); } } as never;
+ const trend = await loadPublicFleetAccuracyTrend(broken, NOW);
+ expect(trend).toHaveLength(PUBLIC_FLEET_TREND_WEEKS);
+ for (const week of trend) expect(week).toMatchObject({ verdicts: null, accuracyPct: null });
+ });
+});