feat(retention): bound orb_signals by folding it into per-day rollups

migrations/0202_orb_signal_rollups.sql

@@ -0,0 +1,59 @@
+-- #9783: orb_signals grows without bound. Every self-host instance exports into it hourly, and it is a
+-- PUBLIC data source -- computeFleetAnalytics reads it for the /fairness headline, and #9775's weekly fleet
+-- trend reads it again.
+--
+-- It could not simply be added to RETENTION_POLICY. Two live surfaces depend on history the raw rows carry:
+-- fleet-admin.ts's listFleetInstances reports a LIFETIME signalCount per instance, and
+-- /v1/internal/fleet/analytics accepts an arbitrary ?days= window. A plain prune would silently turn the
+-- first into "signals in the retention window" with no change of label, and make the second report a window
+-- it no longer has data for -- the same window/denominator mismatch #9676 and #9793 have been correcting.
+--
+-- So this is the #9474 pattern (orb_pr_outcomes -> orb_outcome_rollups), which exists for exactly this
+-- shape: fold the aging rows into a durable aggregate in the SAME transaction that deletes them, so the
+-- information survives even though the per-PR rows do not.
+--
+-- GRANULARITY IS DAILY, and deliberately keeps the whole confusion matrix rather than a bare count. The
+-- fleet numbers are not a single total: computeFleetAnalytics folds (verdict, outcome, reversal_flag,
+-- gate_reasoncode_bucket) cells through foldInstance to get decisionAccuracy, and #9775's trend buckets the
+-- same cells by day. Rolling up to a scalar would preserve signalCount and destroy every accuracy figure
+-- beyond the window; rolling up these cells per day preserves BOTH -- a historical week reconstructs to the
+-- same numbers it would have shown while the raw rows existed.
+--
+-- What is genuinely dropped is per-PR identity: repo_hash/pr_hash. Those are per-instance HMACs that cannot
+-- be reversed or joined across instances, so individually they carry no long-term analytical value; their
+-- only role is the (instance_id, repo_hash, pr_hash) dedup key, which matters while a row can still be
+-- re-exported and not after it has aged out.
+CREATE TABLE IF NOT EXISTS orb_signal_rollups (
+  instance_id            TEXT    NOT NULL,
+  -- UTC day (YYYY-MM-DD) of the folded rows' decision_timestamp, falling back to received_at -- the same
+  -- COALESCE the weekly trend buckets on, so a folded day lands in the week it would have landed in live.
+  day                    TEXT    NOT NULL,
+  -- NOT NULL with an empty-string sentinel, unlike orb_signals where both of these are nullable. A PRIMARY
+  -- KEY over nullable columns does not work here: SQLite does not enforce NOT NULL on PRIMARY KEY columns of
+  -- a rowid table, and every NULL compares DISTINCT, so two folds of the same day would silently insert
+  -- duplicate cells instead of accumulating into one. The fold COALESCEs to '' on the way in.
+  --
+  -- '' is safe for both readers, which is why a sentinel is preferable to dropping the columns: foldInstance
+  -- only ever tests `verdict === "merge"` / `=== "close"` and `gate_reasoncode_bucket === "policy_action"`,
+  -- and '' fails those exactly as NULL did -- an unset verdict counted as a hold before and still does.
+  gate_verdict           TEXT    NOT NULL DEFAULT '',
+  outcome                TEXT    NOT NULL,
+  reversal_flag          TEXT    NOT NULL,
+  gate_reasoncode_bucket TEXT    NOT NULL DEFAULT '',
+  n                      INTEGER NOT NULL,
+  updated_at             TEXT    NOT NULL,
+  -- One row per cell per instance per day. The prune folds with an upsert that ADDS to `n`, so a day folded
+  -- across two prune runs (rows arriving late, or a slice boundary splitting a day) accumulates rather than
+  -- overwriting.
+  PRIMARY KEY (instance_id, day, gate_verdict, outcome, reversal_flag, gate_reasoncode_bucket)
+);
+
+-- The read path: every consumer scans a day range across all instances, then joins to orb_instances to keep
+-- unregistered ones out of published numbers.
+CREATE INDEX IF NOT EXISTS orb_signal_rollups_day ON orb_signal_rollups (day, instance_id);
+
+-- The prune scans orb_signals by received_at (both the slice-boundary SELECT and the DELETE predicate), so
+-- it needs an index leading with that column or every hourly retention run degrades to a full table scan +
+-- sort. 0193 and 0196 (#9472/#9473) did this for every policy table of their era, and the retention suite
+-- now enforces it for all of them -- this is simply that same index for the entry added here.
+CREATE INDEX IF NOT EXISTS idx_orb_signals_received_at ON orb_signals (received_at);

scripts/check-schema-drift.ts

@@ -66,6 +66,10 @@ export const RAW_SQL_ONLY_TABLES: Set<string> = new Set([
   "orb_reuse_counters",
   "orb_risk_control_arms",
   "orb_signals",
+  // #9783: per-instance/per-day confusion-matrix cells folded from orb_signals (above) by the retention
+  // prune, and unioned back in by computeFleetAnalytics, loadPublicFleetAccuracyTrend and
+  // listFleetInstances -- all via env.DB.prepare, no Drizzle use. Same shape as orb_outcome_rollups.
+  "orb_signal_rollups",
   "orb_webhook_events",
   "override_audit",
   "predicted_gate_calibration_ledger",

src/db/retention.ts

@@ -22,6 +22,13 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [
   // needs would be gone by the time the fold ran, and the rollup would permanently over-count exactly the
   // PRs the live query never counted.
   { table: "orb_pr_outcomes", column: "occurred_at", days: 90 },
+  // #9783: orb_signals had no rule at all and grew without bound, while being a PUBLIC data source
+  // (computeFleetAnalytics' /fairness headline, and #9775's weekly fleet trend). It could not simply be
+  // pruned: listFleetInstances reports a LIFETIME signalCount, and /v1/internal/fleet/analytics takes an
+  // arbitrary ?days=. Both are preserved by folding into orb_signal_rollups first -- see the special case in
+  // pruneExpiredRecords, and migrations/0202 for why the rollup keeps the whole confusion matrix per day
+  // rather than a scalar count.
+  { table: "orb_signals", column: "received_at", days: 90 },
   { table: "audit_events", column: "created_at", days: 90 },
   { table: "ai_usage_events", column: "created_at", days: 90 },
   { table: "product_usage_events", column: "occurred_at", days: 180 },
@@ -120,6 +127,12 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [
 // not scan-optimal, which is acceptable for the lower row counts of the tables that fall back today.
 export const RETENTION_PK_COLUMN: Readonly<Record<string, string>> = {
   audit_events: "id",
+  // #9783: orb_signals has `id INTEGER PRIMARY KEY` (migrations/0060), so it maps cleanly here rather than
+  // joining RETENTION_COMPOSITE_PK_TABLES -- its UNIQUE (instance_id, repo_hash, pr_hash) is the ingest
+  // dedup key, not its primary key. The fold below uses its own bounded-slice path rather than the generic
+  // PK-batched delete, but the mapping still has to exist: the completeness guard requires every policy
+  // table to be in one list or the other, precisely so a new entry cannot ship silently unmapped.
+  orb_signals: "id",
   ai_usage_events: "id",
   product_usage_events: "id",
   github_rate_limit_observations: "id",
@@ -257,6 +270,62 @@ export async function pruneExpiredRecords(
     // is deliberately UNBATCHED for this one table: the aging cohort is one row per fleet-wide PR terminal
     // per day (hundreds at most, vs the six-figure log tables the batching exists for), and a bounded delete
     // would reintroduce the split-commit problem for whatever the bound left behind.
+    // #9783: orb_signals feeds published fleet accuracy AND a lifetime per-instance count, so its rows fold
+    // into per-instance/per-DAY confusion-matrix cells before deletion. Same atomic fold+delete and same
+    // bounded inclusive-boundary slicing as orb_pr_outcomes below, for the same two reasons: a fold and
+    // delete that could commit separately would double- or under-count, and an exclusive boundary against a
+    // run of tied timestamps selects zero rows and spins forever.
+    //
+    // Unlike that one, EVERY expiring row folds -- there is no "the live query never counted these" subset
+    // to skip. The registered-instance filter belongs at READ time (computeFleetAnalytics and the public
+    // trend both join orb_instances), not here: an instance can be registered after its signals arrive, and
+    // discarding an unregistered instance's history at prune time would make that later registration
+    // silently lossy.
+    if (rule.table === "orb_signals") {
+      let deleted = 0;
+      for (;;) {
+        const boundaryRow = await env.DB.prepare(
+          `SELECT ${rule.column} AS boundary FROM ${rule.table} WHERE ${retentionWhere(rule)} ORDER BY ${rule.column} LIMIT 1 OFFSET ${batchSize - 1}`,
+        )
+          .bind(cutoff)
+          .first<{ boundary: string }>();
+        const sliceIsFinal = boundaryRow == null;
+        const sliceWhere = sliceIsFinal ? `${rule.column} < ?1` : `${rule.column} <= ?1`;
+        const sliceBound = sliceIsFinal ? cutoff : boundaryRow.boundary;
+        const batchResults = await env.DB.batch([
+          env.DB.prepare(
+            // Bucketed on COALESCE(decision_timestamp, received_at) -- the same expression the weekly trend
+            // buckets live rows on, so a folded day lands in the week it would have landed in anyway.
+            // gate_verdict / gate_reasoncode_bucket COALESCE to '' because the rollup's PRIMARY KEY spans
+            // them and SQLite compares every NULL as distinct (migrations/0202 has the full note).
+            `INSERT INTO orb_signal_rollups (instance_id, day, gate_verdict, outcome, reversal_flag, gate_reasoncode_bucket, n, updated_at)
+             SELECT s.instance_id,
+                    substr(COALESCE(s.decision_timestamp, s.received_at), 1, 10) AS day,
+                    COALESCE(s.gate_verdict, '') AS gate_verdict,
+                    s.outcome,
+                    s.reversal_flag,
+                    COALESCE(s.gate_reasoncode_bucket, '') AS gate_reasoncode_bucket,
+                    COUNT(*) AS n,
+                    ?2 AS updated_at
+             FROM orb_signals s
+             WHERE s.${sliceWhere}
+             GROUP BY s.instance_id, day, COALESCE(s.gate_verdict, ''), s.outcome, s.reversal_flag, COALESCE(s.gate_reasoncode_bucket, '')
+             ON CONFLICT(instance_id, day, gate_verdict, outcome, reversal_flag, gate_reasoncode_bucket) DO UPDATE SET
+               n = orb_signal_rollups.n + excluded.n,
+               updated_at = excluded.updated_at`,
+          ).bind(sliceBound, nowIso()),
+          env.DB.prepare(`DELETE FROM ${rule.table} WHERE ${sliceWhere}`).bind(sliceBound),
+        ]);
+        /* v8 ignore next 2 -- defensive: batch() returns one result per statement on both backends; the
+         * `?.`/`?? 0` arms only satisfy the driver types and degrade the COUNT, never the prune. */
+        const changes = Number(batchResults[1]?.meta?.changes ?? 0);
+        deleted += changes;
+        if (sliceIsFinal || changes === 0 || deleted >= maxPerTable) break;
+      }
+      results.push({ table: rule.table, column: rule.column, cutoff, deleted });
+      continue;
+    }
+
     if (rule.table === "orb_pr_outcomes") {
       // #9474: this table feeds a CUMULATIVE public counter (getOrbGlobalStats -> the homepage "all-time"
       // merged/closed totals), so every row must be folded into the durable orb_outcome_rollups totals in the

src/orb/analytics.ts

@@ -29,6 +29,11 @@
 // Exported so the federated bundle export (#1970, src/orb/federated-bundle.ts) gates its own published
 // precision on the SAME volume bar the fleet median uses — a bundle must not advertise a precision the fleet
 // would refuse to count.
+// #9783: the retention horizon is imported rather than restated, so the window this module trusts for cycle
+// time cannot drift from the window the prune actually enforces. retention.ts imports only a JSON util, so
+// this introduces no cycle.
+import { retentionCutoffIsoForTable } from "../db/retention";
+
 export const MIN_DECIDED = 5; // an instance needs at least this many decided PRs to count toward the fleet median
 const OUTLIER_BAND = 0.25; // |instance precision − fleet median| beyond this flags the instance
 const GAMING_VOLUME_MULTIPLIER = 2; // an instance's decided count more than this many times the fleet median
@@ -162,6 +167,11 @@ export interface FleetAnalytics {
    *  public surface can label them honestly instead of letting fleet framing imply corroboration that a
    *  single-instance sample cannot provide. */
   fleetFramingEligible: boolean;
+  /** #9783: whether cycleP50Ms/cycleP95Ms cover the REQUESTED window. False when the window reaches past
+   *  orb_signals' retention horizon: the folded rollups carry counts, not durations, so percentiles over the
+   *  surviving rows would describe a shorter window than the caller asked for. The percentiles are null in
+   *  that case rather than quietly narrowed -- this says which it is. */
+  cycleTimeObservable: boolean;
 }
 
 function median(xs: number[]): number | null {
@@ -254,17 +264,51 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
   const windowDays = Number.isFinite(opts.windowDays) && (opts.windowDays as number) > 0 ? Math.min(opts.windowDays as number, 365) : 90;
   // Date-only cutoff (like computeGateEval) so it compares correctly whether received_at is ISO ('…T…Z')
   // or SQLite's CURRENT_TIMESTAMP space format ('YYYY-MM-DD HH:MM:SS').
-  const cutoff = new Date(Date.now() - windowDays * 86_400_000).toISOString().slice(0, 10);
+  const nowMs = Date.now();
+  const cutoff = new Date(nowMs - windowDays * 86_400_000).toISOString().slice(0, 10);
+
+  // #9783 follow-up: cycle time is the ONE metric the rollup cannot carry. The confusion matrix folds into
+  // per-day cells and sums back exactly, but cycleP50Ms/cycleP95Ms are percentiles over individual
+  // time_to_close_ms values -- percentiles are not summable, so no per-day count can reconstruct them, and
+  // orb_signal_rollups deliberately stores counts rather than a sample of durations (an unbounded sample is
+  // the growth problem retention exists to solve).
+  //
+  // So a window reaching past the retention horizon would compute percentiles over only the rows that
+  // survived, and report them as if they described the whole window -- silently, exactly the failure this
+  // change exists to prevent for the matrix. Instead the percentiles go null and this flag says why, the
+  // same posture gamingDetectionEligible/fleetFramingEligible take: a public surface can label the gap
+  // rather than be handed a number that quietly means something narrower than its window.
+  //
+  // Compared as instants, before the date-truncation above: at the default windowDays === the retention
+  // window the two are equal and cycle time is fully observable. (The date-only cutoff then widens the query
+  // by under a day at the oldest edge, where the prune may already have taken rows -- a sub-day sliver out
+  // of ninety, which percentiles are robust to and which predates this change.)
+  const retentionCutoffIso = retentionCutoffIsoForTable("orb_signals", nowMs);
+  const cycleTimeObservable = retentionCutoffIso === null || nowMs - windowDays * 86_400_000 >= Date.parse(retentionCutoffIso);
 
   let cells: Cell[] = [];
   let cycleRows: CycleTime[] = [];
   let registered = new Set<string>();
   try {
+    // #9783: UNION the live rows with the folded rollups. orb_signals is pruned at 90 days into
+    // orb_signal_rollups (per instance, per day, whole confusion matrix), so reading only the raw table
+    // would make every window that reaches past the prune silently under-count -- which is exactly the
+    // failure mode that made adding retention here non-trivial in the first place. Both halves emit the same
+    // Cell shape, and foldInstance sums cells, so duplicate (instance, verdict, outcome, ...) tuples across
+    // the two sources add up correctly rather than needing a merge.
+    //
+    // Both halves take the same bound because `cutoff` above is already date-only, which is exactly what
+    // `day` is -- so the boundary date is included on both sides. (The public weekly trend cannot do this:
+    // it bounds on a full ISO instant, where a bare `day >= ?1` would drop the boundary day because a string
+    // prefix sorts before the string it prefixes. See public-fleet-accuracy-trend.ts.)
     const matrix = await env.DB
       .prepare(
         `SELECT instance_id, gate_verdict AS verdict, outcome, reversal_flag, gate_reasoncode_bucket, COUNT(*) AS n
-         FROM orb_signals WHERE received_at >= ?
-         GROUP BY instance_id, gate_verdict, outcome, reversal_flag, gate_reasoncode_bucket`,
+           FROM orb_signals WHERE received_at >= ?1
+          GROUP BY instance_id, gate_verdict, outcome, reversal_flag, gate_reasoncode_bucket
+         UNION ALL
+         SELECT instance_id, gate_verdict AS verdict, outcome, reversal_flag, gate_reasoncode_bucket, n
+           FROM orb_signal_rollups WHERE day >= ?1`,
       )
       .bind(cutoff)
       .all<Cell>();
@@ -297,6 +341,10 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
       // claim a fleet aggregate. instanceCount is 0 here, so the public surface labels it a self-report and
       // publishes the nulls it already had.
       fleetFramingEligible: false,
+      // The cycle percentiles here are null because there is no data at all, not because the window outran
+      // retention -- so this reports the window's real observability rather than piling a second reason onto
+      // the same nulls.
+      cycleTimeObservable,
     };
   }
 
@@ -391,15 +439,16 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
       reversalRate: median(nums((i) => i.reversalRate)),
       decisionAccuracy: pooledDecisionAccuracy,
       decisionAccuracyMedian: median(nums((i) => i.decisionAccuracy)),
-      cycleP50Ms: percentile(cycle, 50),
-      cycleP95Ms: percentile(cycle, 95),
+      cycleP50Ms: cycleTimeObservable ? percentile(cycle, 50) : null,
+      cycleP95Ms: cycleTimeObservable ? percentile(cycle, 95) : null,
       pooled,
     },
     instances,
     outliers,
     gamingPatternFlags,
     gamingDetectionEligible,
     fleetFramingEligible: eligible.length >= FLEET_FRAMING_MIN_INSTANCES,
+    cycleTimeObservable,
   };
 }
 

src/orb/fleet-admin.ts

@@ -20,7 +20,12 @@ export async function listFleetInstances(env: Env): Promise<{ instances: FleetIn
   const rows = await env.DB.prepare(
     `SELECT i.instance_id AS instanceId, i.registered AS registered, i.first_seen_at AS firstSeenAt,
             i.last_seen_at AS lastSeenAt, i.registered_at AS registeredAt,
-            (SELECT COUNT(*) FROM orb_signals s WHERE s.instance_id = i.instance_id) AS signalCount
+            -- #9783: raw rows PLUS folded rollups, so this stays a genuine LIFETIME count after orb_signals
+            -- starts pruning at 90 days. Reading only the raw table would silently turn it into "signals in
+            -- the retention window" with no change of label -- one of the two reasons a plain prune could not
+            -- be added to this table.
+            ((SELECT COUNT(*) FROM orb_signals s WHERE s.instance_id = i.instance_id)
+             + COALESCE((SELECT SUM(r.n) FROM orb_signal_rollups r WHERE r.instance_id = i.instance_id), 0)) AS signalCount
      FROM orb_instances i ORDER BY i.last_seen_at DESC`,
   ).all<{ instanceId: string; registered: number; firstSeenAt: string; lastSeenAt: string; registeredAt: string | null; signalCount: number }>();
   return { instances: (rows.results ?? []).map((row) => ({ ...row, registered: row.registered === 1 })) };