feat(status): uptime and incident history, derived from persisted samples

migrations/0209_service_status_samples.sql

@@ -0,0 +1,37 @@
+-- #9985 (slice 2 of #9747): the source for "has it been healthy?".
+--
+-- /v1/public/service-status answers "is each component healthy RIGHT NOW" by reading Alertmanager's active
+-- alerts. It cannot answer the past tense, because there is nothing to read: Alertmanager serves active
+-- alerts only, resolved ones are gone, there is no Prometheus on the box (metrics flow through the
+-- otel-collector), and nothing persisted a history. #9983 therefore published no uptime figure rather than
+-- deriving one from a single live sample.
+--
+-- This is that history: one row per component per cron tick. Uptime and incidents are DERIVED from these
+-- rows rather than posted by hand, which is what #9747 means by "incidents appear without manual posting".
+--
+-- `status` carries the same vocabulary the endpoint publishes -- operational | degraded | outage | unknown --
+-- and `unknown` is stored rather than skipped ON PURPOSE. A tick where the alerting source could not be read
+-- is UNMEASURED time, not healthy time and not an outage; dropping those rows would silently shrink the
+-- window and let a period of blindness read as uptime. Keeping them is what lets the read path report
+-- coverage separately from the percentage.
+--
+-- No UNIQUE on (component, sampled_at): two ticks in the same second are harmless to an aggregate over
+-- contiguous runs, whereas a constraint violation on a best-effort telemetry write is not.
+CREATE TABLE IF NOT EXISTS service_status_samples (
+  id          INTEGER PRIMARY KEY AUTOINCREMENT,
+  component   TEXT NOT NULL,
+  status      TEXT NOT NULL,
+  sampled_at  TEXT NOT NULL
+);
+
+-- The read is always "this component, over this trailing window, in time order", which this covers exactly.
+CREATE INDEX IF NOT EXISTS service_status_samples_component_time
+  ON service_status_samples (component, sampled_at);
+
+-- The retention prune deletes by age alone (`sampled_at < cutoff`, across every component), so it needs an
+-- index LEADING with that column -- the composite above leads with `component` and cannot serve it. Both
+-- exist because the two access patterns are genuinely different: the read is always per-component over a
+-- window, the prune is always all-components before a cutoff. test/unit/retention.test.ts enforces the
+-- latter for every table in RETENTION_POLICY, which is how the omission surfaced.
+CREATE INDEX IF NOT EXISTS service_status_samples_sampled_at
+  ON service_status_samples (sampled_at);

scripts/check-schema-drift.ts

@@ -46,6 +46,10 @@ export const RAW_SQL_ONLY_TABLES: Set<string> = new Set([
   "decision_ledger_anchors",
   "decision_records",
   "decision_replay_inputs",
+  // #9985: written and read only through raw statements in selfhost/service-status-history.ts, the same
+  // posture as the decision_* tables above. Declaring it in the drizzle schema would add an export nothing
+  // imports (dead-exports:check rejects that, correctly) purely to satisfy this checker.
+  "service_status_samples",
   // #9028: same raw-SQL-only posture as its sibling above — an operator-private blob table the drizzle
   // schema never queries (writes go through persistDecisionReplayPrompt's raw statement; reads happen only
   // in the operator's own extract SQL for the replay CLI).

src/api/routes.ts

@@ -309,6 +309,7 @@ import { loadDecisionLedgerTip, loadPublicDecisionRecord, loadPublicLedgerRow, v
 import { buildEvalScoreRecordsFromRulePrecision, filterEvalScoreRecords } from "../review/eval-score-records";
 import { buildPublicCorpusCommitments } from "../review/public-eval-corpus";
 import { isServiceStatusEnabled, loadServiceStatus } from "../selfhost/service-status";
+import { buildComponentHistory, loadServiceStatusSamples } from "../selfhost/service-status-history";
 import { anchorSigningInput, buildLedgerAnchorPayload, currentAnchorKey, diagnoseAnchorPublicKeys, parseAnchorPublicKeys, publicAnchorStatus, signLedgerAnchorPayload } from "../review/ledger-anchor";
 import { resolveProofPage } from "../review/proof-summary";
 import { renderProofBadgeSvg } from "./proof-badge";
@@ -929,10 +930,17 @@ export function createApp() {
   app.get("/v1/public/service-status", async (c) => {
     if (!isServiceStatusEnabled(c.env)) return c.json({ error: "not_found" }, 404);
     const status = await loadServiceStatus(c.env);
+    // #9985: the past tense, derived from persisted samples rather than posted by hand. Attached per
+    // component so a reader never has to correlate two lists, and computed from the SAME status vocabulary
+    // the live board publishes.
+    const now = new Date();
+    const components = await Promise.all(
+      status.components.map(async (entry) => ({ ...entry, ...buildComponentHistory(await loadServiceStatusSamples(c.env, entry.component, now), now.getTime()) })),
+    );
     // Shorter than the sibling public surfaces on purpose: this is the endpoint people refresh DURING an
     // incident, and a 60s cache would keep serving "operational" for a minute after an outage started.
     c.header("Cache-Control", "public, max-age=15, stale-while-revalidate=30");
-    return c.json(status);
+    return c.json({ ...status, components });
   });
 
   app.get("/v1/public/eval-scores", async (c) => {

src/db/retention.ts

@@ -22,6 +22,12 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [
   // needs would be gone by the time the fold ran, and the rollup would permanently over-count exactly the
   // PRs the live query never counted.
   { table: "orb_pr_outcomes", column: "occurred_at", days: 90 },
+  // #9985: status samples are high-volume (one row per component per ~2-min tick) and low-value once the
+  // window that reads them has passed. 35 days deliberately EXCEEDS the widest reported window (30), so the
+  // 30-day uptime figure never silently loses its own tail to the prune -- a retention equal to the window
+  // would make the oldest day of every month quietly unmeasured. No rollup is needed while that inequality
+  // holds; widening UPTIME_WINDOW_DAYS past 35 without widening this would break it.
+  { table: "service_status_samples", column: "sampled_at", days: 35 },
   // #9783: orb_signals had no rule at all and grew without bound, while being a PUBLIC data source
   // (computeFleetAnalytics' /fairness headline, and #9775's weekly fleet trend). It could not simply be
   // pruned: listFleetInstances reports a LIFETIME signalCount, and /v1/internal/fleet/analytics takes an
@@ -127,6 +133,7 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [
 // not scan-optimal, which is acceptable for the lower row counts of the tables that fall back today.
 export const RETENTION_PK_COLUMN: Readonly<Record<string, string>> = {
   audit_events: "id",
+  service_status_samples: "id",
   // #9783: orb_signals has `id INTEGER PRIMARY KEY` (migrations/0060), so it maps cleanly here rather than
   // joining RETENTION_COMPOSITE_PK_TABLES -- its UNIQUE (instance_id, repo_hash, pr_hash) is the ingest
   // dedup key, not its primary key. The fold below uses its own bounded-slice path rather than the generic

src/index.ts

@@ -143,6 +143,10 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController):
   // impossible. The job's own decideLedgerAnchorSchedule (in ledger-anchor-scheduler.ts) is what actually
   // decides whether THIS tick does anything; a cheap two-query no-op the overwhelming majority of ticks.
   jobs.push({ type: "anchor-decision-ledger", requestedBy: "schedule", isHourly });
+  // #9985: sampled every tick, unconditionally. The board it feeds is only as good as its resolution, and a
+  // deployment with no alerting source configured simply records `unknown` -- which the aggregation reports
+  // as unmeasured coverage rather than as health.
+  jobs.push({ type: "sample-service-status", requestedBy: "schedule" });
   const selfHostedReviews = isSelfHostedReviewRuntime(env);
   const queueSnapshot = selfHostedReviews
     ? await queueSnapshotFromBinding(env.JOBS).catch((error) => {

src/queue/job-dispatch.ts

@@ -53,6 +53,8 @@ import { withInstallationTokenRetry } from "../github/app";
 import { runScheduledLedgerAnchor, resolveGitAnchorTarget } from "../review/ledger-anchor-scheduler";
 import { submitToGitAnchor } from "../review/ledger-anchor-git";
 import { incr } from "../selfhost/metrics";
+import { loadServiceStatus } from "../selfhost/service-status";
+import { recordServiceStatusSamples } from "../selfhost/service-status-history";
 import { generateSignalSnapshots } from "./signal-snapshot";
 import { isDecisionAuditEnabled, runDecisionAuditSample } from "../review/decision-audit";
 import { isRiskControlEnabled, runRiskControlRecalibration } from "../review/risk-control-wire";
@@ -97,6 +99,18 @@ export async function processJob(env: Env, message: JobMessage): Promise<void> {
     case "refresh-registry":
       await refreshRegistry(env);
       return;
+    case "sample-service-status": {
+      // #9985: read the live board through the SAME loader the public route uses, then persist one row per
+      // component. Reusing the loader is what keeps the history and the live status the same measurement --
+      // a second read path here could report a component healthy while the endpoint called it degraded, and
+      // the derived uptime would then describe a board nobody ever saw.
+      const status = await loadServiceStatus(env);
+      await recordServiceStatusSamples(
+        env,
+        status.components.map((entry) => ({ component: entry.component, status: entry.status })),
+      );
+      return;
+    }
     case "anchor-decision-ledger": {
       // Git anchoring runs only when BOTH the target (owner/repo, #9273) and an installation id (to mint a
       // token through) are configured -- unset means that backend simply isn't wired up yet; Rekor still

src/selfhost/queue-common.ts

@@ -801,6 +801,9 @@ const IMMEDIATE_SCHEDULED_JOB_TYPES = new Set<string>([
   // must fire independent of the hourly clock), and is a cheap 1-2 query no-op the overwhelming majority of
   // ticks -- not a heavy per-repo fan-out parent, so it needs no phase-spreading either.
   "anchor-decision-ledger",
+  // #9985: a handful of INSERTs, no fan-out. Phase-spreading it would jitter the sample cadence, and the
+  // derived uptime reads evenly-spaced samples as evenly-weighted time.
+  "sample-service-status",
 ]);
 
 export function scheduledEnqueueDelaySeconds(jobType: string): number {