feat(status): publish per-component service status from the existing alerting stack

apps/loopover-ui/public/openapi.json

@@ -27612,6 +27612,23 @@
           }
         }
       }
+    },
+    "/v1/public/service-status": {
+      "get": {
+        "operationId": "getPublicServiceStatus",
+        "tags": [
+          "Public"
+        ],
+        "summary": "Per-component service status — is each component healthy right now",
+        "responses": {
+          "200": {
+            "description": "{ generatedAt, overall, components: [{ component, label, status, since, reason? }] } — `status` is `operational` | `degraded` | `outage` | `unknown`, and `overall` is the worst of them. `unknown` means the alerting source could not be read and is NEVER reported as healthy; `reason` is then a category, never a URL or hostname. `since` is the earliest firing alert's start for a non-operational component, else null. Public-safe: no host names, instance ids, capacity figures or raw alert labels"
+          },
+          "404": {
+            "description": "This deployment has no alerting source configured, so it publishes no status board"
+          }
+        }
+      }
     }
   },
   "servers": [

apps/loopover-ui/src/lib/selfhost-env-reference.ts

@@ -265,6 +265,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "INTERNAL_JOB_TOKEN",
     firstReference: "src/selfhost/preflight.ts",
   },
+  {
+    name: "LOOPOVER_ALERTMANAGER_URL",
+    firstReference: "src/selfhost/service-status.ts",
+  },
   {
     name: "LOOPOVER_API_TOKEN",
     firstReference: "src/selfhost/preflight.ts",
@@ -783,6 +787,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `GITHUB_WEBHOOK_SECRET` | `src/selfhost/preflight.ts` |",
   "| `HOME` | `src/selfhost/ai.ts` |",
   "| `INTERNAL_JOB_TOKEN` | `src/selfhost/preflight.ts` |",
+  "| `LOOPOVER_ALERTMANAGER_URL` | `src/selfhost/service-status.ts` |",
   "| `LOOPOVER_API_TOKEN` | `src/selfhost/preflight.ts` |",
   "| `LOOPOVER_CENTRAL_POSTHOG_KEY` | `src/selfhost/posthog.ts` |",
   "| `LOOPOVER_ENABLE_PAGERDUTY` | `src/services/notify-pagerduty.ts` |",

src/api/routes.ts

@@ -308,6 +308,7 @@ import { isRagEnabled } from "../review/rag-wire";
 import { loadDecisionLedgerTip, loadPublicDecisionRecord, loadPublicLedgerRow, verifyDecisionLedger } from "../review/decision-record";
 import { buildEvalScoreRecordsFromRulePrecision, filterEvalScoreRecords } from "../review/eval-score-records";
 import { buildPublicCorpusCommitments } from "../review/public-eval-corpus";
+import { isServiceStatusEnabled, loadServiceStatus } from "../selfhost/service-status";
 import { anchorSigningInput, buildLedgerAnchorPayload, currentAnchorKey, diagnoseAnchorPublicKeys, parseAnchorPublicKeys, publicAnchorStatus, signLedgerAnchorPayload } from "../review/ledger-anchor";
 import { resolveProofPage } from "../review/proof-summary";
 import { renderProofBadgeSvg } from "./proof-badge";
@@ -919,6 +920,21 @@ export function createApp() {
     return c.json(corpus);
   });
 
+  // #9983 (slice of #9747): the public status board, sourced from THIS deployment's own alerting stack --
+  // the same Grafana-managed rules that page the on-call rotation through Alertmanager. Reusing that source
+  // rather than adding a status-page-only probe means the page cannot disagree with what actually pages a
+  // human. 404s where no alerting source is configured (the hosted Worker) instead of publishing a board that
+  // reads "unknown" forever. Reachable publicly on the Orb through the existing Cloudflare Tunnel, which
+  // already routes /v1/public/* -- no tunnel change was needed to ship this.
+  app.get("/v1/public/service-status", async (c) => {
+    if (!isServiceStatusEnabled(c.env)) return c.json({ error: "not_found" }, 404);
+    const status = await loadServiceStatus(c.env);
+    // Shorter than the sibling public surfaces on purpose: this is the endpoint people refresh DURING an
+    // incident, and a 60s cache would keep serving "operational" for a minute after an outage started.
+    c.header("Cache-Control", "public, max-age=15, stale-while-revalidate=30");
+    return c.json(status);
+  });
+
   app.get("/v1/public/eval-scores", async (c) => {
     const publicStatsManifestOverride = await resolvePublicStatsManifestOverride(c.env);
     if (!isPublicStatsEnabled(c.env, publicStatsManifestOverride)) return c.json({ error: "not_found" }, 404);

src/auth/route-auth.ts

@@ -44,6 +44,9 @@ export function requiresApiToken(path: string): boolean {
   // repos, no metadata beyond the one confidence field replay needs), so there is nothing here an
   // Authorization header would be protecting.
   if (path === "/v1/public/eval-corpus") return false;
+  // #9983: component name + status only, no host, instance or capacity detail -- added in the SAME PR as its
+  // route so the two cannot drift.
+  if (path === "/v1/public/service-status") return false;
   // #9269: the single-row read, added in the SAME PR as its route so the two can never drift the way #9120's
   // sibling did. Regex (not a literal) because of the :seq path parameter.
   if (/^\/v1\/public\/decision-ledger\/row\/[^/]+$/.test(path)) return false;

src/env.d.ts

@@ -614,6 +614,12 @@ declare global {
      *  when this is off -- see isProofPageEnabledForRepo's recorded decision in review/proof-summary.ts. */
     LOOPOVER_PUBLIC_PROOF?: string;
     LOOPOVER_PUBLIC_STATS?: string;
+    /** #9983: base URL of this deployment's Alertmanager (the source behind `/v1/public/service-status`).
+     *  Unset means the deployment has no alerting stack -- the hosted Worker -- and the route 404s rather
+     *  than publishing a board that reads "unknown" for every component forever. On the Orb this is the
+     *  compose-network address, never a public one: the app reads it server-side and publishes only a
+     *  component name and status, so the URL itself is internal topology and never leaves the process. */
+    LOOPOVER_ALERTMANAGER_URL?: string;
     /** Proof of Power (#1059): comma-separated allowlist of repo full-names ("owner/repo") whose OWN historical
      *  review ledger (audit_events "published a review surface" + pull_requests terminal state) counts toward
      *  the public stats counter. DELIBERATELY SEPARATE from LOOPOVER_REVIEW_REPOS (the live per-PR-feature

src/openapi/spec.ts

@@ -1970,6 +1970,24 @@ export function buildOpenApiSpec() {
       404: { description: "No decision record persisted yet for this PR" },
     },
   });
+  // #9983 (slice of #9747): the public status board, sourced from this deployment's own Alertmanager -- the
+  // same alerting stack that pages the on-call rotation -- so the page cannot disagree with what actually
+  // fires. 404s where no alerting source is configured rather than publishing an all-unknown board.
+  registry.registerPath({
+    method: "get",
+    path: "/v1/public/service-status",
+    operationId: "getPublicServiceStatus",
+    tags: ["Public"],
+    summary: "Per-component service status — is each component healthy right now",
+    responses: {
+      200: {
+        description:
+          "{ generatedAt, overall, components: [{ component, label, status, since, reason? }] } — `status` is `operational` | `degraded` | `outage` | `unknown`, and `overall` is the worst of them. `unknown` means the alerting source could not be read and is NEVER reported as healthy; `reason` is then a category, never a URL or hostname. `since` is the earliest firing alert's start for a non-operational component, else null. Public-safe: no host names, instance ids, capacity figures or raw alert labels",
+      },
+      404: { description: "This deployment has no alerting source configured, so it publishes no status board" },
+    },
+  });
+
   registry.registerPath({
     method: "get",
     path: "/v1/public/eval-corpus",

src/selfhost/service-status.ts

@@ -0,0 +1,212 @@
+// Public service status, sourced from the deployment's OWN alerting stack (#9983, slice of #9747).
+//
+// WHY ALERTMANAGER RATHER THAN A NEW PROBE. #9747 says to reuse the existing health/alerting
+// instrumentation, and on the Orb that is Grafana-managed rules -> Alertmanager -> PagerDuty. A second,
+// status-page-only probe would be a second opinion about the same components, free to disagree with the one
+// that actually pages a human — and when they disagree, the public page is the one nobody is watching. Reading
+// the alerting stack means the page says exactly what the on-call rotation already believes.
+//
+// NO NEW INFRASTRUCTURE. The app container already reaches Alertmanager on the compose network (verified:
+// `fetch("http://alertmanager:9093/api/v2/alerts")` returns `[]` from inside the running container), and the
+// existing Cloudflare Tunnel already routes `^/v1/public/.*$` to this app, so the route is publicly reachable
+// the moment it ships — no tunnel edit and no new hostname, which matters because there is no `cert.pem` on
+// the box to mint one with.
+//
+// UNREACHABLE IS `unknown`, NEVER `operational`. This is the whole correctness content of the module. A status
+// page that renders green because it could not reach its source is worse than no status page: it actively
+// tells people the thing is fine at exactly the moment nobody can confirm it. Same false-green class as a
+// verifier that skips every claim and exits zero. So every failure path — unset URL, network error, timeout,
+// non-200, unparseable body — lands on `unknown` with a stated reason, and `operational` is only ever reached
+// by successfully reading the alert list and finding nothing firing for that component.
+//
+// PUBLIC-SAFE BY CONSTRUCTION. The response carries component name, status, and a since-timestamp. It never
+// carries a hostname, an instance id, a capacity figure, an alert annotation, or a label bag — alert payloads
+// routinely contain internal host and job labels, so this maps them to a fixed component vocabulary and drops
+// the original rather than filtering it. Whitelisting the shape is what keeps a new alert label from leaking
+// through a filter nobody updated.
+
+/** The components this deployment reports on. A FIXED vocabulary, not whatever labels the alerts happen to
+ *  carry: the mapping is what turns internal topology into a public name, and an unmapped alert must widen
+ *  this list deliberately rather than publish a label verbatim. */
+export const SERVICE_STATUS_COMPONENTS = ["review", "testing", "discovery"] as const;
+export type ServiceComponent = (typeof SERVICE_STATUS_COMPONENTS)[number];
+
+/** Human-facing names, so the public surface never shows an internal identifier. */
+const SERVICE_COMPONENT_LABELS: Record<ServiceComponent, string> = {
+  review: "ORB review service",
+  testing: "AMS testing service",
+  discovery: "Discovery index",
+};
+
+/**
+ * Which alert `service` label belongs to which public component.
+ *
+ * Read from the alert's `service` label only. Alertmanager alerts also carry `job`, `instance`, `pod` and
+ * similar, all of which name infrastructure; binding to one deliberate label keeps the public mapping
+ * explicit and keeps host identifiers out of the decision entirely.
+ */
+const SERVICE_LABEL_TO_COMPONENT: Record<string, ServiceComponent> = {
+  loopover: "review",
+  orb: "review",
+  review: "review",
+  ams: "testing",
+  testing: "testing",
+  "discovery-index": "discovery",
+  discovery: "discovery",
+};
+
+/** `operational` — read the source, nothing firing. `degraded` — a warning-severity alert is firing.
+ *  `outage` — a critical one is. `unknown` — the source could not be read, which is never green. */
+export type ComponentStatus = "operational" | "degraded" | "outage" | "unknown";
+
+export type ServiceComponentState = {
+  component: ServiceComponent;
+  label: string;
+  status: ComponentStatus;
+  /** When the current state began: the earliest firing alert's start for a non-operational component, else
+   *  null. Null rather than "now" — inventing a timestamp would imply a transition that did not happen. */
+  since: string | null;
+  /** Present only when status is `unknown`, saying why the source could not be read. Never carries a URL or
+   *  hostname: the reason is a category, not a connection string. */
+  reason?: string;
+};
+
+export type ServiceStatusPayload = {
+  generatedAt: string;
+  /** The worst component status, so a caller can answer "is anything wrong" without walking the list. */
+  overall: ComponentStatus;
+  components: ServiceComponentState[];
+};
+
+/** One Alertmanager alert, narrowed to the two fields this reads. Structural, so a newer Alertmanager that
+ *  adds fields still parses. */
+export type AlertmanagerAlert = {
+  labels?: Record<string, unknown> | undefined;
+  startsAt?: unknown;
+  status?: { state?: unknown } | undefined;
+};
+
+/** PURE. Severity ranking, so `overall` and per-component rollups agree on which state is worse. */
+const SEVERITY_RANK: Record<ComponentStatus, number> = { operational: 0, unknown: 1, degraded: 2, outage: 3 };
+
+/** PURE. The worse of two statuses. `unknown` outranks `operational` — a component we could not read must not
+ *  be averaged away by one we could. */
+export function worseStatus(a: ComponentStatus, b: ComponentStatus): ComponentStatus {
+  return SEVERITY_RANK[b] > SEVERITY_RANK[a] ? b : a;
+}
+
+/** PURE. An alert's severity mapped to a status. Anything that is not explicitly `critical` is treated as
+ *  `degraded`: an unrecognised severity is still a firing alert, and rounding it DOWN to operational would
+ *  hide it. */
+export function statusForSeverity(severity: unknown): ComponentStatus {
+  return typeof severity === "string" && severity.toLowerCase() === "critical" ? "outage" : "degraded";
+}
+
+/** PURE. Is this alert actually firing? Alertmanager returns suppressed alerts too when asked; a silenced or
+ *  inhibited alert is deliberately not paging anyone, so it must not colour the public board either. */
+export function isFiring(alert: AlertmanagerAlert): boolean {
+  const state = alert.status?.state;
+  return typeof state !== "string" || state === "active";
+}
+
+/**
+ * PURE. Fold firing alerts into a per-component board.
+ *
+ * Every component in the vocabulary appears, whether or not it has alerts — a status page that omits a healthy
+ * component is indistinguishable from one that forgot to check it. Alerts whose `service` label is unmapped
+ * are IGNORED rather than published under a made-up component: publishing an unrecognised internal label is
+ * exactly the leak this module's whitelist exists to prevent.
+ */
+export function buildServiceStatus(alerts: readonly AlertmanagerAlert[], generatedAt: string): ServiceStatusPayload {
+  const states = new Map<ServiceComponent, { status: ComponentStatus; since: string | null }>();
+  for (const component of SERVICE_STATUS_COMPONENTS) states.set(component, { status: "operational", since: null });
+
+  for (const alert of alerts) {
+    if (!isFiring(alert)) continue;
+    const service = alert.labels?.["service"];
+    const component = typeof service === "string" ? SERVICE_LABEL_TO_COMPONENT[service.toLowerCase()] : undefined;
+    if (component === undefined) continue;
+    const current = states.get(component)!;
+    const status = worseStatus(current.status, statusForSeverity(alert.labels?.["severity"]));
+    // Earliest start wins: the incident began when the FIRST alert for this component fired, not when the
+    // most recent one did.
+    const startsAt = typeof alert.startsAt === "string" ? alert.startsAt : null;
+    const since = current.since === null ? startsAt : startsAt === null ? current.since : startsAt < current.since ? startsAt : current.since;
+    states.set(component, { status, since });
+  }
+
+  const components = SERVICE_STATUS_COMPONENTS.map((component) => {
+    const state = states.get(component)!;
+    return {
+      component,
+      label: SERVICE_COMPONENT_LABELS[component],
+      status: state.status,
+      // `since` is only meaningful for a component that is currently in a non-operational state.
+      since: state.status === "operational" ? null : state.since,
+    };
+  });
+
+  return { generatedAt, overall: components.reduce<ComponentStatus>((worst, entry) => worseStatus(worst, entry.status), "operational"), components };
+}
+
+/** PURE. Every component `unknown`, for the paths where the source could not be read at all. */
+export function unknownServiceStatus(generatedAt: string, reason: string): ServiceStatusPayload {
+  return {
+    generatedAt,
+    overall: "unknown",
+    components: SERVICE_STATUS_COMPONENTS.map((component) => ({
+      component,
+      label: SERVICE_COMPONENT_LABELS[component],
+      status: "unknown" as const,
+      since: null,
+      reason,
+    })),
+  };
+}
+
+/** Is a status surface configured on this deployment? The hosted Worker has no Alertmanager, and a board that
+ *  reads "unknown" for everything forever is not a status page — the route 404s there instead. */
+export function isServiceStatusEnabled(env: { LOOPOVER_ALERTMANAGER_URL?: string | undefined }): boolean {
+  return (env.LOOPOVER_ALERTMANAGER_URL ?? "").trim() !== "";
+}
+
+/** How long to wait on the alerting source. A status endpoint that hangs is itself an outage, and this route
+ *  is unauthenticated and cacheable, so it must always answer. */
+const ALERTMANAGER_TIMEOUT_MS = 4_000;
+
+/**
+ * Read the deployment's alerting source and build the public board.
+ *
+ * NEVER THROWS, and never degrades to `operational`. Every failure lands on {@link unknownServiceStatus} with
+ * a category reason — the reason deliberately does not include the configured URL, which is internal topology.
+ */
+export async function loadServiceStatus(
+  env: { LOOPOVER_ALERTMANAGER_URL?: string | undefined },
+  options: { now?: Date | undefined; fetchImpl?: typeof fetch | undefined } = {},
+): Promise<ServiceStatusPayload> {
+  const generatedAt = (options.now ?? new Date()).toISOString();
+  const base = (env.LOOPOVER_ALERTMANAGER_URL ?? "").trim();
+  if (base === "") return unknownServiceStatus(generatedAt, "alerting source not configured");
+
+  const doFetch = options.fetchImpl ?? fetch;
+  let response: Response;
+  try {
+    response = await doFetch(`${base.replace(/\/+$/, "")}/api/v2/alerts`, { signal: AbortSignal.timeout(ALERTMANAGER_TIMEOUT_MS) });
+  } catch {
+    // Network error or timeout. Both mean the same thing publicly: we could not check.
+    return unknownServiceStatus(generatedAt, "alerting source unreachable");
+  }
+  if (!response.ok) return unknownServiceStatus(generatedAt, "alerting source returned an error");
+
+  let alerts: unknown;
+  try {
+    alerts = await response.json();
+  } catch {
+    return unknownServiceStatus(generatedAt, "alerting source returned an unreadable response");
+  }
+  // A non-array body is a source we do not understand, which is not the same as no alerts — treating it as
+  // "nothing firing" would publish green off a payload we failed to parse.
+  if (!Array.isArray(alerts)) return unknownServiceStatus(generatedAt, "alerting source returned an unexpected shape");
+
+  return buildServiceStatus(alerts as AlertmanagerAlert[], generatedAt);
+}