From ac6c0461087418fcee4de5a07e42c3664f8768b9 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:24:24 -0700 Subject: [PATCH 1/2] feat(status): publish per-component service status from the existing alerting stack #9747's backend slice, taking option C: source the public board from the deployment's OWN Grafana-managed rules -> Alertmanager -> PagerDuty stack rather than adding a status-page-only probe. A second probe would be a second opinion about the same components, free to disagree with the one that actually pages a human -- and when they disagree, the public page is the one nobody is watching. NO NEW INFRASTRUCTURE, verified on the box rather than assumed. The app container already reaches Alertmanager on the compose network (`fetch( "http://alertmanager:9093/api/v2/alerts")` returns `[]` from inside the running container), and /etc/cloudflared/config.yml already routes `^/v1/public/.*$` on shots.loopover.ai to this app -- so the route is publicly reachable the moment it ships, with no tunnel edit and no new hostname. That last part matters: there is no cert.pem on the box, so a new hostname could not be minted there. UNREACHABLE IS `unknown`, NEVER `operational`. This is the module's whole correctness content. A status page that renders green because it could not reach its source is worse than no status page -- it says the thing is fine at exactly the moment nobody can confirm it. Every failure path lands on `unknown`: unset URL, network error, timeout, non-200, unparseable body, and a body that parses but is not an array. That last one is the subtle case: `{}` is valid JSON with no alerts in it, and reading it as "nothing firing" would publish green off a payload we failed to understand. `unknown` also outranks `operational` in the rollup, so a component we could not read is never averaged away by one we could. An unrecognised severity rounds UP to degraded. A firing alert whose severity we do not know is still firing, and rounding down would hide it. PUBLIC-SAFE BY CONSTRUCTION, not by filtering. Alert payloads routinely carry `job`, `instance`, `pod` and other host identifiers, so alerts are mapped onto a fixed component vocabulary and the originals dropped -- an unmapped `service` label is ignored rather than published verbatim. Whitelisting the shape is what keeps a new alert label from leaking through a filter nobody updated. The failure reason is a category, never the configured URL, and a test asserts the serialized response contains no host, port, instance or capacity token. 404s where no alerting source is configured (the hosted Worker) instead of publishing a board that reads "unknown" for every component forever. Cached for 15s, not the sibling surfaces' 60s: this is the endpoint people refresh DURING an incident, and a minute of stale "operational" is the wrong failure. Uptime percentages and incident history are deliberately NOT here -- Alertmanager serves active alerts only, there is no Prometheus on the box, and no sample table, so there is no honest source for "has it been healthy?" yet. Fabricating one from a single live sample is the option this declines. Closes #9983 --- apps/loopover-ui/public/openapi.json | 17 ++ .../src/lib/selfhost-env-reference.ts | 5 + src/api/routes.ts | 16 ++ src/auth/route-auth.ts | 3 + src/env.d.ts | 6 + src/openapi/spec.ts | 18 ++ src/selfhost/service-status.ts | 212 +++++++++++++++ test/unit/service-status.test.ts | 252 ++++++++++++++++++ 8 files changed, 529 insertions(+) create mode 100644 src/selfhost/service-status.ts create mode 100644 test/unit/service-status.test.ts diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json index a451e5600d..3a40a3a5b4 100644 --- a/apps/loopover-ui/public/openapi.json +++ b/apps/loopover-ui/public/openapi.json @@ -27612,6 +27612,23 @@ } } } + }, + "/v1/public/service-status": { + "get": { + "operationId": "getPublicServiceStatus", + "tags": [ + "Public" + ], + "summary": "Per-component service status — is each component healthy right now", + "responses": { + "200": { + "description": "{ generatedAt, overall, components: [{ component, label, status, since, reason? }] } — `status` is `operational` | `degraded` | `outage` | `unknown`, and `overall` is the worst of them. `unknown` means the alerting source could not be read and is NEVER reported as healthy; `reason` is then a category, never a URL or hostname. `since` is the earliest firing alert's start for a non-operational component, else null. Public-safe: no host names, instance ids, capacity figures or raw alert labels" + }, + "404": { + "description": "This deployment has no alerting source configured, so it publishes no status board" + } + } + } } }, "servers": [ diff --git a/apps/loopover-ui/src/lib/selfhost-env-reference.ts b/apps/loopover-ui/src/lib/selfhost-env-reference.ts index 45d9ef0c67..a85f36d758 100644 --- a/apps/loopover-ui/src/lib/selfhost-env-reference.ts +++ b/apps/loopover-ui/src/lib/selfhost-env-reference.ts @@ -265,6 +265,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "INTERNAL_JOB_TOKEN", firstReference: "src/selfhost/preflight.ts", }, + { + name: "LOOPOVER_ALERTMANAGER_URL", + firstReference: "src/selfhost/service-status.ts", + }, { name: "LOOPOVER_API_TOKEN", firstReference: "src/selfhost/preflight.ts", @@ -783,6 +787,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `GITHUB_WEBHOOK_SECRET` | `src/selfhost/preflight.ts` |", "| `HOME` | `src/selfhost/ai.ts` |", "| `INTERNAL_JOB_TOKEN` | `src/selfhost/preflight.ts` |", + "| `LOOPOVER_ALERTMANAGER_URL` | `src/selfhost/service-status.ts` |", "| `LOOPOVER_API_TOKEN` | `src/selfhost/preflight.ts` |", "| `LOOPOVER_CENTRAL_POSTHOG_KEY` | `src/selfhost/posthog.ts` |", "| `LOOPOVER_ENABLE_PAGERDUTY` | `src/services/notify-pagerduty.ts` |", diff --git a/src/api/routes.ts b/src/api/routes.ts index d46a7deb00..1b0ab81da5 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -308,6 +308,7 @@ import { isRagEnabled } from "../review/rag-wire"; import { loadDecisionLedgerTip, loadPublicDecisionRecord, loadPublicLedgerRow, verifyDecisionLedger } from "../review/decision-record"; import { buildEvalScoreRecordsFromRulePrecision, filterEvalScoreRecords } from "../review/eval-score-records"; import { buildPublicCorpusCommitments } from "../review/public-eval-corpus"; +import { isServiceStatusEnabled, loadServiceStatus } from "../selfhost/service-status"; import { anchorSigningInput, buildLedgerAnchorPayload, currentAnchorKey, diagnoseAnchorPublicKeys, parseAnchorPublicKeys, publicAnchorStatus, signLedgerAnchorPayload } from "../review/ledger-anchor"; import { resolveProofPage } from "../review/proof-summary"; import { renderProofBadgeSvg } from "./proof-badge"; @@ -919,6 +920,21 @@ export function createApp() { return c.json(corpus); }); + // #9983 (slice of #9747): the public status board, sourced from THIS deployment's own alerting stack -- + // the same Grafana-managed rules that page the on-call rotation through Alertmanager. Reusing that source + // rather than adding a status-page-only probe means the page cannot disagree with what actually pages a + // human. 404s where no alerting source is configured (the hosted Worker) instead of publishing a board that + // reads "unknown" forever. Reachable publicly on the Orb through the existing Cloudflare Tunnel, which + // already routes /v1/public/* -- no tunnel change was needed to ship this. + app.get("/v1/public/service-status", async (c) => { + if (!isServiceStatusEnabled(c.env)) return c.json({ error: "not_found" }, 404); + const status = await loadServiceStatus(c.env); + // Shorter than the sibling public surfaces on purpose: this is the endpoint people refresh DURING an + // incident, and a 60s cache would keep serving "operational" for a minute after an outage started. + c.header("Cache-Control", "public, max-age=15, stale-while-revalidate=30"); + return c.json(status); + }); + app.get("/v1/public/eval-scores", async (c) => { const publicStatsManifestOverride = await resolvePublicStatsManifestOverride(c.env); if (!isPublicStatsEnabled(c.env, publicStatsManifestOverride)) return c.json({ error: "not_found" }, 404); diff --git a/src/auth/route-auth.ts b/src/auth/route-auth.ts index f54b2cb584..3e407bd86b 100644 --- a/src/auth/route-auth.ts +++ b/src/auth/route-auth.ts @@ -44,6 +44,9 @@ export function requiresApiToken(path: string): boolean { // repos, no metadata beyond the one confidence field replay needs), so there is nothing here an // Authorization header would be protecting. if (path === "/v1/public/eval-corpus") return false; + // #9983: component name + status only, no host, instance or capacity detail -- added in the SAME PR as its + // route so the two cannot drift. + if (path === "/v1/public/service-status") return false; // #9269: the single-row read, added in the SAME PR as its route so the two can never drift the way #9120's // sibling did. Regex (not a literal) because of the :seq path parameter. if (/^\/v1\/public\/decision-ledger\/row\/[^/]+$/.test(path)) return false; diff --git a/src/env.d.ts b/src/env.d.ts index 5df8680ab8..a5bee78c28 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -614,6 +614,12 @@ declare global { * when this is off -- see isProofPageEnabledForRepo's recorded decision in review/proof-summary.ts. */ LOOPOVER_PUBLIC_PROOF?: string; LOOPOVER_PUBLIC_STATS?: string; + /** #9983: base URL of this deployment's Alertmanager (the source behind `/v1/public/service-status`). + * Unset means the deployment has no alerting stack -- the hosted Worker -- and the route 404s rather + * than publishing a board that reads "unknown" for every component forever. On the Orb this is the + * compose-network address, never a public one: the app reads it server-side and publishes only a + * component name and status, so the URL itself is internal topology and never leaves the process. */ + LOOPOVER_ALERTMANAGER_URL?: string; /** Proof of Power (#1059): comma-separated allowlist of repo full-names ("owner/repo") whose OWN historical * review ledger (audit_events "published a review surface" + pull_requests terminal state) counts toward * the public stats counter. DELIBERATELY SEPARATE from LOOPOVER_REVIEW_REPOS (the live per-PR-feature diff --git a/src/openapi/spec.ts b/src/openapi/spec.ts index 1be794cc7e..c0e67e7eff 100644 --- a/src/openapi/spec.ts +++ b/src/openapi/spec.ts @@ -1970,6 +1970,24 @@ export function buildOpenApiSpec() { 404: { description: "No decision record persisted yet for this PR" }, }, }); + // #9983 (slice of #9747): the public status board, sourced from this deployment's own Alertmanager -- the + // same alerting stack that pages the on-call rotation -- so the page cannot disagree with what actually + // fires. 404s where no alerting source is configured rather than publishing an all-unknown board. + registry.registerPath({ + method: "get", + path: "/v1/public/service-status", + operationId: "getPublicServiceStatus", + tags: ["Public"], + summary: "Per-component service status — is each component healthy right now", + responses: { + 200: { + description: + "{ generatedAt, overall, components: [{ component, label, status, since, reason? }] } — `status` is `operational` | `degraded` | `outage` | `unknown`, and `overall` is the worst of them. `unknown` means the alerting source could not be read and is NEVER reported as healthy; `reason` is then a category, never a URL or hostname. `since` is the earliest firing alert's start for a non-operational component, else null. Public-safe: no host names, instance ids, capacity figures or raw alert labels", + }, + 404: { description: "This deployment has no alerting source configured, so it publishes no status board" }, + }, + }); + registry.registerPath({ method: "get", path: "/v1/public/eval-corpus", diff --git a/src/selfhost/service-status.ts b/src/selfhost/service-status.ts new file mode 100644 index 0000000000..d34cb4cf07 --- /dev/null +++ b/src/selfhost/service-status.ts @@ -0,0 +1,212 @@ +// Public service status, sourced from the deployment's OWN alerting stack (#9983, slice of #9747). +// +// WHY ALERTMANAGER RATHER THAN A NEW PROBE. #9747 says to reuse the existing health/alerting +// instrumentation, and on the Orb that is Grafana-managed rules -> Alertmanager -> PagerDuty. A second, +// status-page-only probe would be a second opinion about the same components, free to disagree with the one +// that actually pages a human — and when they disagree, the public page is the one nobody is watching. Reading +// the alerting stack means the page says exactly what the on-call rotation already believes. +// +// NO NEW INFRASTRUCTURE. The app container already reaches Alertmanager on the compose network (verified: +// `fetch("http://alertmanager:9093/api/v2/alerts")` returns `[]` from inside the running container), and the +// existing Cloudflare Tunnel already routes `^/v1/public/.*$` to this app, so the route is publicly reachable +// the moment it ships — no tunnel edit and no new hostname, which matters because there is no `cert.pem` on +// the box to mint one with. +// +// UNREACHABLE IS `unknown`, NEVER `operational`. This is the whole correctness content of the module. A status +// page that renders green because it could not reach its source is worse than no status page: it actively +// tells people the thing is fine at exactly the moment nobody can confirm it. Same false-green class as a +// verifier that skips every claim and exits zero. So every failure path — unset URL, network error, timeout, +// non-200, unparseable body — lands on `unknown` with a stated reason, and `operational` is only ever reached +// by successfully reading the alert list and finding nothing firing for that component. +// +// PUBLIC-SAFE BY CONSTRUCTION. The response carries component name, status, and a since-timestamp. It never +// carries a hostname, an instance id, a capacity figure, an alert annotation, or a label bag — alert payloads +// routinely contain internal host and job labels, so this maps them to a fixed component vocabulary and drops +// the original rather than filtering it. Whitelisting the shape is what keeps a new alert label from leaking +// through a filter nobody updated. + +/** The components this deployment reports on. A FIXED vocabulary, not whatever labels the alerts happen to + * carry: the mapping is what turns internal topology into a public name, and an unmapped alert must widen + * this list deliberately rather than publish a label verbatim. */ +export const SERVICE_STATUS_COMPONENTS = ["review", "testing", "discovery"] as const; +export type ServiceComponent = (typeof SERVICE_STATUS_COMPONENTS)[number]; + +/** Human-facing names, so the public surface never shows an internal identifier. */ +const SERVICE_COMPONENT_LABELS: Record = { + review: "ORB review service", + testing: "AMS testing service", + discovery: "Discovery index", +}; + +/** + * Which alert `service` label belongs to which public component. + * + * Read from the alert's `service` label only. Alertmanager alerts also carry `job`, `instance`, `pod` and + * similar, all of which name infrastructure; binding to one deliberate label keeps the public mapping + * explicit and keeps host identifiers out of the decision entirely. + */ +const SERVICE_LABEL_TO_COMPONENT: Record = { + loopover: "review", + orb: "review", + review: "review", + ams: "testing", + testing: "testing", + "discovery-index": "discovery", + discovery: "discovery", +}; + +/** `operational` — read the source, nothing firing. `degraded` — a warning-severity alert is firing. + * `outage` — a critical one is. `unknown` — the source could not be read, which is never green. */ +export type ComponentStatus = "operational" | "degraded" | "outage" | "unknown"; + +export type ServiceComponentState = { + component: ServiceComponent; + label: string; + status: ComponentStatus; + /** When the current state began: the earliest firing alert's start for a non-operational component, else + * null. Null rather than "now" — inventing a timestamp would imply a transition that did not happen. */ + since: string | null; + /** Present only when status is `unknown`, saying why the source could not be read. Never carries a URL or + * hostname: the reason is a category, not a connection string. */ + reason?: string; +}; + +export type ServiceStatusPayload = { + generatedAt: string; + /** The worst component status, so a caller can answer "is anything wrong" without walking the list. */ + overall: ComponentStatus; + components: ServiceComponentState[]; +}; + +/** One Alertmanager alert, narrowed to the two fields this reads. Structural, so a newer Alertmanager that + * adds fields still parses. */ +export type AlertmanagerAlert = { + labels?: Record | undefined; + startsAt?: unknown; + status?: { state?: unknown } | undefined; +}; + +/** PURE. Severity ranking, so `overall` and per-component rollups agree on which state is worse. */ +const SEVERITY_RANK: Record = { operational: 0, unknown: 1, degraded: 2, outage: 3 }; + +/** PURE. The worse of two statuses. `unknown` outranks `operational` — a component we could not read must not + * be averaged away by one we could. */ +export function worseStatus(a: ComponentStatus, b: ComponentStatus): ComponentStatus { + return SEVERITY_RANK[b] > SEVERITY_RANK[a] ? b : a; +} + +/** PURE. An alert's severity mapped to a status. Anything that is not explicitly `critical` is treated as + * `degraded`: an unrecognised severity is still a firing alert, and rounding it DOWN to operational would + * hide it. */ +export function statusForSeverity(severity: unknown): ComponentStatus { + return typeof severity === "string" && severity.toLowerCase() === "critical" ? "outage" : "degraded"; +} + +/** PURE. Is this alert actually firing? Alertmanager returns suppressed alerts too when asked; a silenced or + * inhibited alert is deliberately not paging anyone, so it must not colour the public board either. */ +export function isFiring(alert: AlertmanagerAlert): boolean { + const state = alert.status?.state; + return typeof state !== "string" || state === "active"; +} + +/** + * PURE. Fold firing alerts into a per-component board. + * + * Every component in the vocabulary appears, whether or not it has alerts — a status page that omits a healthy + * component is indistinguishable from one that forgot to check it. Alerts whose `service` label is unmapped + * are IGNORED rather than published under a made-up component: publishing an unrecognised internal label is + * exactly the leak this module's whitelist exists to prevent. + */ +export function buildServiceStatus(alerts: readonly AlertmanagerAlert[], generatedAt: string): ServiceStatusPayload { + const states = new Map(); + for (const component of SERVICE_STATUS_COMPONENTS) states.set(component, { status: "operational", since: null }); + + for (const alert of alerts) { + if (!isFiring(alert)) continue; + const service = alert.labels?.["service"]; + const component = typeof service === "string" ? SERVICE_LABEL_TO_COMPONENT[service.toLowerCase()] : undefined; + if (component === undefined) continue; + const current = states.get(component)!; + const status = worseStatus(current.status, statusForSeverity(alert.labels?.["severity"])); + // Earliest start wins: the incident began when the FIRST alert for this component fired, not when the + // most recent one did. + const startsAt = typeof alert.startsAt === "string" ? alert.startsAt : null; + const since = current.since === null ? startsAt : startsAt === null ? current.since : startsAt < current.since ? startsAt : current.since; + states.set(component, { status, since }); + } + + const components = SERVICE_STATUS_COMPONENTS.map((component) => { + const state = states.get(component)!; + return { + component, + label: SERVICE_COMPONENT_LABELS[component], + status: state.status, + // `since` is only meaningful for a component that is currently in a non-operational state. + since: state.status === "operational" ? null : state.since, + }; + }); + + return { generatedAt, overall: components.reduce((worst, entry) => worseStatus(worst, entry.status), "operational"), components }; +} + +/** PURE. Every component `unknown`, for the paths where the source could not be read at all. */ +export function unknownServiceStatus(generatedAt: string, reason: string): ServiceStatusPayload { + return { + generatedAt, + overall: "unknown", + components: SERVICE_STATUS_COMPONENTS.map((component) => ({ + component, + label: SERVICE_COMPONENT_LABELS[component], + status: "unknown" as const, + since: null, + reason, + })), + }; +} + +/** Is a status surface configured on this deployment? The hosted Worker has no Alertmanager, and a board that + * reads "unknown" for everything forever is not a status page — the route 404s there instead. */ +export function isServiceStatusEnabled(env: { LOOPOVER_ALERTMANAGER_URL?: string | undefined }): boolean { + return (env.LOOPOVER_ALERTMANAGER_URL ?? "").trim() !== ""; +} + +/** How long to wait on the alerting source. A status endpoint that hangs is itself an outage, and this route + * is unauthenticated and cacheable, so it must always answer. */ +const ALERTMANAGER_TIMEOUT_MS = 4_000; + +/** + * Read the deployment's alerting source and build the public board. + * + * NEVER THROWS, and never degrades to `operational`. Every failure lands on {@link unknownServiceStatus} with + * a category reason — the reason deliberately does not include the configured URL, which is internal topology. + */ +export async function loadServiceStatus( + env: { LOOPOVER_ALERTMANAGER_URL?: string | undefined }, + options: { now?: Date | undefined; fetchImpl?: typeof fetch | undefined } = {}, +): Promise { + const generatedAt = (options.now ?? new Date()).toISOString(); + const base = (env.LOOPOVER_ALERTMANAGER_URL ?? "").trim(); + if (base === "") return unknownServiceStatus(generatedAt, "alerting source not configured"); + + const doFetch = options.fetchImpl ?? fetch; + let response: Response; + try { + response = await doFetch(`${base.replace(/\/+$/, "")}/api/v2/alerts`, { signal: AbortSignal.timeout(ALERTMANAGER_TIMEOUT_MS) }); + } catch { + // Network error or timeout. Both mean the same thing publicly: we could not check. + return unknownServiceStatus(generatedAt, "alerting source unreachable"); + } + if (!response.ok) return unknownServiceStatus(generatedAt, "alerting source returned an error"); + + let alerts: unknown; + try { + alerts = await response.json(); + } catch { + return unknownServiceStatus(generatedAt, "alerting source returned an unreadable response"); + } + // A non-array body is a source we do not understand, which is not the same as no alerts — treating it as + // "nothing firing" would publish green off a payload we failed to parse. + if (!Array.isArray(alerts)) return unknownServiceStatus(generatedAt, "alerting source returned an unexpected shape"); + + return buildServiceStatus(alerts as AlertmanagerAlert[], generatedAt); +} diff --git a/test/unit/service-status.test.ts b/test/unit/service-status.test.ts new file mode 100644 index 0000000000..2cf3211e52 --- /dev/null +++ b/test/unit/service-status.test.ts @@ -0,0 +1,252 @@ +import { describe, expect, it } from "vitest"; + +import { createApp } from "../../src/api/routes"; +import { + buildServiceStatus, + isFiring, + isServiceStatusEnabled, + loadServiceStatus, + SERVICE_STATUS_COMPONENTS, + statusForSeverity, + unknownServiceStatus, + worseStatus, + type AlertmanagerAlert, +} from "../../src/selfhost/service-status"; +import { createTestEnv } from "../helpers/d1"; + +// #9983 (slice of #9747): the public status board. +// +// The correctness content here is almost entirely about NOT reporting green. A status page that renders +// "operational" because it could not reach its source is worse than no status page: it tells people the thing +// is fine at exactly the moment nobody can confirm it. So the failure paths are tested harder than the happy +// one, and the second concern -- that no host, instance or capacity detail escapes into a public payload -- +// is asserted over the serialized response rather than field by field. + +const NOW = new Date("2026-07-31T12:00:00.000Z"); +const alert = (over: Partial & { service?: string; severity?: string } = {}): AlertmanagerAlert => ({ + labels: { service: over.service ?? "loopover", severity: over.severity ?? "warning" }, + startsAt: "2026-07-31T11:00:00.000Z", + status: { state: "active" }, + ...over, +}); + +describe("worseStatus", () => { + it("REGRESSION: unknown outranks operational, so an unreadable component is never averaged away", () => { + // The single most important ordering in the module. If `operational` won, one healthy component would + // mask one we could not read. + expect(worseStatus("operational", "unknown")).toBe("unknown"); + expect(worseStatus("unknown", "operational")).toBe("unknown"); + }); + + it("ranks outage above degraded above unknown", () => { + expect(worseStatus("degraded", "outage")).toBe("outage"); + expect(worseStatus("unknown", "degraded")).toBe("degraded"); + expect(worseStatus("outage", "degraded")).toBe("outage"); + }); +}); + +describe("statusForSeverity", () => { + it("maps critical to outage and everything else to degraded", () => { + expect(statusForSeverity("critical")).toBe("outage"); + expect(statusForSeverity("CRITICAL")).toBe("outage"); + expect(statusForSeverity("warning")).toBe("degraded"); + }); + + it("REGRESSION: an unrecognised or missing severity rounds UP to degraded, never down to operational", () => { + // A firing alert with a severity we do not recognise is still firing. Rounding it down would hide it. + expect(statusForSeverity("catastrophe")).toBe("degraded"); + expect(statusForSeverity(undefined)).toBe("degraded"); + expect(statusForSeverity(42)).toBe("degraded"); + }); +}); + +describe("isFiring", () => { + it("excludes suppressed alerts, which are deliberately paging nobody", () => { + expect(isFiring({ status: { state: "suppressed" } })).toBe(false); + expect(isFiring({ status: { state: "active" } })).toBe(true); + }); + + it("treats a missing state as firing, since an alert we cannot classify is still an alert", () => { + expect(isFiring({})).toBe(true); + }); +}); + +describe("buildServiceStatus", () => { + it("reports every component as operational when nothing is firing", () => { + const payload = buildServiceStatus([], NOW.toISOString()); + expect(payload.overall).toBe("operational"); + expect(payload.components.map((c) => c.component)).toEqual([...SERVICE_STATUS_COMPONENTS]); + expect(payload.components.every((c) => c.status === "operational" && c.since === null)).toBe(true); + }); + + it("INVARIANT: a healthy component still appears -- omitting it is indistinguishable from not checking it", () => { + const payload = buildServiceStatus([alert({ service: "ams", severity: "critical" })], NOW.toISOString()); + expect(payload.components).toHaveLength(SERVICE_STATUS_COMPONENTS.length); + expect(payload.components.find((c) => c.component === "review")?.status).toBe("operational"); + }); + + it("maps an alert onto its component and raises overall to match", () => { + const payload = buildServiceStatus([alert({ service: "ams", severity: "critical" })], NOW.toISOString()); + const testing = payload.components.find((c) => c.component === "testing"); + expect(testing).toMatchObject({ status: "outage", since: "2026-07-31T11:00:00.000Z" }); + expect(payload.overall).toBe("outage"); + }); + + it("takes the WORST severity when a component has several alerts firing", () => { + const payload = buildServiceStatus([alert({ severity: "warning" }), alert({ severity: "critical" })], NOW.toISOString()); + expect(payload.components.find((c) => c.component === "review")?.status).toBe("outage"); + }); + + it("dates the incident from the EARLIEST alert, not the most recent", () => { + // The incident began when the first alert fired. Reporting the latest would keep resetting `since` as an + // ongoing outage generates more alerts, making a long incident look perpetually new. + const payload = buildServiceStatus( + [alert({ startsAt: "2026-07-31T11:30:00.000Z" }), alert({ startsAt: "2026-07-31T10:00:00.000Z" })], + NOW.toISOString(), + ); + expect(payload.components.find((c) => c.component === "review")?.since).toBe("2026-07-31T10:00:00.000Z"); + }); + + it("ignores suppressed alerts", () => { + const payload = buildServiceStatus([alert({ severity: "critical", status: { state: "suppressed" } })], NOW.toISOString()); + expect(payload.overall).toBe("operational"); + }); + + it("REGRESSION: an UNMAPPED service label is ignored, never published as a component", () => { + // Alert labels are internal topology. Publishing an unrecognised one verbatim is the leak the fixed + // component vocabulary exists to prevent. + const payload = buildServiceStatus([alert({ service: "postgres-primary-node-3", severity: "critical" })], NOW.toISOString()); + expect(payload.overall).toBe("operational"); + expect(payload.components.map((c) => c.component)).toEqual([...SERVICE_STATUS_COMPONENTS]); + }); + + it("clears `since` for a component that is operational", () => { + expect(buildServiceStatus([], NOW.toISOString()).components.every((c) => c.since === null)).toBe(true); + }); +}); + +describe("loadServiceStatus — every failure path lands on unknown, never operational", () => { + const cases: { name: string; env: { LOOPOVER_ALERTMANAGER_URL?: string }; fetchImpl: typeof fetch; reason: RegExp }[] = [ + { + name: "the alerting source is not configured", + env: {}, + fetchImpl: (() => Promise.reject(new Error("should not be called"))) as unknown as typeof fetch, + reason: /not configured/, + }, + { + name: "the request throws (network error or timeout)", + env: { LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" }, + fetchImpl: (() => Promise.reject(new Error("ECONNREFUSED"))) as unknown as typeof fetch, + reason: /unreachable/, + }, + { + name: "the source answers non-200", + env: { LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" }, + fetchImpl: (() => Promise.resolve(new Response("nope", { status: 503 }))) as unknown as typeof fetch, + reason: /returned an error/, + }, + { + name: "the body is not JSON", + env: { LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" }, + fetchImpl: (() => Promise.resolve(new Response("", { status: 200 }))) as unknown as typeof fetch, + reason: /unreadable/, + }, + { + name: "the body parses but is not an array", + // The subtlest one: `{}` is valid JSON with no alerts in it, and treating that as "nothing firing" + // would publish green off a payload we failed to understand. + env: { LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" }, + fetchImpl: (() => Promise.resolve(Response.json({ status: "success" }))) as unknown as typeof fetch, + reason: /unexpected shape/, + }, + ]; + + for (const testCase of cases) { + it(`reports unknown when ${testCase.name}`, async () => { + const payload = await loadServiceStatus(testCase.env, { now: NOW, fetchImpl: testCase.fetchImpl }); + expect(payload.overall).toBe("unknown"); + expect(payload.components.every((c) => c.status === "unknown")).toBe(true); + expect(payload.components[0]?.reason).toMatch(testCase.reason); + }); + } + + it("INVARIANT: no failure reason leaks the configured URL", async () => { + // The reason is a category, not a connection string -- the Alertmanager address is internal topology. + const payload = await loadServiceStatus( + { LOOPOVER_ALERTMANAGER_URL: "http://alertmanager.internal.example:9093" }, + { now: NOW, fetchImpl: (() => Promise.reject(new Error("ECONNREFUSED"))) as unknown as typeof fetch }, + ); + expect(JSON.stringify(payload)).not.toMatch(/alertmanager\.internal|9093/); + }); + + it("reads the real shape Alertmanager returns and reports operational for an empty list", async () => { + // `[]` is exactly what the live Orb returns today. + const payload = await loadServiceStatus( + { LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" }, + { now: NOW, fetchImpl: (() => Promise.resolve(Response.json([]))) as unknown as typeof fetch }, + ); + expect(payload.overall).toBe("operational"); + }); + + it("requests the v2 alerts endpoint under the configured base, tolerating a trailing slash", async () => { + const seen: string[] = []; + const capture = ((url: string) => { + seen.push(String(url)); + return Promise.resolve(Response.json([])); + }) as unknown as typeof fetch; + await loadServiceStatus({ LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093/" }, { now: NOW, fetchImpl: capture }); + expect(seen[0]).toBe("http://alertmanager:9093/api/v2/alerts"); + }); +}); + +describe("isServiceStatusEnabled", () => { + it("is off when unset or blank, so the hosted Worker never serves an all-unknown board", () => { + expect(isServiceStatusEnabled({})).toBe(false); + expect(isServiceStatusEnabled({ LOOPOVER_ALERTMANAGER_URL: " " })).toBe(false); + }); + + it("is on when a source is configured", () => { + expect(isServiceStatusEnabled({ LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" })).toBe(true); + }); +}); + +describe("unknownServiceStatus", () => { + it("covers every component, so a degraded read never silently drops one", () => { + const payload = unknownServiceStatus(NOW.toISOString(), "because"); + expect(payload.components.map((c) => c.component)).toEqual([...SERVICE_STATUS_COMPONENTS]); + }); +}); + +describe("GET /v1/public/service-status", () => { + const get = (env: Env) => createApp().request("/v1/public/service-status", {}, env); + + it("404s where no alerting source is configured", async () => { + const response = await get(createTestEnv()); + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ error: "not_found" }); + }); + + it("serves the board where one is, without requiring any credential", async () => { + const env = createTestEnv({ LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" } as Partial); + const response = await get(env); + expect(response.status).toBe(200); + const body = (await response.json()) as { overall: string; components: unknown[] }; + // Alertmanager is unreachable from the test process, so this exercises the honest-degradation path end + // to end: a 200 that says "unknown", not a 500 and not a green board. + expect(body.overall).toBe("unknown"); + expect(body.components).toHaveLength(SERVICE_STATUS_COMPONENTS.length); + }); + + it("INVARIANT: the payload carries no host, instance, capacity or alert-label detail", async () => { + const env = createTestEnv({ LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" } as Partial); + const serialized = await (await get(env)).text(); + for (const forbidden of ["alertmanager", "9093", "instance", "job", "pod", "localhost", "cpu", "memory", "disk"]) { + expect(serialized.toLowerCase(), forbidden).not.toContain(forbidden); + } + }); + + it("caches briefly -- this is the endpoint people refresh during an incident", async () => { + const env = createTestEnv({ LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" } as Partial); + expect((await get(env)).headers.get("Cache-Control")).toBe("public, max-age=15, stale-while-revalidate=30"); + }); +}); From 581cfab1489c3987f9809448552903b4d98d1b38 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:59:05 -0700 Subject: [PATCH 2/2] test(status): cover the remaining branches in the alert fold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit codecov/patch flagged 94% against the 99% gate. Branch coverage on service-status.ts was 92.3%, with three arms of the per-component fold unexercised -- all of them in the paths that decide what a malformed or partial alert does: • a `service` label that is not a string. Alertmanager's contract says Record, but this reads a parsed JSON body from a source that can ship anything, and a non-string must take the same ignore path an unmapped label takes rather than throwing on .toLowerCase(); • a firing alert with no usable `startsAt`. Still degraded -- an alert with no timestamp is still an alert -- but no invented `since`; • the KEEP arms of the earliest-wins comparison. The existing test put the earlier alert second, which only exercises the replace arm; the mirror case (earlier first, and a later untimestamped one) leaves half the comparison free to invert unnoticed. Branch coverage is now 100% (39/39), statements and lines already were. No production code changed. --- test/unit/service-status.test.ts | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test/unit/service-status.test.ts b/test/unit/service-status.test.ts index 2cf3211e52..d27b2cbd8e 100644 --- a/test/unit/service-status.test.ts +++ b/test/unit/service-status.test.ts @@ -120,6 +120,37 @@ describe("buildServiceStatus", () => { expect(payload.components.map((c) => c.component)).toEqual([...SERVICE_STATUS_COMPONENTS]); }); + it("ignores an alert whose service label is not a string at all", () => { + // Alertmanager labels are `Record` by contract, but this reads a parsed JSON body from a + // source that could ship anything. A non-string label must take the same ignore path an unmapped one + // takes, not throw on `.toLowerCase()`. + const payload = buildServiceStatus([{ labels: { service: 42 }, startsAt: "x", status: { state: "active" } }, { labels: {} }], NOW.toISOString()); + expect(payload.overall).toBe("operational"); + }); + + it("records no `since` when the firing alert carries no usable start time", () => { + // `startsAt` absent or non-string: the component is still degraded — an alert with no timestamp is still + // an alert — but inventing a start would imply a transition that was never reported. + const payload = buildServiceStatus([alert({ startsAt: undefined })], NOW.toISOString()); + const review = payload.components.find((c) => c.component === "review"); + expect(review?.status).toBe("degraded"); + expect(review?.since).toBeNull(); + }); + + it("keeps an established `since` when a later alert for the same component has no start time", () => { + // The timestamped alert established the incident start; a subsequent untimestamped one must not erase it. + const payload = buildServiceStatus([alert({ startsAt: "2026-07-31T09:00:00.000Z" }), alert({ startsAt: null })], NOW.toISOString()); + expect(payload.components.find((c) => c.component === "review")?.since).toBe("2026-07-31T09:00:00.000Z"); + }); + + it("keeps the earlier `since` when the alerts arrive in chronological order too", () => { + // The mirror of the earliest-wins test above. That one has the earlier alert SECOND, so it only exercises + // the replace arm; this one has it first and exercises the keep arm. Without both, half the comparison is + // free to invert unnoticed. + const payload = buildServiceStatus([alert({ startsAt: "2026-07-31T09:00:00.000Z" }), alert({ startsAt: "2026-07-31T11:30:00.000Z" })], NOW.toISOString()); + expect(payload.components.find((c) => c.component === "review")?.since).toBe("2026-07-31T09:00:00.000Z"); + }); + it("clears `since` for a component that is operational", () => { expect(buildServiceStatus([], NOW.toISOString()).components.every((c) => c.since === null)).toBe(true); });