diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json index a451e5600d..3a40a3a5b4 100644 --- a/apps/loopover-ui/public/openapi.json +++ b/apps/loopover-ui/public/openapi.json @@ -27612,6 +27612,23 @@ } } } + }, + "/v1/public/service-status": { + "get": { + "operationId": "getPublicServiceStatus", + "tags": [ + "Public" + ], + "summary": "Per-component service status — is each component healthy right now", + "responses": { + "200": { + "description": "{ generatedAt, overall, components: [{ component, label, status, since, reason? }] } — `status` is `operational` | `degraded` | `outage` | `unknown`, and `overall` is the worst of them. `unknown` means the alerting source could not be read and is NEVER reported as healthy; `reason` is then a category, never a URL or hostname. `since` is the earliest firing alert's start for a non-operational component, else null. Public-safe: no host names, instance ids, capacity figures or raw alert labels" + }, + "404": { + "description": "This deployment has no alerting source configured, so it publishes no status board" + } + } + } } }, "servers": [ diff --git a/apps/loopover-ui/src/lib/selfhost-env-reference.ts b/apps/loopover-ui/src/lib/selfhost-env-reference.ts index 45d9ef0c67..a85f36d758 100644 --- a/apps/loopover-ui/src/lib/selfhost-env-reference.ts +++ b/apps/loopover-ui/src/lib/selfhost-env-reference.ts @@ -265,6 +265,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "INTERNAL_JOB_TOKEN", firstReference: "src/selfhost/preflight.ts", }, + { + name: "LOOPOVER_ALERTMANAGER_URL", + firstReference: "src/selfhost/service-status.ts", + }, { name: "LOOPOVER_API_TOKEN", firstReference: "src/selfhost/preflight.ts", @@ -783,6 +787,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `GITHUB_WEBHOOK_SECRET` | `src/selfhost/preflight.ts` |", "| `HOME` | `src/selfhost/ai.ts` |", "| `INTERNAL_JOB_TOKEN` | `src/selfhost/preflight.ts` |", + "| `LOOPOVER_ALERTMANAGER_URL` | `src/selfhost/service-status.ts` |", "| `LOOPOVER_API_TOKEN` | `src/selfhost/preflight.ts` |", "| `LOOPOVER_CENTRAL_POSTHOG_KEY` | `src/selfhost/posthog.ts` |", "| `LOOPOVER_ENABLE_PAGERDUTY` | `src/services/notify-pagerduty.ts` |", diff --git a/src/api/routes.ts b/src/api/routes.ts index d46a7deb00..1b0ab81da5 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -308,6 +308,7 @@ import { isRagEnabled } from "../review/rag-wire"; import { loadDecisionLedgerTip, loadPublicDecisionRecord, loadPublicLedgerRow, verifyDecisionLedger } from "../review/decision-record"; import { buildEvalScoreRecordsFromRulePrecision, filterEvalScoreRecords } from "../review/eval-score-records"; import { buildPublicCorpusCommitments } from "../review/public-eval-corpus"; +import { isServiceStatusEnabled, loadServiceStatus } from "../selfhost/service-status"; import { anchorSigningInput, buildLedgerAnchorPayload, currentAnchorKey, diagnoseAnchorPublicKeys, parseAnchorPublicKeys, publicAnchorStatus, signLedgerAnchorPayload } from "../review/ledger-anchor"; import { resolveProofPage } from "../review/proof-summary"; import { renderProofBadgeSvg } from "./proof-badge"; @@ -919,6 +920,21 @@ export function createApp() { return c.json(corpus); }); + // #9983 (slice of #9747): the public status board, sourced from THIS deployment's own alerting stack -- + // the same Grafana-managed rules that page the on-call rotation through Alertmanager. Reusing that source + // rather than adding a status-page-only probe means the page cannot disagree with what actually pages a + // human. 404s where no alerting source is configured (the hosted Worker) instead of publishing a board that + // reads "unknown" forever. Reachable publicly on the Orb through the existing Cloudflare Tunnel, which + // already routes /v1/public/* -- no tunnel change was needed to ship this. + app.get("/v1/public/service-status", async (c) => { + if (!isServiceStatusEnabled(c.env)) return c.json({ error: "not_found" }, 404); + const status = await loadServiceStatus(c.env); + // Shorter than the sibling public surfaces on purpose: this is the endpoint people refresh DURING an + // incident, and a 60s cache would keep serving "operational" for a minute after an outage started. + c.header("Cache-Control", "public, max-age=15, stale-while-revalidate=30"); + return c.json(status); + }); + app.get("/v1/public/eval-scores", async (c) => { const publicStatsManifestOverride = await resolvePublicStatsManifestOverride(c.env); if (!isPublicStatsEnabled(c.env, publicStatsManifestOverride)) return c.json({ error: "not_found" }, 404); diff --git a/src/auth/route-auth.ts b/src/auth/route-auth.ts index f54b2cb584..3e407bd86b 100644 --- a/src/auth/route-auth.ts +++ b/src/auth/route-auth.ts @@ -44,6 +44,9 @@ export function requiresApiToken(path: string): boolean { // repos, no metadata beyond the one confidence field replay needs), so there is nothing here an // Authorization header would be protecting. if (path === "/v1/public/eval-corpus") return false; + // #9983: component name + status only, no host, instance or capacity detail -- added in the SAME PR as its + // route so the two cannot drift. + if (path === "/v1/public/service-status") return false; // #9269: the single-row read, added in the SAME PR as its route so the two can never drift the way #9120's // sibling did. Regex (not a literal) because of the :seq path parameter. if (/^\/v1\/public\/decision-ledger\/row\/[^/]+$/.test(path)) return false; diff --git a/src/env.d.ts b/src/env.d.ts index 5df8680ab8..a5bee78c28 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -614,6 +614,12 @@ declare global { * when this is off -- see isProofPageEnabledForRepo's recorded decision in review/proof-summary.ts. */ LOOPOVER_PUBLIC_PROOF?: string; LOOPOVER_PUBLIC_STATS?: string; + /** #9983: base URL of this deployment's Alertmanager (the source behind `/v1/public/service-status`). + * Unset means the deployment has no alerting stack -- the hosted Worker -- and the route 404s rather + * than publishing a board that reads "unknown" for every component forever. On the Orb this is the + * compose-network address, never a public one: the app reads it server-side and publishes only a + * component name and status, so the URL itself is internal topology and never leaves the process. */ + LOOPOVER_ALERTMANAGER_URL?: string; /** Proof of Power (#1059): comma-separated allowlist of repo full-names ("owner/repo") whose OWN historical * review ledger (audit_events "published a review surface" + pull_requests terminal state) counts toward * the public stats counter. DELIBERATELY SEPARATE from LOOPOVER_REVIEW_REPOS (the live per-PR-feature diff --git a/src/openapi/spec.ts b/src/openapi/spec.ts index 1be794cc7e..c0e67e7eff 100644 --- a/src/openapi/spec.ts +++ b/src/openapi/spec.ts @@ -1970,6 +1970,24 @@ export function buildOpenApiSpec() { 404: { description: "No decision record persisted yet for this PR" }, }, }); + // #9983 (slice of #9747): the public status board, sourced from this deployment's own Alertmanager -- the + // same alerting stack that pages the on-call rotation -- so the page cannot disagree with what actually + // fires. 404s where no alerting source is configured rather than publishing an all-unknown board. + registry.registerPath({ + method: "get", + path: "/v1/public/service-status", + operationId: "getPublicServiceStatus", + tags: ["Public"], + summary: "Per-component service status — is each component healthy right now", + responses: { + 200: { + description: + "{ generatedAt, overall, components: [{ component, label, status, since, reason? }] } — `status` is `operational` | `degraded` | `outage` | `unknown`, and `overall` is the worst of them. `unknown` means the alerting source could not be read and is NEVER reported as healthy; `reason` is then a category, never a URL or hostname. `since` is the earliest firing alert's start for a non-operational component, else null. Public-safe: no host names, instance ids, capacity figures or raw alert labels", + }, + 404: { description: "This deployment has no alerting source configured, so it publishes no status board" }, + }, + }); + registry.registerPath({ method: "get", path: "/v1/public/eval-corpus", diff --git a/src/selfhost/service-status.ts b/src/selfhost/service-status.ts new file mode 100644 index 0000000000..d34cb4cf07 --- /dev/null +++ b/src/selfhost/service-status.ts @@ -0,0 +1,212 @@ +// Public service status, sourced from the deployment's OWN alerting stack (#9983, slice of #9747). +// +// WHY ALERTMANAGER RATHER THAN A NEW PROBE. #9747 says to reuse the existing health/alerting +// instrumentation, and on the Orb that is Grafana-managed rules -> Alertmanager -> PagerDuty. A second, +// status-page-only probe would be a second opinion about the same components, free to disagree with the one +// that actually pages a human — and when they disagree, the public page is the one nobody is watching. Reading +// the alerting stack means the page says exactly what the on-call rotation already believes. +// +// NO NEW INFRASTRUCTURE. The app container already reaches Alertmanager on the compose network (verified: +// `fetch("http://alertmanager:9093/api/v2/alerts")` returns `[]` from inside the running container), and the +// existing Cloudflare Tunnel already routes `^/v1/public/.*$` to this app, so the route is publicly reachable +// the moment it ships — no tunnel edit and no new hostname, which matters because there is no `cert.pem` on +// the box to mint one with. +// +// UNREACHABLE IS `unknown`, NEVER `operational`. This is the whole correctness content of the module. A status +// page that renders green because it could not reach its source is worse than no status page: it actively +// tells people the thing is fine at exactly the moment nobody can confirm it. Same false-green class as a +// verifier that skips every claim and exits zero. So every failure path — unset URL, network error, timeout, +// non-200, unparseable body — lands on `unknown` with a stated reason, and `operational` is only ever reached +// by successfully reading the alert list and finding nothing firing for that component. +// +// PUBLIC-SAFE BY CONSTRUCTION. The response carries component name, status, and a since-timestamp. It never +// carries a hostname, an instance id, a capacity figure, an alert annotation, or a label bag — alert payloads +// routinely contain internal host and job labels, so this maps them to a fixed component vocabulary and drops +// the original rather than filtering it. Whitelisting the shape is what keeps a new alert label from leaking +// through a filter nobody updated. + +/** The components this deployment reports on. A FIXED vocabulary, not whatever labels the alerts happen to + * carry: the mapping is what turns internal topology into a public name, and an unmapped alert must widen + * this list deliberately rather than publish a label verbatim. */ +export const SERVICE_STATUS_COMPONENTS = ["review", "testing", "discovery"] as const; +export type ServiceComponent = (typeof SERVICE_STATUS_COMPONENTS)[number]; + +/** Human-facing names, so the public surface never shows an internal identifier. */ +const SERVICE_COMPONENT_LABELS: Record = { + review: "ORB review service", + testing: "AMS testing service", + discovery: "Discovery index", +}; + +/** + * Which alert `service` label belongs to which public component. + * + * Read from the alert's `service` label only. Alertmanager alerts also carry `job`, `instance`, `pod` and + * similar, all of which name infrastructure; binding to one deliberate label keeps the public mapping + * explicit and keeps host identifiers out of the decision entirely. + */ +const SERVICE_LABEL_TO_COMPONENT: Record = { + loopover: "review", + orb: "review", + review: "review", + ams: "testing", + testing: "testing", + "discovery-index": "discovery", + discovery: "discovery", +}; + +/** `operational` — read the source, nothing firing. `degraded` — a warning-severity alert is firing. + * `outage` — a critical one is. `unknown` — the source could not be read, which is never green. */ +export type ComponentStatus = "operational" | "degraded" | "outage" | "unknown"; + +export type ServiceComponentState = { + component: ServiceComponent; + label: string; + status: ComponentStatus; + /** When the current state began: the earliest firing alert's start for a non-operational component, else + * null. Null rather than "now" — inventing a timestamp would imply a transition that did not happen. */ + since: string | null; + /** Present only when status is `unknown`, saying why the source could not be read. Never carries a URL or + * hostname: the reason is a category, not a connection string. */ + reason?: string; +}; + +export type ServiceStatusPayload = { + generatedAt: string; + /** The worst component status, so a caller can answer "is anything wrong" without walking the list. */ + overall: ComponentStatus; + components: ServiceComponentState[]; +}; + +/** One Alertmanager alert, narrowed to the two fields this reads. Structural, so a newer Alertmanager that + * adds fields still parses. */ +export type AlertmanagerAlert = { + labels?: Record | undefined; + startsAt?: unknown; + status?: { state?: unknown } | undefined; +}; + +/** PURE. Severity ranking, so `overall` and per-component rollups agree on which state is worse. */ +const SEVERITY_RANK: Record = { operational: 0, unknown: 1, degraded: 2, outage: 3 }; + +/** PURE. The worse of two statuses. `unknown` outranks `operational` — a component we could not read must not + * be averaged away by one we could. */ +export function worseStatus(a: ComponentStatus, b: ComponentStatus): ComponentStatus { + return SEVERITY_RANK[b] > SEVERITY_RANK[a] ? b : a; +} + +/** PURE. An alert's severity mapped to a status. Anything that is not explicitly `critical` is treated as + * `degraded`: an unrecognised severity is still a firing alert, and rounding it DOWN to operational would + * hide it. */ +export function statusForSeverity(severity: unknown): ComponentStatus { + return typeof severity === "string" && severity.toLowerCase() === "critical" ? "outage" : "degraded"; +} + +/** PURE. Is this alert actually firing? Alertmanager returns suppressed alerts too when asked; a silenced or + * inhibited alert is deliberately not paging anyone, so it must not colour the public board either. */ +export function isFiring(alert: AlertmanagerAlert): boolean { + const state = alert.status?.state; + return typeof state !== "string" || state === "active"; +} + +/** + * PURE. Fold firing alerts into a per-component board. + * + * Every component in the vocabulary appears, whether or not it has alerts — a status page that omits a healthy + * component is indistinguishable from one that forgot to check it. Alerts whose `service` label is unmapped + * are IGNORED rather than published under a made-up component: publishing an unrecognised internal label is + * exactly the leak this module's whitelist exists to prevent. + */ +export function buildServiceStatus(alerts: readonly AlertmanagerAlert[], generatedAt: string): ServiceStatusPayload { + const states = new Map(); + for (const component of SERVICE_STATUS_COMPONENTS) states.set(component, { status: "operational", since: null }); + + for (const alert of alerts) { + if (!isFiring(alert)) continue; + const service = alert.labels?.["service"]; + const component = typeof service === "string" ? SERVICE_LABEL_TO_COMPONENT[service.toLowerCase()] : undefined; + if (component === undefined) continue; + const current = states.get(component)!; + const status = worseStatus(current.status, statusForSeverity(alert.labels?.["severity"])); + // Earliest start wins: the incident began when the FIRST alert for this component fired, not when the + // most recent one did. + const startsAt = typeof alert.startsAt === "string" ? alert.startsAt : null; + const since = current.since === null ? startsAt : startsAt === null ? current.since : startsAt < current.since ? startsAt : current.since; + states.set(component, { status, since }); + } + + const components = SERVICE_STATUS_COMPONENTS.map((component) => { + const state = states.get(component)!; + return { + component, + label: SERVICE_COMPONENT_LABELS[component], + status: state.status, + // `since` is only meaningful for a component that is currently in a non-operational state. + since: state.status === "operational" ? null : state.since, + }; + }); + + return { generatedAt, overall: components.reduce((worst, entry) => worseStatus(worst, entry.status), "operational"), components }; +} + +/** PURE. Every component `unknown`, for the paths where the source could not be read at all. */ +export function unknownServiceStatus(generatedAt: string, reason: string): ServiceStatusPayload { + return { + generatedAt, + overall: "unknown", + components: SERVICE_STATUS_COMPONENTS.map((component) => ({ + component, + label: SERVICE_COMPONENT_LABELS[component], + status: "unknown" as const, + since: null, + reason, + })), + }; +} + +/** Is a status surface configured on this deployment? The hosted Worker has no Alertmanager, and a board that + * reads "unknown" for everything forever is not a status page — the route 404s there instead. */ +export function isServiceStatusEnabled(env: { LOOPOVER_ALERTMANAGER_URL?: string | undefined }): boolean { + return (env.LOOPOVER_ALERTMANAGER_URL ?? "").trim() !== ""; +} + +/** How long to wait on the alerting source. A status endpoint that hangs is itself an outage, and this route + * is unauthenticated and cacheable, so it must always answer. */ +const ALERTMANAGER_TIMEOUT_MS = 4_000; + +/** + * Read the deployment's alerting source and build the public board. + * + * NEVER THROWS, and never degrades to `operational`. Every failure lands on {@link unknownServiceStatus} with + * a category reason — the reason deliberately does not include the configured URL, which is internal topology. + */ +export async function loadServiceStatus( + env: { LOOPOVER_ALERTMANAGER_URL?: string | undefined }, + options: { now?: Date | undefined; fetchImpl?: typeof fetch | undefined } = {}, +): Promise { + const generatedAt = (options.now ?? new Date()).toISOString(); + const base = (env.LOOPOVER_ALERTMANAGER_URL ?? "").trim(); + if (base === "") return unknownServiceStatus(generatedAt, "alerting source not configured"); + + const doFetch = options.fetchImpl ?? fetch; + let response: Response; + try { + response = await doFetch(`${base.replace(/\/+$/, "")}/api/v2/alerts`, { signal: AbortSignal.timeout(ALERTMANAGER_TIMEOUT_MS) }); + } catch { + // Network error or timeout. Both mean the same thing publicly: we could not check. + return unknownServiceStatus(generatedAt, "alerting source unreachable"); + } + if (!response.ok) return unknownServiceStatus(generatedAt, "alerting source returned an error"); + + let alerts: unknown; + try { + alerts = await response.json(); + } catch { + return unknownServiceStatus(generatedAt, "alerting source returned an unreadable response"); + } + // A non-array body is a source we do not understand, which is not the same as no alerts — treating it as + // "nothing firing" would publish green off a payload we failed to parse. + if (!Array.isArray(alerts)) return unknownServiceStatus(generatedAt, "alerting source returned an unexpected shape"); + + return buildServiceStatus(alerts as AlertmanagerAlert[], generatedAt); +} diff --git a/test/unit/service-status.test.ts b/test/unit/service-status.test.ts new file mode 100644 index 0000000000..d27b2cbd8e --- /dev/null +++ b/test/unit/service-status.test.ts @@ -0,0 +1,283 @@ +import { describe, expect, it } from "vitest"; + +import { createApp } from "../../src/api/routes"; +import { + buildServiceStatus, + isFiring, + isServiceStatusEnabled, + loadServiceStatus, + SERVICE_STATUS_COMPONENTS, + statusForSeverity, + unknownServiceStatus, + worseStatus, + type AlertmanagerAlert, +} from "../../src/selfhost/service-status"; +import { createTestEnv } from "../helpers/d1"; + +// #9983 (slice of #9747): the public status board. +// +// The correctness content here is almost entirely about NOT reporting green. A status page that renders +// "operational" because it could not reach its source is worse than no status page: it tells people the thing +// is fine at exactly the moment nobody can confirm it. So the failure paths are tested harder than the happy +// one, and the second concern -- that no host, instance or capacity detail escapes into a public payload -- +// is asserted over the serialized response rather than field by field. + +const NOW = new Date("2026-07-31T12:00:00.000Z"); +const alert = (over: Partial & { service?: string; severity?: string } = {}): AlertmanagerAlert => ({ + labels: { service: over.service ?? "loopover", severity: over.severity ?? "warning" }, + startsAt: "2026-07-31T11:00:00.000Z", + status: { state: "active" }, + ...over, +}); + +describe("worseStatus", () => { + it("REGRESSION: unknown outranks operational, so an unreadable component is never averaged away", () => { + // The single most important ordering in the module. If `operational` won, one healthy component would + // mask one we could not read. + expect(worseStatus("operational", "unknown")).toBe("unknown"); + expect(worseStatus("unknown", "operational")).toBe("unknown"); + }); + + it("ranks outage above degraded above unknown", () => { + expect(worseStatus("degraded", "outage")).toBe("outage"); + expect(worseStatus("unknown", "degraded")).toBe("degraded"); + expect(worseStatus("outage", "degraded")).toBe("outage"); + }); +}); + +describe("statusForSeverity", () => { + it("maps critical to outage and everything else to degraded", () => { + expect(statusForSeverity("critical")).toBe("outage"); + expect(statusForSeverity("CRITICAL")).toBe("outage"); + expect(statusForSeverity("warning")).toBe("degraded"); + }); + + it("REGRESSION: an unrecognised or missing severity rounds UP to degraded, never down to operational", () => { + // A firing alert with a severity we do not recognise is still firing. Rounding it down would hide it. + expect(statusForSeverity("catastrophe")).toBe("degraded"); + expect(statusForSeverity(undefined)).toBe("degraded"); + expect(statusForSeverity(42)).toBe("degraded"); + }); +}); + +describe("isFiring", () => { + it("excludes suppressed alerts, which are deliberately paging nobody", () => { + expect(isFiring({ status: { state: "suppressed" } })).toBe(false); + expect(isFiring({ status: { state: "active" } })).toBe(true); + }); + + it("treats a missing state as firing, since an alert we cannot classify is still an alert", () => { + expect(isFiring({})).toBe(true); + }); +}); + +describe("buildServiceStatus", () => { + it("reports every component as operational when nothing is firing", () => { + const payload = buildServiceStatus([], NOW.toISOString()); + expect(payload.overall).toBe("operational"); + expect(payload.components.map((c) => c.component)).toEqual([...SERVICE_STATUS_COMPONENTS]); + expect(payload.components.every((c) => c.status === "operational" && c.since === null)).toBe(true); + }); + + it("INVARIANT: a healthy component still appears -- omitting it is indistinguishable from not checking it", () => { + const payload = buildServiceStatus([alert({ service: "ams", severity: "critical" })], NOW.toISOString()); + expect(payload.components).toHaveLength(SERVICE_STATUS_COMPONENTS.length); + expect(payload.components.find((c) => c.component === "review")?.status).toBe("operational"); + }); + + it("maps an alert onto its component and raises overall to match", () => { + const payload = buildServiceStatus([alert({ service: "ams", severity: "critical" })], NOW.toISOString()); + const testing = payload.components.find((c) => c.component === "testing"); + expect(testing).toMatchObject({ status: "outage", since: "2026-07-31T11:00:00.000Z" }); + expect(payload.overall).toBe("outage"); + }); + + it("takes the WORST severity when a component has several alerts firing", () => { + const payload = buildServiceStatus([alert({ severity: "warning" }), alert({ severity: "critical" })], NOW.toISOString()); + expect(payload.components.find((c) => c.component === "review")?.status).toBe("outage"); + }); + + it("dates the incident from the EARLIEST alert, not the most recent", () => { + // The incident began when the first alert fired. Reporting the latest would keep resetting `since` as an + // ongoing outage generates more alerts, making a long incident look perpetually new. + const payload = buildServiceStatus( + [alert({ startsAt: "2026-07-31T11:30:00.000Z" }), alert({ startsAt: "2026-07-31T10:00:00.000Z" })], + NOW.toISOString(), + ); + expect(payload.components.find((c) => c.component === "review")?.since).toBe("2026-07-31T10:00:00.000Z"); + }); + + it("ignores suppressed alerts", () => { + const payload = buildServiceStatus([alert({ severity: "critical", status: { state: "suppressed" } })], NOW.toISOString()); + expect(payload.overall).toBe("operational"); + }); + + it("REGRESSION: an UNMAPPED service label is ignored, never published as a component", () => { + // Alert labels are internal topology. Publishing an unrecognised one verbatim is the leak the fixed + // component vocabulary exists to prevent. + const payload = buildServiceStatus([alert({ service: "postgres-primary-node-3", severity: "critical" })], NOW.toISOString()); + expect(payload.overall).toBe("operational"); + expect(payload.components.map((c) => c.component)).toEqual([...SERVICE_STATUS_COMPONENTS]); + }); + + it("ignores an alert whose service label is not a string at all", () => { + // Alertmanager labels are `Record` by contract, but this reads a parsed JSON body from a + // source that could ship anything. A non-string label must take the same ignore path an unmapped one + // takes, not throw on `.toLowerCase()`. + const payload = buildServiceStatus([{ labels: { service: 42 }, startsAt: "x", status: { state: "active" } }, { labels: {} }], NOW.toISOString()); + expect(payload.overall).toBe("operational"); + }); + + it("records no `since` when the firing alert carries no usable start time", () => { + // `startsAt` absent or non-string: the component is still degraded — an alert with no timestamp is still + // an alert — but inventing a start would imply a transition that was never reported. + const payload = buildServiceStatus([alert({ startsAt: undefined })], NOW.toISOString()); + const review = payload.components.find((c) => c.component === "review"); + expect(review?.status).toBe("degraded"); + expect(review?.since).toBeNull(); + }); + + it("keeps an established `since` when a later alert for the same component has no start time", () => { + // The timestamped alert established the incident start; a subsequent untimestamped one must not erase it. + const payload = buildServiceStatus([alert({ startsAt: "2026-07-31T09:00:00.000Z" }), alert({ startsAt: null })], NOW.toISOString()); + expect(payload.components.find((c) => c.component === "review")?.since).toBe("2026-07-31T09:00:00.000Z"); + }); + + it("keeps the earlier `since` when the alerts arrive in chronological order too", () => { + // The mirror of the earliest-wins test above. That one has the earlier alert SECOND, so it only exercises + // the replace arm; this one has it first and exercises the keep arm. Without both, half the comparison is + // free to invert unnoticed. + const payload = buildServiceStatus([alert({ startsAt: "2026-07-31T09:00:00.000Z" }), alert({ startsAt: "2026-07-31T11:30:00.000Z" })], NOW.toISOString()); + expect(payload.components.find((c) => c.component === "review")?.since).toBe("2026-07-31T09:00:00.000Z"); + }); + + it("clears `since` for a component that is operational", () => { + expect(buildServiceStatus([], NOW.toISOString()).components.every((c) => c.since === null)).toBe(true); + }); +}); + +describe("loadServiceStatus — every failure path lands on unknown, never operational", () => { + const cases: { name: string; env: { LOOPOVER_ALERTMANAGER_URL?: string }; fetchImpl: typeof fetch; reason: RegExp }[] = [ + { + name: "the alerting source is not configured", + env: {}, + fetchImpl: (() => Promise.reject(new Error("should not be called"))) as unknown as typeof fetch, + reason: /not configured/, + }, + { + name: "the request throws (network error or timeout)", + env: { LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" }, + fetchImpl: (() => Promise.reject(new Error("ECONNREFUSED"))) as unknown as typeof fetch, + reason: /unreachable/, + }, + { + name: "the source answers non-200", + env: { LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" }, + fetchImpl: (() => Promise.resolve(new Response("nope", { status: 503 }))) as unknown as typeof fetch, + reason: /returned an error/, + }, + { + name: "the body is not JSON", + env: { LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" }, + fetchImpl: (() => Promise.resolve(new Response("", { status: 200 }))) as unknown as typeof fetch, + reason: /unreadable/, + }, + { + name: "the body parses but is not an array", + // The subtlest one: `{}` is valid JSON with no alerts in it, and treating that as "nothing firing" + // would publish green off a payload we failed to understand. + env: { LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" }, + fetchImpl: (() => Promise.resolve(Response.json({ status: "success" }))) as unknown as typeof fetch, + reason: /unexpected shape/, + }, + ]; + + for (const testCase of cases) { + it(`reports unknown when ${testCase.name}`, async () => { + const payload = await loadServiceStatus(testCase.env, { now: NOW, fetchImpl: testCase.fetchImpl }); + expect(payload.overall).toBe("unknown"); + expect(payload.components.every((c) => c.status === "unknown")).toBe(true); + expect(payload.components[0]?.reason).toMatch(testCase.reason); + }); + } + + it("INVARIANT: no failure reason leaks the configured URL", async () => { + // The reason is a category, not a connection string -- the Alertmanager address is internal topology. + const payload = await loadServiceStatus( + { LOOPOVER_ALERTMANAGER_URL: "http://alertmanager.internal.example:9093" }, + { now: NOW, fetchImpl: (() => Promise.reject(new Error("ECONNREFUSED"))) as unknown as typeof fetch }, + ); + expect(JSON.stringify(payload)).not.toMatch(/alertmanager\.internal|9093/); + }); + + it("reads the real shape Alertmanager returns and reports operational for an empty list", async () => { + // `[]` is exactly what the live Orb returns today. + const payload = await loadServiceStatus( + { LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" }, + { now: NOW, fetchImpl: (() => Promise.resolve(Response.json([]))) as unknown as typeof fetch }, + ); + expect(payload.overall).toBe("operational"); + }); + + it("requests the v2 alerts endpoint under the configured base, tolerating a trailing slash", async () => { + const seen: string[] = []; + const capture = ((url: string) => { + seen.push(String(url)); + return Promise.resolve(Response.json([])); + }) as unknown as typeof fetch; + await loadServiceStatus({ LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093/" }, { now: NOW, fetchImpl: capture }); + expect(seen[0]).toBe("http://alertmanager:9093/api/v2/alerts"); + }); +}); + +describe("isServiceStatusEnabled", () => { + it("is off when unset or blank, so the hosted Worker never serves an all-unknown board", () => { + expect(isServiceStatusEnabled({})).toBe(false); + expect(isServiceStatusEnabled({ LOOPOVER_ALERTMANAGER_URL: " " })).toBe(false); + }); + + it("is on when a source is configured", () => { + expect(isServiceStatusEnabled({ LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" })).toBe(true); + }); +}); + +describe("unknownServiceStatus", () => { + it("covers every component, so a degraded read never silently drops one", () => { + const payload = unknownServiceStatus(NOW.toISOString(), "because"); + expect(payload.components.map((c) => c.component)).toEqual([...SERVICE_STATUS_COMPONENTS]); + }); +}); + +describe("GET /v1/public/service-status", () => { + const get = (env: Env) => createApp().request("/v1/public/service-status", {}, env); + + it("404s where no alerting source is configured", async () => { + const response = await get(createTestEnv()); + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ error: "not_found" }); + }); + + it("serves the board where one is, without requiring any credential", async () => { + const env = createTestEnv({ LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" } as Partial); + const response = await get(env); + expect(response.status).toBe(200); + const body = (await response.json()) as { overall: string; components: unknown[] }; + // Alertmanager is unreachable from the test process, so this exercises the honest-degradation path end + // to end: a 200 that says "unknown", not a 500 and not a green board. + expect(body.overall).toBe("unknown"); + expect(body.components).toHaveLength(SERVICE_STATUS_COMPONENTS.length); + }); + + it("INVARIANT: the payload carries no host, instance, capacity or alert-label detail", async () => { + const env = createTestEnv({ LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" } as Partial); + const serialized = await (await get(env)).text(); + for (const forbidden of ["alertmanager", "9093", "instance", "job", "pod", "localhost", "cpu", "memory", "disk"]) { + expect(serialized.toLowerCase(), forbidden).not.toContain(forbidden); + } + }); + + it("caches briefly -- this is the endpoint people refresh during an incident", async () => { + const env = createTestEnv({ LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" } as Partial); + expect((await get(env)).headers.get("Cache-Control")).toBe("public, max-age=15, stale-while-revalidate=30"); + }); +});