From 5a12f456cff3dbaa6481624fa38aefa910c94dd9 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:46:20 -0700 Subject: [PATCH] feat(status): uptime and incident history, derived from persisted samples #9983 shipped the live board and deliberately published no uptime figure: there was no source for one. Alertmanager serves ACTIVE alerts only, resolved ones are gone, there is no Prometheus on the box, and nothing persisted a history. Deriving a percentage from a single live sample would have been fabrication. This is that source -- one sample per component per cron tick -- and the aggregation over it. Incidents are DERIVED from contiguous non-operational runs rather than posted by hand, which is what #9747 means by "incidents appear without manual posting". Three ways a plausible implementation lies here, and what this does instead: THE WINDOW LYING ABOUT ITSELF. A "30-day uptime" over three days of samples is the same falsehood as a verifier reporting green after checking nothing. Every window carries `since`, `measured`, `unmeasured` and `partial`, so "99.9% over a full month" is distinguishable from "99.9% over the two days we have been recording". Partiality is decided by the oldest sample OVERALL, not the oldest one inside the window -- an in-window sample can never predate the window start, so reading it from there marks every window partial forever. That was a real bug in the first cut, caught by its own test. UNMEASURED TIME COUNTED AS UPTIME. An `unknown` sample means the source could not be read. Counting it as up inflates the figure exactly when we were blind; counting it as down invents an outage nobody saw. It is excluded from the percentage and reported separately, and stored rather than skipped -- dropping those rows would silently shrink the window. BLINDNESS MANUFACTURING INCIDENTS. An `unknown` tick neither opens nor closes an incident. Letting it close one would split a single outage in two and double the incident count on any night the alerting source was flaky. A run's severity is the worst status within it, so an escalation is one incident, not two. A run still open at the newest sample reports a null end rather than being closed at "now", which would assert a recovery nobody observed. An empty or unreadable history yields a null uptime -- no data must render as no claim, never 0% (reads as a total outage) or 100% (reads as perfect). The sampler rides every cron tick: a handful of INSERTs, no fan-out, no GitHub budget. Coarser sampling would make short incidents invisible. It reads through the SAME loader the public route uses, so the history and the live board are the same measurement rather than two read paths free to disagree. Retention is 35 days, deliberately EXCEEDING the widest reported window (30), so the 30-day figure never loses its own tail to the prune. That inequality is asserted in a test, not just commented, because widening the windows past it would break it silently. Two things the checkers caught and taught: the table is registered as raw-SQL-only rather than declared in the drizzle schema (a declaration nothing imports is a dead export), and it carries a second index leading with `sampled_at` because the prune deletes by age across all components while the read is per-component over a window. Closes #9985 --- migrations/0209_service_status_samples.sql | 37 + scripts/check-schema-drift.ts | 4 + src/api/routes.ts | 10 +- src/db/retention.ts | 7 + src/index.ts | 4 + src/queue/job-dispatch.ts | 14 + src/selfhost/queue-common.ts | 3 + src/selfhost/service-status-history.ts | 187 ++++ src/types.ts | 8 + test/unit/index.test.ts | 1047 ++++++++++++++++---- test/unit/service-status-history.test.ts | 235 +++++ 11 files changed, 1367 insertions(+), 189 deletions(-) create mode 100644 migrations/0209_service_status_samples.sql create mode 100644 src/selfhost/service-status-history.ts create mode 100644 test/unit/service-status-history.test.ts diff --git a/migrations/0209_service_status_samples.sql b/migrations/0209_service_status_samples.sql new file mode 100644 index 0000000000..05c2a625ca --- /dev/null +++ b/migrations/0209_service_status_samples.sql @@ -0,0 +1,37 @@ +-- #9985 (slice 2 of #9747): the source for "has it been healthy?". +-- +-- /v1/public/service-status answers "is each component healthy RIGHT NOW" by reading Alertmanager's active +-- alerts. It cannot answer the past tense, because there is nothing to read: Alertmanager serves active +-- alerts only, resolved ones are gone, there is no Prometheus on the box (metrics flow through the +-- otel-collector), and nothing persisted a history. #9983 therefore published no uptime figure rather than +-- deriving one from a single live sample. +-- +-- This is that history: one row per component per cron tick. Uptime and incidents are DERIVED from these +-- rows rather than posted by hand, which is what #9747 means by "incidents appear without manual posting". +-- +-- `status` carries the same vocabulary the endpoint publishes -- operational | degraded | outage | unknown -- +-- and `unknown` is stored rather than skipped ON PURPOSE. A tick where the alerting source could not be read +-- is UNMEASURED time, not healthy time and not an outage; dropping those rows would silently shrink the +-- window and let a period of blindness read as uptime. Keeping them is what lets the read path report +-- coverage separately from the percentage. +-- +-- No UNIQUE on (component, sampled_at): two ticks in the same second are harmless to an aggregate over +-- contiguous runs, whereas a constraint violation on a best-effort telemetry write is not. +CREATE TABLE IF NOT EXISTS service_status_samples ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + component TEXT NOT NULL, + status TEXT NOT NULL, + sampled_at TEXT NOT NULL +); + +-- The read is always "this component, over this trailing window, in time order", which this covers exactly. +CREATE INDEX IF NOT EXISTS service_status_samples_component_time + ON service_status_samples (component, sampled_at); + +-- The retention prune deletes by age alone (`sampled_at < cutoff`, across every component), so it needs an +-- index LEADING with that column -- the composite above leads with `component` and cannot serve it. Both +-- exist because the two access patterns are genuinely different: the read is always per-component over a +-- window, the prune is always all-components before a cutoff. test/unit/retention.test.ts enforces the +-- latter for every table in RETENTION_POLICY, which is how the omission surfaced. +CREATE INDEX IF NOT EXISTS service_status_samples_sampled_at + ON service_status_samples (sampled_at); diff --git a/scripts/check-schema-drift.ts b/scripts/check-schema-drift.ts index 259540a7ce..21f579ed56 100644 --- a/scripts/check-schema-drift.ts +++ b/scripts/check-schema-drift.ts @@ -46,6 +46,10 @@ export const RAW_SQL_ONLY_TABLES: Set = new Set([ "decision_ledger_anchors", "decision_records", "decision_replay_inputs", + // #9985: written and read only through raw statements in selfhost/service-status-history.ts, the same + // posture as the decision_* tables above. Declaring it in the drizzle schema would add an export nothing + // imports (dead-exports:check rejects that, correctly) purely to satisfy this checker. + "service_status_samples", // #9028: same raw-SQL-only posture as its sibling above — an operator-private blob table the drizzle // schema never queries (writes go through persistDecisionReplayPrompt's raw statement; reads happen only // in the operator's own extract SQL for the replay CLI). diff --git a/src/api/routes.ts b/src/api/routes.ts index 1b0ab81da5..33025838bd 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -309,6 +309,7 @@ import { loadDecisionLedgerTip, loadPublicDecisionRecord, loadPublicLedgerRow, v import { buildEvalScoreRecordsFromRulePrecision, filterEvalScoreRecords } from "../review/eval-score-records"; import { buildPublicCorpusCommitments } from "../review/public-eval-corpus"; import { isServiceStatusEnabled, loadServiceStatus } from "../selfhost/service-status"; +import { buildComponentHistory, loadServiceStatusSamples } from "../selfhost/service-status-history"; import { anchorSigningInput, buildLedgerAnchorPayload, currentAnchorKey, diagnoseAnchorPublicKeys, parseAnchorPublicKeys, publicAnchorStatus, signLedgerAnchorPayload } from "../review/ledger-anchor"; import { resolveProofPage } from "../review/proof-summary"; import { renderProofBadgeSvg } from "./proof-badge"; @@ -929,10 +930,17 @@ export function createApp() { app.get("/v1/public/service-status", async (c) => { if (!isServiceStatusEnabled(c.env)) return c.json({ error: "not_found" }, 404); const status = await loadServiceStatus(c.env); + // #9985: the past tense, derived from persisted samples rather than posted by hand. Attached per + // component so a reader never has to correlate two lists, and computed from the SAME status vocabulary + // the live board publishes. + const now = new Date(); + const components = await Promise.all( + status.components.map(async (entry) => ({ ...entry, ...buildComponentHistory(await loadServiceStatusSamples(c.env, entry.component, now), now.getTime()) })), + ); // Shorter than the sibling public surfaces on purpose: this is the endpoint people refresh DURING an // incident, and a 60s cache would keep serving "operational" for a minute after an outage started. c.header("Cache-Control", "public, max-age=15, stale-while-revalidate=30"); - return c.json(status); + return c.json({ ...status, components }); }); app.get("/v1/public/eval-scores", async (c) => { diff --git a/src/db/retention.ts b/src/db/retention.ts index cc5acd5350..16c9fb0287 100644 --- a/src/db/retention.ts +++ b/src/db/retention.ts @@ -22,6 +22,12 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [ // needs would be gone by the time the fold ran, and the rollup would permanently over-count exactly the // PRs the live query never counted. { table: "orb_pr_outcomes", column: "occurred_at", days: 90 }, + // #9985: status samples are high-volume (one row per component per ~2-min tick) and low-value once the + // window that reads them has passed. 35 days deliberately EXCEEDS the widest reported window (30), so the + // 30-day uptime figure never silently loses its own tail to the prune -- a retention equal to the window + // would make the oldest day of every month quietly unmeasured. No rollup is needed while that inequality + // holds; widening UPTIME_WINDOW_DAYS past 35 without widening this would break it. + { table: "service_status_samples", column: "sampled_at", days: 35 }, // #9783: orb_signals had no rule at all and grew without bound, while being a PUBLIC data source // (computeFleetAnalytics' /fairness headline, and #9775's weekly fleet trend). It could not simply be // pruned: listFleetInstances reports a LIFETIME signalCount, and /v1/internal/fleet/analytics takes an @@ -127,6 +133,7 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [ // not scan-optimal, which is acceptable for the lower row counts of the tables that fall back today. export const RETENTION_PK_COLUMN: Readonly> = { audit_events: "id", + service_status_samples: "id", // #9783: orb_signals has `id INTEGER PRIMARY KEY` (migrations/0060), so it maps cleanly here rather than // joining RETENTION_COMPOSITE_PK_TABLES -- its UNIQUE (instance_id, repo_hash, pr_hash) is the ingest // dedup key, not its primary key. The fold below uses its own bounded-slice path rather than the generic diff --git a/src/index.ts b/src/index.ts index 7b6777a85c..86f071072c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -143,6 +143,10 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController): // impossible. The job's own decideLedgerAnchorSchedule (in ledger-anchor-scheduler.ts) is what actually // decides whether THIS tick does anything; a cheap two-query no-op the overwhelming majority of ticks. jobs.push({ type: "anchor-decision-ledger", requestedBy: "schedule", isHourly }); + // #9985: sampled every tick, unconditionally. The board it feeds is only as good as its resolution, and a + // deployment with no alerting source configured simply records `unknown` -- which the aggregation reports + // as unmeasured coverage rather than as health. + jobs.push({ type: "sample-service-status", requestedBy: "schedule" }); const selfHostedReviews = isSelfHostedReviewRuntime(env); const queueSnapshot = selfHostedReviews ? await queueSnapshotFromBinding(env.JOBS).catch((error) => { diff --git a/src/queue/job-dispatch.ts b/src/queue/job-dispatch.ts index dd09bddfb4..796f20bc79 100644 --- a/src/queue/job-dispatch.ts +++ b/src/queue/job-dispatch.ts @@ -53,6 +53,8 @@ import { withInstallationTokenRetry } from "../github/app"; import { runScheduledLedgerAnchor, resolveGitAnchorTarget } from "../review/ledger-anchor-scheduler"; import { submitToGitAnchor } from "../review/ledger-anchor-git"; import { incr } from "../selfhost/metrics"; +import { loadServiceStatus } from "../selfhost/service-status"; +import { recordServiceStatusSamples } from "../selfhost/service-status-history"; import { generateSignalSnapshots } from "./signal-snapshot"; import { isDecisionAuditEnabled, runDecisionAuditSample } from "../review/decision-audit"; import { isRiskControlEnabled, runRiskControlRecalibration } from "../review/risk-control-wire"; @@ -97,6 +99,18 @@ export async function processJob(env: Env, message: JobMessage): Promise { case "refresh-registry": await refreshRegistry(env); return; + case "sample-service-status": { + // #9985: read the live board through the SAME loader the public route uses, then persist one row per + // component. Reusing the loader is what keeps the history and the live status the same measurement -- + // a second read path here could report a component healthy while the endpoint called it degraded, and + // the derived uptime would then describe a board nobody ever saw. + const status = await loadServiceStatus(env); + await recordServiceStatusSamples( + env, + status.components.map((entry) => ({ component: entry.component, status: entry.status })), + ); + return; + } case "anchor-decision-ledger": { // Git anchoring runs only when BOTH the target (owner/repo, #9273) and an installation id (to mint a // token through) are configured -- unset means that backend simply isn't wired up yet; Rekor still diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index fd734ec9ad..fa3b4b66f1 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -801,6 +801,9 @@ const IMMEDIATE_SCHEDULED_JOB_TYPES = new Set([ // must fire independent of the hourly clock), and is a cheap 1-2 query no-op the overwhelming majority of // ticks -- not a heavy per-repo fan-out parent, so it needs no phase-spreading either. "anchor-decision-ledger", + // #9985: a handful of INSERTs, no fan-out. Phase-spreading it would jitter the sample cadence, and the + // derived uptime reads evenly-spaced samples as evenly-weighted time. + "sample-service-status", ]); export function scheduledEnqueueDelaySeconds(jobType: string): number { diff --git a/src/selfhost/service-status-history.ts b/src/selfhost/service-status-history.ts new file mode 100644 index 0000000000..605694b3af --- /dev/null +++ b/src/selfhost/service-status-history.ts @@ -0,0 +1,187 @@ +// Uptime and incident history for the public status board (#9985, slice 2 of #9747). +// +// #9983 publishes the live board and deliberately published no uptime figure, because there was no source +// for one: Alertmanager serves ACTIVE alerts only, resolved ones are gone, and nothing persisted a history. +// This is that source -- one sample per component per cron tick -- and the aggregation over it. +// +// ── THE WINDOW MUST NOT LIE ABOUT ITSELF ───────────────────────────────────────────────────────────── +// A "30-day uptime" computed over three days of samples is the same class of falsehood as a verifier that +// reports green because it checked nothing. So every window reports its COVERAGE -- how much of the window +// actually has samples -- alongside the percentage, and `since` says when history begins. A reader can then +// tell "99.9% over a full month" from "99.9% over the two days we have been recording". +// +// ── UNMEASURED TIME IS NOT UPTIME, AND NOT DOWNTIME ────────────────────────────────────────────────── +// An `unknown` sample means the alerting source could not be read on that tick. It is excluded from the +// percentage entirely rather than counted either way: counting it as up inflates the figure exactly when we +// were blind, and counting it as down invents an outage nobody observed. It is reported as its own number so +// the blindness is visible instead of averaged away. + +/** The status vocabulary the live board publishes, mirrored here because samples store it verbatim. */ +export type SampledStatus = "operational" | "degraded" | "outage" | "unknown"; + +export type StatusSample = { status: SampledStatus; sampledAt: string }; + +/** A contiguous run of non-operational, non-unknown samples: an incident, derived rather than posted. + * `endedAt` is null while the run reaches the newest sample -- the incident is still open. */ +export type StatusIncident = { status: "degraded" | "outage"; startedAt: string; endedAt: string | null }; + +export type UptimeWindow = { + windowDays: number; + /** Share of MEASURED samples that were operational, 0..1. Null when nothing was measured -- no data must + * render as no claim, never as 0% (which reads as a total outage) or 100% (which reads as perfect). */ + uptime: number | null; + /** Samples that carried a real reading. The denominator of `uptime`. */ + measured: number; + /** Samples where the source could not be read. Reported, never folded into `uptime`. */ + unmeasured: number; + /** Oldest sample in the window, or null when the window is empty. What makes a partial window legible. */ + since: string | null; + /** True when history does not reach back across the whole window, so the figure covers less than it says. */ + partial: boolean; +}; + +/** PURE. Is this a state a reader should see as an incident? `unknown` is deliberately NOT one -- we did not + * observe a problem, we failed to look. */ +export function isIncidentStatus(status: SampledStatus): status is "degraded" | "outage" { + return status === "degraded" || status === "outage"; +} + +/** + * PURE. Uptime over a trailing window, with the honesty fields that make it readable. + * + * `samples` may be in any order and may include rows outside the window; both are filtered here so the + * caller's query does not have to be the thing that is correct. + */ +export function computeUptimeWindow(samples: readonly StatusSample[], windowDays: number, nowMs: number): UptimeWindow { + const startMs = nowMs - windowDays * 86_400_000; + const inWindow = samples.filter((sample) => { + const at = Date.parse(sample.sampledAt); + return Number.isFinite(at) && at >= startMs && at <= nowMs; + }); + const measuredSamples = inWindow.filter((sample) => sample.status !== "unknown"); + const operational = measuredSamples.filter((sample) => sample.status === "operational").length; + const since = inWindow.length === 0 ? null : inWindow.reduce((oldest, s) => (s.sampledAt < oldest ? s.sampledAt : oldest), inWindow[0]!.sampledAt); + // Partiality is decided by the oldest sample OVERALL, not the oldest one inside the window. `inWindow` is + // filtered to `>= startMs`, so an in-window `since` can never predate the window start -- deriving + // partiality from it would mark every window partial forever, including ones with years of history behind + // them. What actually answers "does history reach back across this window" is whether recording began + // before the window did. + const oldestOverall = samples + .map((entry) => Date.parse(entry.sampledAt)) + .filter((ms) => Number.isFinite(ms)) + .reduce((oldest, ms) => (oldest === null || ms < oldest ? ms : oldest), null); + return { + windowDays, + // Guard the denominator, else null -- the same discipline every other published rate here uses. + uptime: measuredSamples.length > 0 ? operational / measuredSamples.length : null, + measured: measuredSamples.length, + unmeasured: inWindow.length - measuredSamples.length, + since, + // An empty history is partial too: it covers none of the period it names. + partial: oldestOverall === null || oldestOverall > startMs, + }; +} + +/** + * PURE. Incidents as contiguous runs of non-operational, non-unknown samples. + * + * DERIVED, NEVER POSTED -- that is #9747's "incidents appear without manual posting". An `unknown` sample + * does NOT close an incident and does not start one: a tick where we could not read the source is no evidence + * either way, and letting it split one outage into two would manufacture incidents out of our own blindness. + * A run's severity is the WORST status within it, so a degradation that escalates reports as an outage rather + * than as two adjacent incidents. + */ +export function computeIncidents(samples: readonly StatusSample[], windowDays: number, nowMs: number): StatusIncident[] { + const startMs = nowMs - windowDays * 86_400_000; + const ordered = samples + .filter((sample) => { + const at = Date.parse(sample.sampledAt); + return Number.isFinite(at) && at >= startMs && at <= nowMs; + }) + .sort((a, b) => a.sampledAt.localeCompare(b.sampledAt)); + + const incidents: StatusIncident[] = []; + let open: StatusIncident | null = null; + for (const sample of ordered) { + if (isIncidentStatus(sample.status)) { + if (open === null) open = { status: sample.status, startedAt: sample.sampledAt, endedAt: null }; + else if (sample.status === "outage") open.status = "outage"; + continue; + } + // `unknown` is not evidence of recovery, so only an `operational` sample closes a run. + if (sample.status === "operational" && open !== null) { + open.endedAt = sample.sampledAt; + incidents.push(open); + open = null; + } + } + // Still open at the newest sample: reported with a null end rather than closed at "now", which would + // assert a recovery that was never observed. + if (open !== null) incidents.push(open); + return incidents; +} + +/** The windows the public board reports. Fixed rather than caller-supplied: a status page whose windows vary + * per request is not comparable to itself. */ +export const UPTIME_WINDOW_DAYS: readonly number[] = [1, 7, 30]; + +export type ComponentHistory = { uptime: UptimeWindow[]; incidents: StatusIncident[] }; + +/** PURE. Everything the board publishes about one component's past. */ +export function buildComponentHistory(samples: readonly StatusSample[], nowMs: number): ComponentHistory { + return { + uptime: UPTIME_WINDOW_DAYS.map((days) => computeUptimeWindow(samples, days, nowMs)), + // Over the widest window, so the incident list and the longest uptime figure describe the same period. + incidents: computeIncidents(samples, Math.max(...UPTIME_WINDOW_DAYS), nowMs), + }; +} + +// ── PERSISTENCE ─────────────────────────────────────────────────────────────────────────────────────── +// Kept in this module rather than a sibling so the write shape and the read shape are one file apart: the +// sampler stores exactly the vocabulary the aggregation above reads, and a change to one is visibly a change +// to the other. + +/** BEST EFFORT. Record one sample per component. A telemetry write must never fail the tick that carries it, + * and a missed sample is already representable -- it simply widens the gap the coverage fields report. */ +export async function recordServiceStatusSamples( + env: Env, + components: readonly { component: string; status: SampledStatus }[], + now: Date = new Date(), +): Promise { + const sampledAt = now.toISOString(); + // try/catch around the whole loop, not `.catch()` on the promise: `prepare()` throws SYNCHRONOUSLY when the + // binding is unusable, so a promise-only guard lets that escape and fail the tick this write is riding on -- + // exactly what "best effort" is supposed to prevent. Caught once around the loop rather than per row because + // a binding that fails for one component fails for all of them, and a partial sample set is worse than none: + // it would look like a real reading of only some components. + try { + for (const entry of components) { + await env.DB.prepare(`INSERT INTO service_status_samples (component, status, sampled_at) VALUES (?, ?, ?)`) + .bind(entry.component, entry.status, sampledAt) + .run(); + } + } catch { + // A missed sample is already representable: it simply widens the gap the coverage fields report. + } +} + +/** + * Load one component's samples over the widest reported window. + * + * FAIL-SAFE, like every other read behind this public surface: a read error yields no samples, which the + * aggregation reports as an empty, PARTIAL window with a null uptime -- "we cannot say", never a fabricated + * figure and never a thrown public endpoint. + */ +export async function loadServiceStatusSamples(env: Env, component: string, now: Date = new Date()): Promise { + const sinceIso = new Date(now.getTime() - Math.max(...UPTIME_WINDOW_DAYS) * 86_400_000).toISOString(); + try { + const result = await env.DB.prepare( + `SELECT status, sampled_at FROM service_status_samples WHERE component = ? AND sampled_at >= ? ORDER BY sampled_at`, + ) + .bind(component, sinceIso) + .all<{ status: string; sampled_at: string }>(); + return (result.results ?? []).map((row) => ({ status: row.status as SampledStatus, sampledAt: row.sampled_at })); + } catch { + return []; + } +} diff --git a/src/types.ts b/src/types.ts index 9ec45be661..43a8892a79 100644 --- a/src/types.ts +++ b/src/types.ts @@ -64,6 +64,14 @@ export type JobMessage = requestedBy: "schedule" | "test"; isHourly: boolean; } + | { + // #9985 (slice 2 of #9747): one status sample per component per tick, so the public board can answer + // "has it been healthy?" as well as "is it now". A single INSERT per component -- no fan-out, no + // GitHub budget -- which is why it rides every tick rather than the hourly clock: coarser sampling + // would make short incidents invisible to the derived history. + type: "sample-service-status"; + requestedBy: "schedule" | "test"; + } | { type: "sync-brokered-installed-repos"; requestedBy: "schedule" | "api" | "test"; diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index a9caac5793..576ee83870 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -19,7 +19,10 @@ describe("worker entrypoint", () => { it("delegates fetch requests to the Hono app", async () => { const env = createTestEnv(); - const response = await worker.fetch(new Request("https://loopover.test/health"), env); + const response = await worker.fetch( + new Request("https://loopover.test/health"), + env, + ); expect(response.status).toBe(200); }); @@ -32,7 +35,12 @@ describe("worker entrypoint", () => { messages: [ { id: "dlq-msg-1", - body: { type: "github-webhook", deliveryId: "d-dlq", eventName: "pull_request", payload: {} }, + body: { + type: "github-webhook", + deliveryId: "d-dlq", + eventName: "pull_request", + payload: {}, + }, ack: () => acked.push("dlq-msg-1"), retry: () => retried.push("dlq-msg-1"), }, @@ -54,7 +62,13 @@ describe("worker entrypoint", () => { messages: [ { id: "wh-dlq-1", - body: { type: "github-webhook", deliveryId: "d-wh-dlq", eventName: "pull_request", payload: {}, redriven: true }, + body: { + type: "github-webhook", + deliveryId: "d-wh-dlq", + eventName: "pull_request", + payload: {}, + redriven: true, + }, ack: () => acked.push("wh-dlq-1"), retry: () => retried.push("wh-dlq-1"), }, @@ -71,14 +85,22 @@ describe("worker entrypoint", () => { const env = createTestEnv(); delete env.SELFHOST_TRANSIENT_CACHE; const sent: import("../../src/types").JobMessage[] = []; - env.WEBHOOKS = { send: async (message: import("../../src/types").JobMessage) => void sent.push(message) } as unknown as Queue; + env.WEBHOOKS = { + send: async (message: import("../../src/types").JobMessage) => + void sent.push(message), + } as unknown as Queue; const acked: string[] = []; const batch = { queue: "loopover-webhooks-dlq", messages: [ { id: "wh-dlq-broker-only", - body: { type: "github-webhook", deliveryId: "d-broker-only", eventName: "pull_request", payload: {} }, + body: { + type: "github-webhook", + deliveryId: "d-broker-only", + eventName: "pull_request", + payload: {}, + }, ack: () => acked.push("wh-dlq-broker-only"), retry: () => undefined, }, @@ -94,14 +116,21 @@ describe("worker entrypoint", () => { it("acks and ignores stale review-execution jobs from a broker-only Cloudflare runtime", async () => { const env = createTestEnv(); delete env.SELFHOST_TRANSIENT_CACHE; - const warned = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const warned = vi + .spyOn(console, "warn") + .mockImplementation(() => undefined); const acked: string[] = []; const retried: string[] = []; const batch = { messages: [ { id: "hosted-review-job", - body: { type: "github-webhook", deliveryId: "d-hosted-review", eventName: "pull_request", payload: {} }, + body: { + type: "github-webhook", + deliveryId: "d-hosted-review", + eventName: "pull_request", + payload: {}, + }, ack: () => acked.push("hosted-review-job"), retry: () => retried.push("hosted-review-job"), }, @@ -120,7 +149,10 @@ describe("worker entrypoint", () => { it("acks successful queue messages and retries failed messages", async () => { const env = createTestEnv(); - vi.stubGlobal("fetch", async () => new Response("missing", { status: 404 })); + vi.stubGlobal( + "fetch", + async () => new Response("missing", { status: 404 }), + ); const acked: string[] = []; const retried: string[] = []; const batch = { @@ -149,8 +181,20 @@ describe("worker entrypoint", () => { vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(new Date("2026-06-24T12:00:00.000Z")); const env = createTestEnv(); - await recordGitHubRateLimitObservation(env, { repoFullName: "owner/repo", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 5, resetAt: "2026-06-24T12:30:00.000Z", observedAt: "2026-06-24T12:00:00.000Z" }); - vi.stubGlobal("fetch", async () => new Response("missing", { status: 404 })); + await recordGitHubRateLimitObservation(env, { + repoFullName: "owner/repo", + resource: "rest", + path: "/x", + statusCode: 200, + limitValue: 5000, + remaining: 5, + resetAt: "2026-06-24T12:30:00.000Z", + observedAt: "2026-06-24T12:00:00.000Z", + }); + vi.stubGlobal( + "fetch", + async () => new Response("missing", { status: 404 }), + ); const retries: Array<{ delaySeconds?: number } | undefined> = []; const batch = { messages: [ @@ -175,20 +219,47 @@ describe("worker entrypoint", () => { vi.setSystemTime(new Date("2026-06-24T12:00:00.000Z")); const env = createTestEnv(); // Scoped to this job's own installation bucket (#audit-rate-scoping) — installationId 123 below. - await recordGitHubRateLimitObservation(env, { repoFullName: "owner/repo", admissionKey: "installation:123", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 120, resetAt: "2026-06-24T12:10:00.000Z", observedAt: "2026-06-24T12:00:00.000Z" }); + await recordGitHubRateLimitObservation(env, { + repoFullName: "owner/repo", + admissionKey: "installation:123", + resource: "rest", + path: "/x", + statusCode: 200, + limitValue: 5000, + remaining: 120, + resetAt: "2026-06-24T12:10:00.000Z", + observedAt: "2026-06-24T12:00:00.000Z", + }); const acked: string[] = []; const retries: Array<{ delaySeconds?: number } | undefined> = []; - const requeued: Array<{ message: import("../../src/types").JobMessage; delaySeconds?: number }> = []; + const requeued: Array<{ + message: import("../../src/types").JobMessage; + delaySeconds?: number; + }> = []; env.JOBS = { - async send(message: import("../../src/types").JobMessage, options?: { delaySeconds?: number }) { - requeued.push({ message, ...(options?.delaySeconds === undefined ? {} : { delaySeconds: options.delaySeconds }) }); + async send( + message: import("../../src/types").JobMessage, + options?: { delaySeconds?: number }, + ) { + requeued.push({ + message, + ...(options?.delaySeconds === undefined + ? {} + : { delaySeconds: options.delaySeconds }), + }); }, } as unknown as Queue; const batch = { messages: [ { id: "background-regate", - body: { type: "agent-regate-pr", deliveryId: "sweep:owner/repo#7", repoFullName: "owner/repo", prNumber: 7, installationId: 123 }, + body: { + type: "agent-regate-pr", + deliveryId: "sweep:owner/repo#7", + repoFullName: "owner/repo", + prNumber: 7, + installationId: 123, + }, ack: () => acked.push("background-regate"), retry: (options?: { delaySeconds?: number }) => retries.push(options), }, @@ -247,13 +318,33 @@ describe("worker entrypoint", () => { const env = createTestEnv(); // reconcile-open-prs has no per-installation field, so it draws from the SAME shared (no-admissionKey) // observation refresh-registry's own equivalent test above uses. - await recordGitHubRateLimitObservation(env, { repoFullName: "owner/repo", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 5, resetAt: "2026-06-24T12:30:00.000Z", observedAt: "2026-06-24T12:00:00.000Z" }); + await recordGitHubRateLimitObservation(env, { + repoFullName: "owner/repo", + resource: "rest", + path: "/x", + statusCode: 200, + limitValue: 5000, + remaining: 5, + resetAt: "2026-06-24T12:30:00.000Z", + observedAt: "2026-06-24T12:00:00.000Z", + }); const acked: string[] = []; const retries: Array<{ delaySeconds?: number } | undefined> = []; - const requeued: Array<{ message: import("../../src/types").JobMessage; delaySeconds?: number }> = []; + const requeued: Array<{ + message: import("../../src/types").JobMessage; + delaySeconds?: number; + }> = []; env.JOBS = { - async send(message: import("../../src/types").JobMessage, options?: { delaySeconds?: number }) { - requeued.push({ message, ...(options?.delaySeconds === undefined ? {} : { delaySeconds: options.delaySeconds }) }); + async send( + message: import("../../src/types").JobMessage, + options?: { delaySeconds?: number }, + ) { + requeued.push({ + message, + ...(options?.delaySeconds === undefined + ? {} + : { delaySeconds: options.delaySeconds }), + }); }, } as unknown as Queue; const batch = { @@ -274,7 +365,12 @@ describe("worker entrypoint", () => { // budget would have been silently ignored and runOpenPrReconciliation would have run immediately. expect(acked).toEqual(["reconcile-tick"]); expect(retries).toEqual([]); - expect(requeued).toEqual([{ message: { type: "reconcile-open-prs", requestedBy: "schedule" }, delaySeconds: 900 }]); // delayUntil clamps to [30, 900] + expect(requeued).toEqual([ + { + message: { type: "reconcile-open-prs", requestedBy: "schedule" }, + delaySeconds: 900, + }, + ]); // delayUntil clamps to [30, 900] vi.useRealTimers(); }); @@ -285,7 +381,17 @@ describe("worker entrypoint", () => { // refresh-installation-health carries no installationId (it loops over every installation internally), so // githubRateLimitAdmissionKeyForJob resolves it to `null` -- its dequeue-time check reads the unscoped // (admissionKey: undefined) latest-observations window, which any recent exhausted REST observation trips. - await recordGitHubRateLimitObservation(env, { repoFullName: null, admissionKey: "installation:1", resource: "rest", path: "/app/installations/1", statusCode: 200, limitValue: 5000, remaining: 5, resetAt: "2026-06-24T12:30:00.000Z", observedAt: "2026-06-24T12:00:00.000Z" }); + await recordGitHubRateLimitObservation(env, { + repoFullName: null, + admissionKey: "installation:1", + resource: "rest", + path: "/app/installations/1", + statusCode: 200, + limitValue: 5000, + remaining: 5, + resetAt: "2026-06-24T12:30:00.000Z", + observedAt: "2026-06-24T12:00:00.000Z", + }); const fetchCalls: string[] = []; vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { fetchCalls.push(input.toString()); @@ -293,17 +399,31 @@ describe("worker entrypoint", () => { }); const acked: string[] = []; const retries: Array<{ delaySeconds?: number } | undefined> = []; - const requeued: Array<{ message: import("../../src/types").JobMessage; delaySeconds?: number }> = []; + const requeued: Array<{ + message: import("../../src/types").JobMessage; + delaySeconds?: number; + }> = []; env.JOBS = { - async send(message: import("../../src/types").JobMessage, options?: { delaySeconds?: number }) { - requeued.push({ message, ...(options?.delaySeconds === undefined ? {} : { delaySeconds: options.delaySeconds }) }); + async send( + message: import("../../src/types").JobMessage, + options?: { delaySeconds?: number }, + ) { + requeued.push({ + message, + ...(options?.delaySeconds === undefined + ? {} + : { delaySeconds: options.delaySeconds }), + }); }, } as unknown as Queue; const batch = { messages: [ { id: "installation-health-tick", - body: { type: "refresh-installation-health", requestedBy: "schedule" }, + body: { + type: "refresh-installation-health", + requestedBy: "schedule", + }, ack: () => acked.push("installation-health-tick"), retry: (options?: { delaySeconds?: number }) => retries.push(options), }, @@ -318,7 +438,15 @@ describe("worker entrypoint", () => { expect(fetchCalls).toEqual([]); expect(acked).toEqual(["installation-health-tick"]); expect(retries).toEqual([]); - expect(requeued).toEqual([{ message: { type: "refresh-installation-health", requestedBy: "schedule" }, delaySeconds: 900 }]); // delayUntil clamps to [30, 900] + expect(requeued).toEqual([ + { + message: { + type: "refresh-installation-health", + requestedBy: "schedule", + }, + delaySeconds: 900, + }, + ]); // delayUntil clamps to [30, 900] vi.useRealTimers(); }); @@ -327,22 +455,22 @@ describe("worker entrypoint", () => { vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { const url = input.toString(); if (url.includes("master_repositories.json")) return Response.json({}); - if (url.includes("api.gittensor.io") || url.includes("mirror.gittensor.io")) return new Response("missing", { status: 404 }); + if ( + url.includes("api.gittensor.io") || + url.includes("mirror.gittensor.io") + ) + return new Response("missing", { status: 404 }); return Response.json([]); }); const waitUntil: Promise[] = []; - await worker.scheduled( - {} as ScheduledController, - env, - { - waitUntil: (promise: Promise) => { - waitUntil.push(promise); - }, - passThroughOnException: () => {}, - exports: {}, - props: {}, - } as unknown as ExecutionContext, - ); + await worker.scheduled({} as ScheduledController, env, { + waitUntil: (promise: Promise) => { + waitUntil.push(promise); + }, + passThroughOnException: () => {}, + exports: {}, + props: {}, + } as unknown as ExecutionContext); await Promise.allSettled(waitUntil); expect(waitUntil).toHaveLength(1); }); @@ -358,12 +486,24 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // A regular */2 tick (not :00, not :30) enqueues ONLY the light auto-maintain sweep — the heavier sync/health // jobs are gated to :00/:30, so the tight cadence stays cheap while merges/closes fire promptly. - expect(sent).toEqual([{ type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }, { type: "agent-regate-sweep", requestedBy: "schedule" }]); + expect(sent).toEqual([ + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, + { type: "agent-regate-sweep", requestedBy: "schedule" }, + ]); }); it("enqueues the APR repo-transfer poll on an hourly tick only when LOOPOVER_APR_TRANSFER_POLL is set (#7741)", async () => { @@ -378,10 +518,17 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); - expect(captured).toContainEqual({ type: "poll-apr-repo-transfers", requestedBy: "schedule" }); + expect(captured).toContainEqual({ + type: "poll-apr-repo-transfers", + requestedBy: "schedule", + }); }); it("keeps enqueueing scheduled sweeps while prior per-PR regate jobs are queued (#2119)", async () => { @@ -398,20 +545,35 @@ describe("worker entrypoint", () => { snapshotCalled = true; return { totals: { pending: 2, processing: 0, dead: 0, due: 2 }, - byType: [{ type: "agent-regate-pr", status: "pending", count: 2, due: 2 }], + byType: [ + { type: "agent-regate-pr", status: "pending", count: 2, due: 2 }, + ], }; }, } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:30:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); expect(sent).toEqual([ - { type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }, + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, { type: "agent-regate-sweep", requestedBy: "schedule" }, - { type: "backfill-registered-repos", requestedBy: "schedule", mode: "light" }, + { + type: "backfill-registered-repos", + requestedBy: "schedule", + mode: "light", + }, { type: "repair-data-fidelity", requestedBy: "schedule" }, { type: "refresh-installation-health", requestedBy: "schedule" }, { type: "backlog-convergence-sweep", requestedBy: "schedule" }, @@ -428,20 +590,40 @@ describe("worker entrypoint", () => { }, snapshot: async () => ({ totals: { pending: 0, processing: 1, dead: 0, due: 0 }, - byType: [{ type: "agent-regate-sweep", status: "processing", count: 1, due: 0 }], + byType: [ + { + type: "agent-regate-sweep", + status: "processing", + count: 1, + due: 0, + }, + ], }), } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:30:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // No SECOND "agent-regate-sweep" trigger is enqueued behind the one already in flight; the other :30 jobs // are unaffected since they never depended on the (removed, broad) backlog check. expect(sent).toEqual([ - { type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }, - { type: "backfill-registered-repos", requestedBy: "schedule", mode: "light" }, + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, + { + type: "backfill-registered-repos", + requestedBy: "schedule", + mode: "light", + }, { type: "repair-data-fidelity", requestedBy: "schedule" }, { type: "refresh-installation-health", requestedBy: "schedule" }, { type: "backlog-convergence-sweep", requestedBy: "schedule" }, @@ -457,21 +639,41 @@ describe("worker entrypoint", () => { }, snapshot: async () => ({ totals: { pending: 0, processing: 1, dead: 0, due: 0 }, - byType: [{ type: "backlog-convergence-sweep", status: "processing", count: 1, due: 0 }], + byType: [ + { + type: "backlog-convergence-sweep", + status: "processing", + count: 1, + due: 0, + }, + ], }), } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:30:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // No SECOND "backlog-convergence-sweep" trigger is enqueued behind the one already in flight; the other :30 // jobs (including agent-regate-sweep, whose OWN backlog is unaffected) still fire normally. expect(sent).toEqual([ - { type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }, + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, { type: "agent-regate-sweep", requestedBy: "schedule" }, - { type: "backfill-registered-repos", requestedBy: "schedule", mode: "light" }, + { + type: "backfill-registered-repos", + requestedBy: "schedule", + mode: "light", + }, { type: "repair-data-fidelity", requestedBy: "schedule" }, { type: "refresh-installation-health", requestedBy: "schedule" }, ]); @@ -492,13 +694,27 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // Fails OPEN on a broken snapshot binding: the sweep still enqueues, and the failure is surfaced (not // silently swallowed) so an operator can see the introspection is unavailable. - expect(sent).toEqual([{ type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }, { type: "agent-regate-sweep", requestedBy: "schedule" }]); - expect(warn).toHaveBeenCalledWith(expect.stringContaining("selfhost_queue_snapshot_failed")); + expect(sent).toEqual([ + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, + { type: "agent-regate-sweep", requestedBy: "schedule" }, + ]); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("selfhost_queue_snapshot_failed"), + ); }); it("does not enqueue review sweeps from a broker-only Cloudflare runtime (anchoring still runs -- it is not a review sweep, #9274)", async () => { @@ -513,10 +729,21 @@ describe("worker entrypoint", () => { delete env.SELFHOST_TRANSIENT_CACHE; const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); - expect(sent).toEqual([{ type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }]); + expect(sent).toEqual([ + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, + ]); }); it("keeps broker-only Cloudflare maintenance cheap on :30 ticks", async () => { @@ -531,11 +758,20 @@ describe("worker entrypoint", () => { delete env.SELFHOST_TRANSIENT_CACHE; const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:30:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); expect(sent).toEqual([ - { type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }, + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, { type: "repair-data-fidelity", requestedBy: "schedule" }, { type: "refresh-installation-health", requestedBy: "schedule" }, ]); @@ -544,12 +780,29 @@ describe("worker entrypoint", () => { it("THROTTLES the sweep when the GitHub REST budget is at/below the maintenance headroom (#6 backpressure)", async () => { const sent: Array = []; const env = createTestEnv({ - JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, }); // Seed a low REST observation (remaining 50 <= MAINTENANCE_RESERVED_HEADROOM=150) with a future reset. - await recordGitHubRateLimitObservation(env, { repoFullName: "owner/repo", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 50, resetAt: new Date(Date.now() + 600_000).toISOString(), observedAt: new Date().toISOString() }); + await recordGitHubRateLimitObservation(env, { + repoFullName: "owner/repo", + resource: "rest", + path: "/x", + statusCode: 200, + limitValue: 5000, + remaining: 50, + resetAt: new Date(Date.now() + 600_000).toISOString(), + observedAt: new Date().toISOString(), + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // The sweep is NOT enqueued this tick — the shared budget is reserved for webhooks; the next tick retries. expect(sent.find((m) => m.type === "agent-regate-sweep")).toBeUndefined(); @@ -558,30 +811,68 @@ describe("worker entrypoint", () => { it("THROTTLES the open-data backfill too when the REST budget is at/below the maintenance headroom, keeping the cheap health jobs (#audit-rate-headroom)", async () => { const sent: Array = []; const env = createTestEnv({ - JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, }); // Low REST budget (remaining 50 <= MAINTENANCE_RESERVED_HEADROOM=150) with a future reset. - await recordGitHubRateLimitObservation(env, { repoFullName: "owner/repo", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 50, resetAt: new Date(Date.now() + 600_000).toISOString(), observedAt: new Date().toISOString() }); + await recordGitHubRateLimitObservation(env, { + repoFullName: "owner/repo", + resource: "rest", + path: "/x", + statusCode: 200, + limitValue: 5000, + remaining: 50, + resetAt: new Date(Date.now() + 600_000).toISOString(), + observedAt: new Date().toISOString(), + }); const waitUntil: Promise[] = []; // A :30 tick is when the backfill would normally enqueue — assert it does NOT while the budget is reserved. - await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:30:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // Backfill (+ sweep) yield the budget to webhooks; the cheap single-call health jobs still run. - expect(sent.some((m) => m.type === "backfill-registered-repos")).toBe(false); + expect(sent.some((m) => m.type === "backfill-registered-repos")).toBe( + false, + ); expect(sent.some((m) => m.type === "agent-regate-sweep")).toBe(false); expect(sent.some((m) => m.type === "repair-data-fidelity")).toBe(true); - expect(sent.some((m) => m.type === "refresh-installation-health")).toBe(true); + expect(sent.some((m) => m.type === "refresh-installation-health")).toBe( + true, + ); }); it("enqueues the open-data backfill on a :30 tick when there is REST headroom (#audit-rate-headroom)", async () => { const sent: Array = []; const env = createTestEnv({ - JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, }); // Ample REST budget (remaining 4000 > MAINTENANCE_RESERVED_HEADROOM=150) → the throttle does NOT engage. - await recordGitHubRateLimitObservation(env, { repoFullName: "owner/repo", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 4000, resetAt: new Date(Date.now() + 600_000).toISOString(), observedAt: new Date().toISOString() }); + await recordGitHubRateLimitObservation(env, { + repoFullName: "owner/repo", + resource: "rest", + path: "/x", + statusCode: 200, + limitValue: 5000, + remaining: 4000, + resetAt: new Date(Date.now() + 600_000).toISOString(), + observedAt: new Date().toISOString(), + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:30:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); expect(sent.some((m) => m.type === "backfill-registered-repos")).toBe(true); expect(sent.some((m) => m.type === "agent-regate-sweep")).toBe(true); @@ -598,16 +889,29 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // refresh-registry is absent: a fresh self-host instance (this test's default createTestEnv) has no repo // opted into the experimental gittensor plugin, so the job is never enqueued (#experimental-gittensor-plugin) // — see "re-includes refresh-registry once a repo opts into the experimental gittensor plugin" below. expect(sent).toEqual([ - { type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: true }, + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: true, + }, + { type: "sample-service-status", requestedBy: "schedule" }, { type: "agent-regate-sweep", requestedBy: "schedule" }, - { type: "backfill-registered-repos", requestedBy: "schedule", mode: "light" }, + { + type: "backfill-registered-repos", + requestedBy: "schedule", + mode: "light", + }, { type: "repair-data-fidelity", requestedBy: "schedule" }, { type: "refresh-installation-health", requestedBy: "schedule" }, { type: "backlog-convergence-sweep", requestedBy: "schedule" }, @@ -631,17 +935,35 @@ describe("worker entrypoint", () => { }, } as unknown as Queue, }); - await (env.DB as unknown as { prepare: (s: string) => { bind: (...v: unknown[]) => { run: () => Promise } } }) - .prepare("INSERT INTO repositories (full_name, owner, name, is_installed, is_registered) VALUES (?, ?, ?, 1, 0)") + await ( + env.DB as unknown as { + prepare: (s: string) => { + bind: (...v: unknown[]) => { run: () => Promise }; + }; + } + ) + .prepare( + "INSERT INTO repositories (full_name, owner, name, is_installed, is_registered) VALUES (?, ?, ?, 1, 0)", + ) .bind("JSONbored/loopover", "JSONbored", "loopover") .run(); - await (await import("../../src/signals/focus-manifest-loader")).upsertRepoFocusManifest(env, "JSONbored/loopover", { experimental: { gittensor: true } }); + await ( + await import("../../src/signals/focus-manifest-loader") + ).upsertRepoFocusManifest(env, "JSONbored/loopover", { + experimental: { gittensor: true }, + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); - expect(sent.some((message) => message.type === "refresh-registry")).toBe(true); + expect(sent.some((message) => message.type === "refresh-registry")).toBe( + true, + ); }); it("always enqueues refresh-registry on a cloud (non-self-hosted) runtime, regardless of gittensor opt-in (#experimental-gittensor-plugin)", async () => { @@ -656,12 +978,18 @@ describe("worker entrypoint", () => { delete env.SELFHOST_TRANSIENT_CACHE; const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // Cloud never consults gittensorEnabledRepoFullNames at all — the `!selfHostedReviews` short-circuit // fires before the opted-in check, exactly like before this narrowing existed. - expect(sent.some((message) => message.type === "refresh-registry")).toBe(true); + expect(sent.some((message) => message.type === "refresh-registry")).toBe( + true, + ); }); it("enqueues full-sync scheduled work every six hours", async () => { @@ -675,14 +1003,27 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T06:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T06:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // refresh-registry is absent here too — see the comment on "enqueues hourly refreshes..." above. expect(sent).toEqual([ - { type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: true }, + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: true, + }, + { type: "sample-service-status", requestedBy: "schedule" }, { type: "agent-regate-sweep", requestedBy: "schedule" }, - { type: "backfill-registered-repos", requestedBy: "schedule", mode: "full" }, + { + type: "backfill-registered-repos", + requestedBy: "schedule", + mode: "full", + }, { type: "repair-data-fidelity", requestedBy: "schedule" }, { type: "refresh-installation-health", requestedBy: "schedule" }, { type: "backlog-convergence-sweep", requestedBy: "schedule" }, @@ -721,13 +1062,18 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T06:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T06:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // The enqueued SET is unchanged — jitter only spreads run_after timing, never which jobs are sent. // refresh-registry is absent here too — see the comment on "enqueues hourly refreshes..." above. expect(sent.map((s) => s.message.type)).toEqual([ "anchor-decision-ledger", + "sample-service-status", "agent-regate-sweep", "backfill-registered-repos", "repair-data-fidelity", @@ -752,12 +1098,16 @@ describe("worker entrypoint", () => { } // The priority sweep specifically stays immediate; at least one periodic job is actually deferred, so a // top-of-6h tick no longer fires every heavy fan-out parent in the same instant. - expect(sent.find((s) => s.message.type === "agent-regate-sweep")?.delaySeconds).toBeUndefined(); + expect( + sent.find((s) => s.message.type === "agent-regate-sweep")?.delaySeconds, + ).toBeUndefined(); expect(sent.some((s) => (s.delaySeconds ?? 0) > 0)).toBe(true); }); it("enqueues the ops-alerts job hourly ONLY when LOOPOVER_REVIEW_OPS is ON (flag-OFF is byte-identical)", async () => { - const sentFor = async (opsFlag?: string): Promise> => { + const sentFor = async ( + opsFlag?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ ...(opsFlag === undefined ? {} : { LOOPOVER_REVIEW_OPS: opsFlag }), @@ -768,17 +1118,25 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // Flag OFF (default) → no ops-alerts job; the enqueued set is unchanged from today. expect((await sentFor()).some((m) => m.type === "ops-alerts")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "ops-alerts")).toBe(false); + expect((await sentFor("false")).some((m) => m.type === "ops-alerts")).toBe( + false, + ); // Flag ON → exactly one ops-alerts job, enqueued in the hourly window. const on = await sentFor("true"); - expect(on.filter((m) => m.type === "ops-alerts")).toEqual([{ type: "ops-alerts", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "ops-alerts")).toEqual([ + { type: "ops-alerts", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue ops-alerts outside the hourly window even when LOOPOVER_REVIEW_OPS is ON", async () => { @@ -792,7 +1150,11 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:15:00.000Z"), env, executionContext(waitUntil)); // non-hourly + await worker.scheduled( + controllerFor("2026-05-25T05:15:00.000Z"), + env, + executionContext(waitUntil), + ); // non-hourly await Promise.all(waitUntil); expect(sent.some((m) => m.type === "ops-alerts")).toBe(false); }); @@ -807,11 +1169,19 @@ describe("worker entrypoint", () => { } as unknown as Queue, LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover", }); - await upsertRepoFocusManifest(env, "JSONbored/loopover", { ops: { enabled: true } }); + await upsertRepoFocusManifest(env, "JSONbored/loopover", { + ops: { enabled: true }, + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); - expect(sent.filter((m) => m.type === "ops-alerts")).toEqual([{ type: "ops-alerts", requestedBy: "schedule" }]); + expect(sent.filter((m) => m.type === "ops-alerts")).toEqual([ + { type: "ops-alerts", requestedBy: "schedule" }, + ]); }); it("a present ops manifest override suppresses ops-alerts hourly even when LOOPOVER_REVIEW_OPS is ON (#6275)", async () => { @@ -825,18 +1195,28 @@ describe("worker entrypoint", () => { } as unknown as Queue, LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover", }); - await upsertRepoFocusManifest(env, "JSONbored/loopover", { ops: { enabled: false } }); + await upsertRepoFocusManifest(env, "JSONbored/loopover", { + ops: { enabled: false }, + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); expect(sent.some((m) => m.type === "ops-alerts")).toBe(false); }); it("enqueues sync-brokered-installed-repos hourly ONLY in broker mode (ORB_ENROLLMENT_SECRET set, flag-OFF is byte-identical)", async () => { - const sentFor = async (enrollmentSecret?: string): Promise> => { + const sentFor = async ( + enrollmentSecret?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ - ...(enrollmentSecret === undefined ? {} : { ORB_ENROLLMENT_SECRET: enrollmentSecret }), + ...(enrollmentSecret === undefined + ? {} + : { ORB_ENROLLMENT_SECRET: enrollmentSecret }), JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); @@ -844,16 +1224,26 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // Non-brokered (default) → no sync job; the enqueued set is unchanged from today. - expect((await sentFor()).some((m) => m.type === "sync-brokered-installed-repos")).toBe(false); + expect( + (await sentFor()).some((m) => m.type === "sync-brokered-installed-repos"), + ).toBe(false); // Brokered (an enrollment secret is configured) → exactly one sync job, enqueued in the hourly window. const brokered = await sentFor("orbsec_x"); - expect(brokered.filter((m) => m.type === "sync-brokered-installed-repos")).toEqual([{ type: "sync-brokered-installed-repos", requestedBy: "schedule" }]); + expect( + brokered.filter((m) => m.type === "sync-brokered-installed-repos"), + ).toEqual([ + { type: "sync-brokered-installed-repos", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue sync-brokered-installed-repos outside the hourly window even in broker mode", async () => { @@ -867,16 +1257,26 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:15:00.000Z"), env, executionContext(waitUntil)); // non-hourly + await worker.scheduled( + controllerFor("2026-05-25T05:15:00.000Z"), + env, + executionContext(waitUntil), + ); // non-hourly await Promise.all(waitUntil); - expect(sent.some((m) => m.type === "sync-brokered-installed-repos")).toBe(false); + expect(sent.some((m) => m.type === "sync-brokered-installed-repos")).toBe( + false, + ); }); it("enqueues the sweep-liveness-watchdog job hourly ONLY when LOOPOVER_SWEEP_WATCHDOG is ON (flag-OFF is byte-identical)", async () => { - const sentFor = async (watchdogFlag?: string): Promise> => { + const sentFor = async ( + watchdogFlag?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ - ...(watchdogFlag === undefined ? {} : { LOOPOVER_SWEEP_WATCHDOG: watchdogFlag }), + ...(watchdogFlag === undefined + ? {} + : { LOOPOVER_SWEEP_WATCHDOG: watchdogFlag }), JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); @@ -884,21 +1284,35 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // Flag OFF (default) → no sweep-liveness-watchdog job; the enqueued set is unchanged from today. - expect((await sentFor()).some((m) => m.type === "sweep-liveness-watchdog")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "sweep-liveness-watchdog")).toBe(false); + expect( + (await sentFor()).some((m) => m.type === "sweep-liveness-watchdog"), + ).toBe(false); + expect( + (await sentFor("false")).some( + (m) => m.type === "sweep-liveness-watchdog", + ), + ).toBe(false); // Flag ON → exactly one sweep-liveness-watchdog job, enqueued in the hourly window. const on = await sentFor("true"); - expect(on.filter((m) => m.type === "sweep-liveness-watchdog")).toEqual([{ type: "sweep-liveness-watchdog", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "sweep-liveness-watchdog")).toEqual([ + { type: "sweep-liveness-watchdog", requestedBy: "schedule" }, + ]); }); it("enqueues the loop-escalation-sweep job hourly ONLY when LOOPOVER_LOOP_ESCALATION is ON (flag-OFF is byte-identical)", async () => { - const sentFor = async (flag?: string): Promise> => { + const sentFor = async ( + flag?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ ...(flag === undefined ? {} : { LOOPOVER_LOOP_ESCALATION: flag }), @@ -909,15 +1323,25 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; - expect((await sentFor()).some((m) => m.type === "loop-escalation-sweep")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "loop-escalation-sweep")).toBe(false); + expect( + (await sentFor()).some((m) => m.type === "loop-escalation-sweep"), + ).toBe(false); + expect( + (await sentFor("false")).some((m) => m.type === "loop-escalation-sweep"), + ).toBe(false); const on = await sentFor("true"); - expect(on.filter((m) => m.type === "loop-escalation-sweep")).toEqual([{ type: "loop-escalation-sweep", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "loop-escalation-sweep")).toEqual([ + { type: "loop-escalation-sweep", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue sweep-liveness-watchdog outside the hourly window even when LOOPOVER_SWEEP_WATCHDOG is ON", async () => { @@ -931,13 +1355,20 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:15:00.000Z"), env, executionContext(waitUntil)); // non-hourly + await worker.scheduled( + controllerFor("2026-05-25T05:15:00.000Z"), + env, + executionContext(waitUntil), + ); // non-hourly await Promise.all(waitUntil); expect(sent.some((m) => m.type === "sweep-liveness-watchdog")).toBe(false); }); it("enqueues the reconcile-open-prs job every 10 minutes ONLY when LOOPOVER_PR_RECONCILIATION is ON (flag-OFF is byte-identical)", async () => { - const sentFor = async (flag?: string, isoTime = "2026-05-25T05:10:00.000Z"): Promise> => { + const sentFor = async ( + flag?: string, + isoTime = "2026-05-25T05:10:00.000Z", + ): Promise> => { const sent: Array = []; const env = createTestEnv({ ...(flag === undefined ? {} : { LOOPOVER_PR_RECONCILIATION: flag }), @@ -948,17 +1379,27 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor(isoTime), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor(isoTime), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // Flag OFF (default) → no reconcile-open-prs job; the enqueued set is unchanged from today. - expect((await sentFor()).some((m) => m.type === "reconcile-open-prs")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "reconcile-open-prs")).toBe(false); + expect((await sentFor()).some((m) => m.type === "reconcile-open-prs")).toBe( + false, + ); + expect( + (await sentFor("false")).some((m) => m.type === "reconcile-open-prs"), + ).toBe(false); // Flag ON, on a 10-minute boundary → exactly one reconcile-open-prs job. const on = await sentFor("true"); - expect(on.filter((m) => m.type === "reconcile-open-prs")).toEqual([{ type: "reconcile-open-prs", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "reconcile-open-prs")).toEqual([ + { type: "reconcile-open-prs", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue reconcile-open-prs outside the 10-minute window even when LOOPOVER_PR_RECONCILIATION is ON", async () => { @@ -972,16 +1413,25 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); // not a 10-minute boundary + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); // not a 10-minute boundary await Promise.all(waitUntil); expect(sent.some((m) => m.type === "reconcile-open-prs")).toBe(false); }); it("enqueues the reconcile-active-review-tracking job every 10 minutes ONLY when LOOPOVER_ACTIVE_REVIEW_RECONCILIATION is ON (flag-OFF is byte-identical)", async () => { - const sentFor = async (flag?: string, isoTime = "2026-05-25T05:10:00.000Z"): Promise> => { + const sentFor = async ( + flag?: string, + isoTime = "2026-05-25T05:10:00.000Z", + ): Promise> => { const sent: Array = []; const env = createTestEnv({ - ...(flag === undefined ? {} : { LOOPOVER_ACTIVE_REVIEW_RECONCILIATION: flag }), + ...(flag === undefined + ? {} + : { LOOPOVER_ACTIVE_REVIEW_RECONCILIATION: flag }), JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); @@ -989,17 +1439,33 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor(isoTime), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor(isoTime), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // Flag OFF (default) → no reconcile-active-review-tracking job; the enqueued set is unchanged from today. - expect((await sentFor()).some((m) => m.type === "reconcile-active-review-tracking")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "reconcile-active-review-tracking")).toBe(false); + expect( + (await sentFor()).some( + (m) => m.type === "reconcile-active-review-tracking", + ), + ).toBe(false); + expect( + (await sentFor("false")).some( + (m) => m.type === "reconcile-active-review-tracking", + ), + ).toBe(false); // Flag ON, on a 10-minute boundary → exactly one reconcile-active-review-tracking job. const on = await sentFor("true"); - expect(on.filter((m) => m.type === "reconcile-active-review-tracking")).toEqual([{ type: "reconcile-active-review-tracking", requestedBy: "schedule" }]); + expect( + on.filter((m) => m.type === "reconcile-active-review-tracking"), + ).toEqual([ + { type: "reconcile-active-review-tracking", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue reconcile-active-review-tracking outside the 10-minute window even when LOOPOVER_ACTIVE_REVIEW_RECONCILIATION is ON", async () => { @@ -1013,38 +1479,80 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); // not a 10-minute boundary + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); // not a 10-minute boundary await Promise.all(waitUntil); - expect(sent.some((m) => m.type === "reconcile-active-review-tracking")).toBe(false); + expect( + sent.some((m) => m.type === "reconcile-active-review-tracking"), + ).toBe(false); }); it("enqueues federated-peer-sync every 10 minutes ONLY when the loopover self-repo manifest opts in (#9148) — absent-manifest default is byte-identical", async () => { - const sentFor = async (enabled?: boolean, isoTime = "2026-05-25T05:10:00.000Z"): Promise> => { + const sentFor = async ( + enabled?: boolean, + isoTime = "2026-05-25T05:10:00.000Z", + ): Promise> => { clearFederatedIntelligenceManifestOverrideCacheForTest(); // each sub-case below is a fresh env; don't let the prior case's cached override leak in const sent: Array = []; - const env = createTestEnv({ LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover", JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue }); - if (enabled !== undefined) await upsertRepoFocusManifest(env, "JSONbored/loopover", { federatedIntelligence: { enabled } }); + const env = createTestEnv({ + LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover", + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, + }); + if (enabled !== undefined) + await upsertRepoFocusManifest(env, "JSONbored/loopover", { + federatedIntelligence: { enabled }, + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor(isoTime), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor(isoTime), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // No manifest block at all (absent federatedIntelligence, the fleet-wide default) → never enqueued. - expect((await sentFor(undefined)).some((m) => m.type === "federated-peer-sync")).toBe(false); + expect( + (await sentFor(undefined)).some((m) => m.type === "federated-peer-sync"), + ).toBe(false); // Manifest present but explicitly disabled → still never enqueued. - expect((await sentFor(false)).some((m) => m.type === "federated-peer-sync")).toBe(false); + expect( + (await sentFor(false)).some((m) => m.type === "federated-peer-sync"), + ).toBe(false); // Opted in, on a 10-minute boundary → exactly one federated-peer-sync job. const on = await sentFor(true); - expect(on.filter((m) => m.type === "federated-peer-sync")).toEqual([{ type: "federated-peer-sync", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "federated-peer-sync")).toEqual([ + { type: "federated-peer-sync", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue federated-peer-sync outside the 10-minute window even when opted in", async () => { const sent: Array = []; - const env = createTestEnv({ LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover", JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue }); - await upsertRepoFocusManifest(env, "JSONbored/loopover", { federatedIntelligence: { enabled: true } }); + const env = createTestEnv({ + LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover", + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, + }); + await upsertRepoFocusManifest(env, "JSONbored/loopover", { + federatedIntelligence: { enabled: true }, + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); // not a 10-minute boundary + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); // not a 10-minute boundary await Promise.all(waitUntil); expect(sent.some((m) => m.type === "federated-peer-sync")).toBe(false); }); @@ -1078,16 +1586,20 @@ describe("worker entrypoint", () => { expect((await sentFor("false")).some((m) => m.type === "selftune")).toBe( false, ); - expect((await sentFor("true")).filter((m) => m.type === "selftune")).toEqual([ - { type: "selftune", requestedBy: "schedule" }, - ]); + expect( + (await sentFor("true")).filter((m) => m.type === "selftune"), + ).toEqual([{ type: "selftune", requestedBy: "schedule" }]); }); it("enqueues the satisfaction-floor-loosening tick hourly only when SATISFACTION_FLOOR_AUTOTUNE_ENABLED is ON (#8158)", async () => { - const sentFor = async (flag?: string): Promise> => { + const sentFor = async ( + flag?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ - ...(flag === undefined ? {} : ({ SATISFACTION_FLOOR_AUTOTUNE_ENABLED: flag } as Partial)), + ...(flag === undefined + ? {} + : ({ SATISFACTION_FLOOR_AUTOTUNE_ENABLED: flag } as Partial)), JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); @@ -1095,30 +1607,52 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // createTestEnv defaults the flag to "false" — the undefined case exercises exactly that shipped default. - expect((await sentFor()).some((m) => m.type === "satisfaction-floor-loosening")).toBe(false); - expect((await sentFor("true")).filter((m) => m.type === "satisfaction-floor-loosening")).toEqual([ + expect( + (await sentFor()).some((m) => m.type === "satisfaction-floor-loosening"), + ).toBe(false); + expect( + (await sentFor("true")).filter( + (m) => m.type === "satisfaction-floor-loosening", + ), + ).toEqual([ { type: "satisfaction-floor-loosening", requestedBy: "schedule" }, ]); // Off the hourly boundary, flag-ON still enqueues nothing. const sent: Array = []; const env = createTestEnv({ SATISFACTION_FLOOR_AUTOTUNE_ENABLED: "true" as never, - JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); - expect(sent.some((m) => m.type === "satisfaction-floor-loosening")).toBe(false); + expect(sent.some((m) => m.type === "satisfaction-floor-loosening")).toBe( + false, + ); }); it("enqueues the rag-index-repo fan-out in the full-sync window ONLY when LOOPOVER_REVIEW_RAG is ON (flag-OFF is byte-identical)", async () => { - const sentFor = async (ragFlag?: string): Promise> => { + const sentFor = async ( + ragFlag?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ ...(ragFlag === undefined ? {} : { LOOPOVER_REVIEW_RAG: ragFlag }), @@ -1129,17 +1663,27 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T06:00:00.000Z"), env, executionContext(waitUntil)); // full-sync window + await worker.scheduled( + controllerFor("2026-05-25T06:00:00.000Z"), + env, + executionContext(waitUntil), + ); // full-sync window await Promise.all(waitUntil); return sent; }; // Flag OFF (default) → no rag-index-repo job; the enqueued set is unchanged from today. - expect((await sentFor()).some((m) => m.type === "rag-index-repo")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "rag-index-repo")).toBe(false); + expect((await sentFor()).some((m) => m.type === "rag-index-repo")).toBe( + false, + ); + expect( + (await sentFor("false")).some((m) => m.type === "rag-index-repo"), + ).toBe(false); // Flag ON → exactly one rag-index-repo fan-out job, enqueued in the full-sync window. const on = await sentFor("true"); - expect(on.filter((m) => m.type === "rag-index-repo")).toEqual([{ type: "rag-index-repo", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "rag-index-repo")).toEqual([ + { type: "rag-index-repo", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue rag-index-repo outside the full-sync window even when LOOPOVER_REVIEW_RAG is ON", async () => { @@ -1153,7 +1697,11 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); // hourly but NOT full-sync + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); // hourly but NOT full-sync await Promise.all(waitUntil); expect(sent.some((m) => m.type === "rag-index-repo")).toBe(false); }); @@ -1169,10 +1717,18 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-06-01T09:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-06-01T09:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); - expect(sent).toEqual(expect.arrayContaining([{ type: "repo-doc-refresh-sweep", requestedBy: "schedule" }])); + expect(sent).toEqual( + expect.arrayContaining([ + { type: "repo-doc-refresh-sweep", requestedBy: "schedule" }, + ]), + ); }); it("does NOT enqueue the repo-doc refresh sweep outside the 09:00 UTC window", async () => { @@ -1186,7 +1742,11 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-06-01T10:00:00.000Z"), env, executionContext(waitUntil)); // hourly but not 09:00 + await worker.scheduled( + controllerFor("2026-06-01T10:00:00.000Z"), + env, + executionContext(waitUntil), + ); // hourly but not 09:00 await Promise.all(waitUntil); expect(sent.some((m) => m.type === "repo-doc-refresh-sweep")).toBe(false); @@ -1196,37 +1756,111 @@ describe("worker entrypoint", () => { const send = (over: Record, when: string) => { const sent: Array = []; const env = createTestEnv({ - JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, ...over, }); const waitUntil: Promise[] = []; - return worker.scheduled(controllerFor(when), env, executionContext(waitUntil)).then(() => Promise.all(waitUntil)).then(() => sent); + return worker + .scheduled(controllerFor(when), env, executionContext(waitUntil)) + .then(() => Promise.all(waitUntil)) + .then(() => sent); + }; + const selfhost = { + SELFHOST_TRANSIENT_CACHE: {} as never, + LOOPOVER_DECISION_AUDIT: "true", }; - const selfhost = { SELFHOST_TRANSIENT_CACHE: {} as never, LOOPOVER_DECISION_AUDIT: "true" }; // 2026-06-02 is a Tuesday. - expect((await send(selfhost, "2026-06-02T08:00:00.000Z")).some((m) => m.type === "decision-audit-sample")).toBe(true); + expect( + (await send(selfhost, "2026-06-02T08:00:00.000Z")).some( + (m) => m.type === "decision-audit-sample", + ), + ).toBe(true); // Wrong hour, wrong day, flag off, and cloud runtime each suppress the enqueue. - expect((await send(selfhost, "2026-06-02T09:00:00.000Z")).some((m) => m.type === "decision-audit-sample")).toBe(false); - expect((await send(selfhost, "2026-06-01T08:00:00.000Z")).some((m) => m.type === "decision-audit-sample")).toBe(false); - expect((await send({ ...selfhost, LOOPOVER_DECISION_AUDIT: "false" }, "2026-06-02T08:00:00.000Z")).some((m) => m.type === "decision-audit-sample")).toBe(false); - expect((await send({ LOOPOVER_DECISION_AUDIT: "true", SELFHOST_TRANSIENT_CACHE: undefined }, "2026-06-02T08:00:00.000Z")).some((m) => m.type === "decision-audit-sample")).toBe(false); + expect( + (await send(selfhost, "2026-06-02T09:00:00.000Z")).some( + (m) => m.type === "decision-audit-sample", + ), + ).toBe(false); + expect( + (await send(selfhost, "2026-06-01T08:00:00.000Z")).some( + (m) => m.type === "decision-audit-sample", + ), + ).toBe(false); + expect( + ( + await send( + { ...selfhost, LOOPOVER_DECISION_AUDIT: "false" }, + "2026-06-02T08:00:00.000Z", + ) + ).some((m) => m.type === "decision-audit-sample"), + ).toBe(false); + expect( + ( + await send( + { + LOOPOVER_DECISION_AUDIT: "true", + SELFHOST_TRANSIENT_CACHE: undefined, + }, + "2026-06-02T08:00:00.000Z", + ) + ).some((m) => m.type === "decision-audit-sample"), + ).toBe(false); }); it("#8835: enqueues the risk-control recalibration daily at 07:00 UTC, flag-ON, on a self-host", async () => { const send = (over: Record, when: string) => { const sent: Array = []; const env = createTestEnv({ - JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, ...over, }); const waitUntil: Promise[] = []; - return worker.scheduled(controllerFor(when), env, executionContext(waitUntil)).then(() => Promise.all(waitUntil)).then(() => sent); + return worker + .scheduled(controllerFor(when), env, executionContext(waitUntil)) + .then(() => Promise.all(waitUntil)) + .then(() => sent); }; - const selfhost = { SELFHOST_TRANSIENT_CACHE: {} as never, LOOPOVER_RISK_CONTROL: "true" }; - expect((await send(selfhost, "2026-06-03T07:00:00.000Z")).some((m) => m.type === "risk-control-recalibrate")).toBe(true); - expect((await send(selfhost, "2026-06-03T08:00:00.000Z")).some((m) => m.type === "risk-control-recalibrate")).toBe(false); - expect((await send({ ...selfhost, LOOPOVER_RISK_CONTROL: "0" }, "2026-06-03T07:00:00.000Z")).some((m) => m.type === "risk-control-recalibrate")).toBe(false); - expect((await send({ LOOPOVER_RISK_CONTROL: "true", SELFHOST_TRANSIENT_CACHE: undefined }, "2026-06-03T07:00:00.000Z")).some((m) => m.type === "risk-control-recalibrate")).toBe(false); + const selfhost = { + SELFHOST_TRANSIENT_CACHE: {} as never, + LOOPOVER_RISK_CONTROL: "true", + }; + expect( + (await send(selfhost, "2026-06-03T07:00:00.000Z")).some( + (m) => m.type === "risk-control-recalibrate", + ), + ).toBe(true); + expect( + (await send(selfhost, "2026-06-03T08:00:00.000Z")).some( + (m) => m.type === "risk-control-recalibrate", + ), + ).toBe(false); + expect( + ( + await send( + { ...selfhost, LOOPOVER_RISK_CONTROL: "0" }, + "2026-06-03T07:00:00.000Z", + ) + ).some((m) => m.type === "risk-control-recalibrate"), + ).toBe(false); + expect( + ( + await send( + { + LOOPOVER_RISK_CONTROL: "true", + SELFHOST_TRANSIENT_CACHE: undefined, + }, + "2026-06-03T07:00:00.000Z", + ) + ).some((m) => m.type === "risk-control-recalibrate"), + ).toBe(false); }); it("enqueues weekly value report generation during the Monday report window", async () => { @@ -1240,22 +1874,35 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-06-01T12:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-06-01T12:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); expect(sent).toEqual( expect.arrayContaining([ { type: "rollup-product-usage", requestedBy: "schedule", days: 7 }, - { type: "generate-weekly-value-report", requestedBy: "schedule", variant: "operator", days: 7 }, + { + type: "generate-weekly-value-report", + requestedBy: "schedule", + variant: "operator", + days: 7, + }, ]), ); }); it("enqueues the maintainer recap digest on the default weekly cadence (Monday 14:00 UTC) ONLY when LOOPOVER_MAINTAINER_RECAP is ON (#2248, flag-OFF is byte-identical)", async () => { - const sentFor = async (recapFlag?: string): Promise> => { + const sentFor = async ( + recapFlag?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ - ...(recapFlag === undefined ? {} : { LOOPOVER_MAINTAINER_RECAP: recapFlag }), + ...(recapFlag === undefined + ? {} + : { LOOPOVER_MAINTAINER_RECAP: recapFlag }), JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); @@ -1263,17 +1910,29 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-06-01T14:00:00.000Z"), env, executionContext(waitUntil)); // Monday, 14:00 UTC + await worker.scheduled( + controllerFor("2026-06-01T14:00:00.000Z"), + env, + executionContext(waitUntil), + ); // Monday, 14:00 UTC await Promise.all(waitUntil); return sent; }; // Flag OFF (default) → no recap job; the enqueued set is unchanged from today. - expect((await sentFor()).some((m) => m.type === "generate-maintainer-recap")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "generate-maintainer-recap")).toBe(false); + expect( + (await sentFor()).some((m) => m.type === "generate-maintainer-recap"), + ).toBe(false); + expect( + (await sentFor("false")).some( + (m) => m.type === "generate-maintainer-recap", + ), + ).toBe(false); // Flag ON, on the default weekly cadence tick → exactly one recap job. const on = await sentFor("true"); - expect(on.filter((m) => m.type === "generate-maintainer-recap")).toEqual([{ type: "generate-maintainer-recap", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "generate-maintainer-recap")).toEqual([ + { type: "generate-maintainer-recap", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue the maintainer recap digest outside its configured cadence even when LOOPOVER_MAINTAINER_RECAP is ON", async () => { @@ -1287,9 +1946,15 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-06-02T14:00:00.000Z"), env, executionContext(waitUntil)); // Tuesday, not the weekly default day + await worker.scheduled( + controllerFor("2026-06-02T14:00:00.000Z"), + env, + executionContext(waitUntil), + ); // Tuesday, not the weekly default day await Promise.all(waitUntil); - expect(sent.some((m) => m.type === "generate-maintainer-recap")).toBe(false); + expect(sent.some((m) => m.type === "generate-maintainer-recap")).toBe( + false, + ); }); it("honors a custom LOOPOVER_RECAP_CADENCE=daily, firing every day at the configured hour", async () => { @@ -1304,9 +1969,15 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-06-02T14:00:00.000Z"), env, executionContext(waitUntil)); // Tuesday — not the weekly default day + await worker.scheduled( + controllerFor("2026-06-02T14:00:00.000Z"), + env, + executionContext(waitUntil), + ); // Tuesday — not the weekly default day await Promise.all(waitUntil); - expect(sent.filter((m) => m.type === "generate-maintainer-recap")).toEqual([{ type: "generate-maintainer-recap", requestedBy: "schedule" }]); + expect(sent.filter((m) => m.type === "generate-maintainer-recap")).toEqual([ + { type: "generate-maintainer-recap", requestedBy: "schedule" }, + ]); }); }); diff --git a/test/unit/service-status-history.test.ts b/test/unit/service-status-history.test.ts new file mode 100644 index 0000000000..34a2eef18e --- /dev/null +++ b/test/unit/service-status-history.test.ts @@ -0,0 +1,235 @@ +import { describe, expect, it } from "vitest"; + +import { createApp } from "../../src/api/routes"; +import { RETENTION_POLICY, retentionDaysForTable } from "../../src/db/retention"; +import { loadServiceStatusSamples, recordServiceStatusSamples } from "../../src/selfhost/service-status-history"; +import { createTestEnv } from "../helpers/d1"; +import { + buildComponentHistory, + computeIncidents, + computeUptimeWindow, + isIncidentStatus, + UPTIME_WINDOW_DAYS, + type StatusSample, +} from "../../src/selfhost/service-status-history"; + +// #9985: uptime and incidents derived from status samples. +// +// Every test here is about a way a plausible implementation would publish something untrue: +// +// • a 30-day figure computed over three days of data, presented as if it covered a month; +// • `unknown` ticks — time we were BLIND — counted as uptime (inflates exactly when we could not see) or as +// downtime (invents an outage nobody observed); +// • an empty window rendering as 0% (reads as a total outage) or 100% (reads as perfect); +// • an `unknown` tick splitting one outage into two incidents, manufacturing incidents from our blindness; +// • an open incident silently closed at "now", asserting a recovery that was never observed. + +const NOW = Date.parse("2026-07-31T12:00:00.000Z"); +const at = (hoursAgo: number): string => new Date(NOW - hoursAgo * 3_600_000).toISOString(); +const sample = (status: StatusSample["status"], hoursAgo: number): StatusSample => ({ status, sampledAt: at(hoursAgo) }); + +describe("isIncidentStatus", () => { + it("counts degraded and outage, and deliberately not unknown", () => { + expect(isIncidentStatus("degraded")).toBe(true); + expect(isIncidentStatus("outage")).toBe(true); + // We did not observe a problem — we failed to look. + expect(isIncidentStatus("unknown")).toBe(false); + expect(isIncidentStatus("operational")).toBe(false); + }); +}); + +describe("computeUptimeWindow", () => { + it("is the operational share of MEASURED samples", () => { + const window = computeUptimeWindow([sample("operational", 1), sample("operational", 2), sample("outage", 3), sample("operational", 4)], 7, NOW); + expect(window.uptime).toBe(0.75); + expect(window.measured).toBe(4); + expect(window.unmeasured).toBe(0); + }); + + it("REGRESSION: unknown ticks are excluded from the percentage and reported separately", () => { + // Two operational, two blind. Counting the blind ticks as up gives 100% — a claim of perfect health over + // a period half of which we could not see. Counting them as down gives 50%, inventing an outage. + const window = computeUptimeWindow([sample("operational", 1), sample("unknown", 2), sample("unknown", 3), sample("operational", 4)], 7, NOW); + expect(window.uptime).toBe(1); + expect(window.measured).toBe(2); + expect(window.unmeasured).toBe(2); + }); + + it("REGRESSION: an empty window is null, never 0% or 100%", () => { + // 0% reads as a total outage and 100% reads as perfect. Both are claims; no data is not a claim. + const window = computeUptimeWindow([], 30, NOW); + expect(window.uptime).toBeNull(); + expect(window.measured).toBe(0); + expect(window.since).toBeNull(); + }); + + it("is null, not 100%, when every sample in the window was unmeasured", () => { + const window = computeUptimeWindow([sample("unknown", 1), sample("unknown", 2)], 7, NOW); + expect(window.uptime).toBeNull(); + expect(window.unmeasured).toBe(2); + }); + + it("REGRESSION: reports a window as PARTIAL when history does not reach back across it", () => { + // The headline honesty field. Three days of samples cannot support a 30-day claim, and the reader has to + // be able to tell. + const samples = [sample("operational", 1), sample("operational", 48)]; + expect(computeUptimeWindow(samples, 30, NOW).partial).toBe(true); + expect(computeUptimeWindow(samples, 30, NOW).since).toBe(at(48)); + // The SAME samples fully cover a 1-day window, because recording began 48h before now — well before that + // window starts. This is the case a naive implementation gets wrong: partiality read off the oldest + // IN-WINDOW sample marks every window partial forever, since in-window samples can never predate the + // window start. + const oneDay = computeUptimeWindow(samples, 1, NOW); + expect(oneDay.partial).toBe(false); + expect(oneDay.since).toBe(at(1)); + }); + + it("treats an empty window as partial — it covers none of the period it names", () => { + expect(computeUptimeWindow([], 7, NOW).partial).toBe(true); + }); + + it("ignores samples outside the window and unparseable timestamps", () => { + const window = computeUptimeWindow( + [sample("operational", 1), sample("outage", 24 * 40), { status: "outage", sampledAt: "not-a-date" }], + 7, + NOW, + ); + expect(window.measured).toBe(1); + expect(window.uptime).toBe(1); + }); +}); + +describe("computeIncidents", () => { + it("derives a closed incident from a contiguous run, with the recovery as its end", () => { + const incidents = computeIncidents([sample("operational", 5), sample("outage", 4), sample("outage", 3), sample("operational", 2)], 30, NOW); + expect(incidents).toEqual([{ status: "outage", startedAt: at(4), endedAt: at(2) }]); + }); + + it("REGRESSION: leaves an ongoing incident OPEN rather than closing it at now", () => { + // Closing it at `now` would assert a recovery nobody observed. + const incidents = computeIncidents([sample("operational", 5), sample("outage", 4), sample("outage", 1)], 30, NOW); + expect(incidents).toEqual([{ status: "outage", startedAt: at(4), endedAt: null }]); + }); + + it("REGRESSION: an unknown tick does not split one incident into two", () => { + // Our blindness is not a recovery. Splitting here would manufacture a second incident out of a failed + // read, and double the incident count on any night the alerting source was flaky. + const incidents = computeIncidents([sample("outage", 5), sample("unknown", 4), sample("outage", 3), sample("operational", 2)], 30, NOW); + expect(incidents).toHaveLength(1); + expect(incidents[0]).toMatchObject({ startedAt: at(5), endedAt: at(2) }); + }); + + it("an unknown tick alone never starts an incident", () => { + expect(computeIncidents([sample("operational", 3), sample("unknown", 2), sample("operational", 1)], 30, NOW)).toEqual([]); + }); + + it("takes the WORST status in a run, so an escalation is one incident and not two", () => { + const incidents = computeIncidents([sample("degraded", 4), sample("outage", 3), sample("operational", 2)], 30, NOW); + expect(incidents).toHaveLength(1); + expect(incidents[0]?.status).toBe("outage"); + }); + + it("separates genuinely distinct incidents", () => { + const incidents = computeIncidents( + [sample("outage", 8), sample("operational", 7), sample("degraded", 4), sample("operational", 3)], + 30, + NOW, + ); + expect(incidents.map((incident) => incident.status)).toEqual(["outage", "degraded"]); + }); + + it("sorts unordered input before folding, so row order cannot change the answer", () => { + const shuffled = [sample("operational", 2), sample("outage", 4), sample("operational", 5), sample("outage", 3)]; + expect(computeIncidents(shuffled, 30, NOW)).toEqual([{ status: "outage", startedAt: at(4), endedAt: at(2) }]); + }); + + it("returns nothing for an all-operational history", () => { + expect(computeIncidents([sample("operational", 2), sample("operational", 1)], 30, NOW)).toEqual([]); + }); +}); + +describe("buildComponentHistory", () => { + it("reports every declared window, and incidents over the widest one", () => { + const history = buildComponentHistory([sample("operational", 1), sample("outage", 24 * 20), sample("operational", 24 * 19)], NOW); + expect(history.uptime.map((w) => w.windowDays)).toEqual([...UPTIME_WINDOW_DAYS]); + // The 20-day-old outage is outside 1d and 7d but inside 30d, so only the widest window sees it — and the + // incident list is computed over that same widest window, so the two describe the same period. + expect(history.uptime.find((w) => w.windowDays === 1)?.measured).toBe(1); + expect(history.incidents).toHaveLength(1); + }); +}); + + +describe("persistence, over a real migrated D1", () => { + it("round-trips a sample per component", async () => { + const env = createTestEnv(); + const now = new Date(NOW); + await recordServiceStatusSamples(env, [{ component: "review", status: "operational" }, { component: "testing", status: "outage" }], now); + + const review = await loadServiceStatusSamples(env, "review", now); + expect(review).toEqual([{ status: "operational", sampledAt: now.toISOString() }]); + // Scoped per component: the testing outage must not appear in the review history. + expect((await loadServiceStatusSamples(env, "testing", now))[0]?.status).toBe("outage"); + }); + + it("stores `unknown` rather than skipping it, so blindness stays visible as unmeasured time", async () => { + // Dropping these rows would silently shrink the window and let a period we could not see read as uptime. + const env = createTestEnv(); + const now = new Date(NOW); + await recordServiceStatusSamples(env, [{ component: "review", status: "unknown" }], now); + const history = buildComponentHistory(await loadServiceStatusSamples(env, "review", now), NOW); + const day = history.uptime.find((window) => window.windowDays === 1)!; + expect(day.unmeasured).toBe(1); + expect(day.uptime).toBeNull(); + }); + + it("BEST EFFORT: a write failure never throws, because telemetry must not fail the tick carrying it", async () => { + const broken = createTestEnv(); + broken.DB = { prepare: () => { throw new Error("boom"); } } as never; + await expect(recordServiceStatusSamples(broken, [{ component: "review", status: "operational" }])).resolves.toBeUndefined(); + }); + + it("FAIL-SAFE: a read failure yields no samples, which reports as unmeasured rather than as healthy", async () => { + const broken = createTestEnv(); + broken.DB = { prepare: () => { throw new Error("boom"); } } as never; + expect(await loadServiceStatusSamples(broken, "review")).toEqual([]); + expect(buildComponentHistory([], NOW).uptime.every((window) => window.uptime === null && window.partial)).toBe(true); + }); + + it("excludes samples older than the widest reported window", async () => { + const env = createTestEnv(); + const now = new Date(NOW); + await recordServiceStatusSamples(env, [{ component: "review", status: "outage" }], new Date(NOW - 40 * 86_400_000)); + await recordServiceStatusSamples(env, [{ component: "review", status: "operational" }], now); + expect(await loadServiceStatusSamples(env, "review", now)).toHaveLength(1); + }); +}); + +describe("retention", () => { + it("INVARIANT: retention EXCEEDS the widest reported window, so the figure never loses its own tail", () => { + // A retention equal to the window would make the oldest day of every month quietly unmeasured. This is + // the inequality that lets the 30-day figure stay whole without a rollup -- and widening + // UPTIME_WINDOW_DAYS past it would break it silently, so it is asserted rather than commented. + const days = retentionDaysForTable("service_status_samples"); + expect(days).not.toBeNull(); + expect(days!).toBeGreaterThan(Math.max(...UPTIME_WINDOW_DAYS)); + }); + + it("is registered in the policy at all", () => { + expect(RETENTION_POLICY.some((rule) => rule.table === "service_status_samples")).toBe(true); + }); +}); + +describe("GET /v1/public/service-status with history", () => { + it("attaches uptime and incidents per component", async () => { + const env = createTestEnv({ LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" } as Partial); + await recordServiceStatusSamples(env, [{ component: "review", status: "operational" }], new Date()); + + const response = await createApp().request("/v1/public/service-status", {}, env); + expect(response.status).toBe(200); + const body = (await response.json()) as { components: { component: string; uptime: unknown[]; incidents: unknown[] }[] }; + const review = body.components.find((entry) => entry.component === "review"); + expect(review?.uptime).toHaveLength(UPTIME_WINDOW_DAYS.length); + expect(Array.isArray(review?.incidents)).toBe(true); + }); +});