diff --git a/migrations/0209_service_status_samples.sql b/migrations/0209_service_status_samples.sql new file mode 100644 index 0000000000..05c2a625ca --- /dev/null +++ b/migrations/0209_service_status_samples.sql @@ -0,0 +1,37 @@ +-- #9985 (slice 2 of #9747): the source for "has it been healthy?". +-- +-- /v1/public/service-status answers "is each component healthy RIGHT NOW" by reading Alertmanager's active +-- alerts. It cannot answer the past tense, because there is nothing to read: Alertmanager serves active +-- alerts only, resolved ones are gone, there is no Prometheus on the box (metrics flow through the +-- otel-collector), and nothing persisted a history. #9983 therefore published no uptime figure rather than +-- deriving one from a single live sample. +-- +-- This is that history: one row per component per cron tick. Uptime and incidents are DERIVED from these +-- rows rather than posted by hand, which is what #9747 means by "incidents appear without manual posting". +-- +-- `status` carries the same vocabulary the endpoint publishes -- operational | degraded | outage | unknown -- +-- and `unknown` is stored rather than skipped ON PURPOSE. A tick where the alerting source could not be read +-- is UNMEASURED time, not healthy time and not an outage; dropping those rows would silently shrink the +-- window and let a period of blindness read as uptime. Keeping them is what lets the read path report +-- coverage separately from the percentage. +-- +-- No UNIQUE on (component, sampled_at): two ticks in the same second are harmless to an aggregate over +-- contiguous runs, whereas a constraint violation on a best-effort telemetry write is not. +CREATE TABLE IF NOT EXISTS service_status_samples ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + component TEXT NOT NULL, + status TEXT NOT NULL, + sampled_at TEXT NOT NULL +); + +-- The read is always "this component, over this trailing window, in time order", which this covers exactly. +CREATE INDEX IF NOT EXISTS service_status_samples_component_time + ON service_status_samples (component, sampled_at); + +-- The retention prune deletes by age alone (`sampled_at < cutoff`, across every component), so it needs an +-- index LEADING with that column -- the composite above leads with `component` and cannot serve it. Both +-- exist because the two access patterns are genuinely different: the read is always per-component over a +-- window, the prune is always all-components before a cutoff. test/unit/retention.test.ts enforces the +-- latter for every table in RETENTION_POLICY, which is how the omission surfaced. +CREATE INDEX IF NOT EXISTS service_status_samples_sampled_at + ON service_status_samples (sampled_at); diff --git a/scripts/check-schema-drift.ts b/scripts/check-schema-drift.ts index 259540a7ce..21f579ed56 100644 --- a/scripts/check-schema-drift.ts +++ b/scripts/check-schema-drift.ts @@ -46,6 +46,10 @@ export const RAW_SQL_ONLY_TABLES: Set = new Set([ "decision_ledger_anchors", "decision_records", "decision_replay_inputs", + // #9985: written and read only through raw statements in selfhost/service-status-history.ts, the same + // posture as the decision_* tables above. Declaring it in the drizzle schema would add an export nothing + // imports (dead-exports:check rejects that, correctly) purely to satisfy this checker. + "service_status_samples", // #9028: same raw-SQL-only posture as its sibling above — an operator-private blob table the drizzle // schema never queries (writes go through persistDecisionReplayPrompt's raw statement; reads happen only // in the operator's own extract SQL for the replay CLI). diff --git a/src/api/routes.ts b/src/api/routes.ts index 1b0ab81da5..33025838bd 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -309,6 +309,7 @@ import { loadDecisionLedgerTip, loadPublicDecisionRecord, loadPublicLedgerRow, v import { buildEvalScoreRecordsFromRulePrecision, filterEvalScoreRecords } from "../review/eval-score-records"; import { buildPublicCorpusCommitments } from "../review/public-eval-corpus"; import { isServiceStatusEnabled, loadServiceStatus } from "../selfhost/service-status"; +import { buildComponentHistory, loadServiceStatusSamples } from "../selfhost/service-status-history"; import { anchorSigningInput, buildLedgerAnchorPayload, currentAnchorKey, diagnoseAnchorPublicKeys, parseAnchorPublicKeys, publicAnchorStatus, signLedgerAnchorPayload } from "../review/ledger-anchor"; import { resolveProofPage } from "../review/proof-summary"; import { renderProofBadgeSvg } from "./proof-badge"; @@ -929,10 +930,17 @@ export function createApp() { app.get("/v1/public/service-status", async (c) => { if (!isServiceStatusEnabled(c.env)) return c.json({ error: "not_found" }, 404); const status = await loadServiceStatus(c.env); + // #9985: the past tense, derived from persisted samples rather than posted by hand. Attached per + // component so a reader never has to correlate two lists, and computed from the SAME status vocabulary + // the live board publishes. + const now = new Date(); + const components = await Promise.all( + status.components.map(async (entry) => ({ ...entry, ...buildComponentHistory(await loadServiceStatusSamples(c.env, entry.component, now), now.getTime()) })), + ); // Shorter than the sibling public surfaces on purpose: this is the endpoint people refresh DURING an // incident, and a 60s cache would keep serving "operational" for a minute after an outage started. c.header("Cache-Control", "public, max-age=15, stale-while-revalidate=30"); - return c.json(status); + return c.json({ ...status, components }); }); app.get("/v1/public/eval-scores", async (c) => { diff --git a/src/db/retention.ts b/src/db/retention.ts index cc5acd5350..16c9fb0287 100644 --- a/src/db/retention.ts +++ b/src/db/retention.ts @@ -22,6 +22,12 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [ // needs would be gone by the time the fold ran, and the rollup would permanently over-count exactly the // PRs the live query never counted. { table: "orb_pr_outcomes", column: "occurred_at", days: 90 }, + // #9985: status samples are high-volume (one row per component per ~2-min tick) and low-value once the + // window that reads them has passed. 35 days deliberately EXCEEDS the widest reported window (30), so the + // 30-day uptime figure never silently loses its own tail to the prune -- a retention equal to the window + // would make the oldest day of every month quietly unmeasured. No rollup is needed while that inequality + // holds; widening UPTIME_WINDOW_DAYS past 35 without widening this would break it. + { table: "service_status_samples", column: "sampled_at", days: 35 }, // #9783: orb_signals had no rule at all and grew without bound, while being a PUBLIC data source // (computeFleetAnalytics' /fairness headline, and #9775's weekly fleet trend). It could not simply be // pruned: listFleetInstances reports a LIFETIME signalCount, and /v1/internal/fleet/analytics takes an @@ -127,6 +133,7 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [ // not scan-optimal, which is acceptable for the lower row counts of the tables that fall back today. export const RETENTION_PK_COLUMN: Readonly> = { audit_events: "id", + service_status_samples: "id", // #9783: orb_signals has `id INTEGER PRIMARY KEY` (migrations/0060), so it maps cleanly here rather than // joining RETENTION_COMPOSITE_PK_TABLES -- its UNIQUE (instance_id, repo_hash, pr_hash) is the ingest // dedup key, not its primary key. The fold below uses its own bounded-slice path rather than the generic diff --git a/src/index.ts b/src/index.ts index 7b6777a85c..86f071072c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -143,6 +143,10 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController): // impossible. The job's own decideLedgerAnchorSchedule (in ledger-anchor-scheduler.ts) is what actually // decides whether THIS tick does anything; a cheap two-query no-op the overwhelming majority of ticks. jobs.push({ type: "anchor-decision-ledger", requestedBy: "schedule", isHourly }); + // #9985: sampled every tick, unconditionally. The board it feeds is only as good as its resolution, and a + // deployment with no alerting source configured simply records `unknown` -- which the aggregation reports + // as unmeasured coverage rather than as health. + jobs.push({ type: "sample-service-status", requestedBy: "schedule" }); const selfHostedReviews = isSelfHostedReviewRuntime(env); const queueSnapshot = selfHostedReviews ? await queueSnapshotFromBinding(env.JOBS).catch((error) => { diff --git a/src/queue/job-dispatch.ts b/src/queue/job-dispatch.ts index dd09bddfb4..796f20bc79 100644 --- a/src/queue/job-dispatch.ts +++ b/src/queue/job-dispatch.ts @@ -53,6 +53,8 @@ import { withInstallationTokenRetry } from "../github/app"; import { runScheduledLedgerAnchor, resolveGitAnchorTarget } from "../review/ledger-anchor-scheduler"; import { submitToGitAnchor } from "../review/ledger-anchor-git"; import { incr } from "../selfhost/metrics"; +import { loadServiceStatus } from "../selfhost/service-status"; +import { recordServiceStatusSamples } from "../selfhost/service-status-history"; import { generateSignalSnapshots } from "./signal-snapshot"; import { isDecisionAuditEnabled, runDecisionAuditSample } from "../review/decision-audit"; import { isRiskControlEnabled, runRiskControlRecalibration } from "../review/risk-control-wire"; @@ -97,6 +99,18 @@ export async function processJob(env: Env, message: JobMessage): Promise { case "refresh-registry": await refreshRegistry(env); return; + case "sample-service-status": { + // #9985: read the live board through the SAME loader the public route uses, then persist one row per + // component. Reusing the loader is what keeps the history and the live status the same measurement -- + // a second read path here could report a component healthy while the endpoint called it degraded, and + // the derived uptime would then describe a board nobody ever saw. + const status = await loadServiceStatus(env); + await recordServiceStatusSamples( + env, + status.components.map((entry) => ({ component: entry.component, status: entry.status })), + ); + return; + } case "anchor-decision-ledger": { // Git anchoring runs only when BOTH the target (owner/repo, #9273) and an installation id (to mint a // token through) are configured -- unset means that backend simply isn't wired up yet; Rekor still diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index fd734ec9ad..fa3b4b66f1 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -801,6 +801,9 @@ const IMMEDIATE_SCHEDULED_JOB_TYPES = new Set([ // must fire independent of the hourly clock), and is a cheap 1-2 query no-op the overwhelming majority of // ticks -- not a heavy per-repo fan-out parent, so it needs no phase-spreading either. "anchor-decision-ledger", + // #9985: a handful of INSERTs, no fan-out. Phase-spreading it would jitter the sample cadence, and the + // derived uptime reads evenly-spaced samples as evenly-weighted time. + "sample-service-status", ]); export function scheduledEnqueueDelaySeconds(jobType: string): number { diff --git a/src/selfhost/service-status-history.ts b/src/selfhost/service-status-history.ts new file mode 100644 index 0000000000..605694b3af --- /dev/null +++ b/src/selfhost/service-status-history.ts @@ -0,0 +1,187 @@ +// Uptime and incident history for the public status board (#9985, slice 2 of #9747). +// +// #9983 publishes the live board and deliberately published no uptime figure, because there was no source +// for one: Alertmanager serves ACTIVE alerts only, resolved ones are gone, and nothing persisted a history. +// This is that source -- one sample per component per cron tick -- and the aggregation over it. +// +// ── THE WINDOW MUST NOT LIE ABOUT ITSELF ───────────────────────────────────────────────────────────── +// A "30-day uptime" computed over three days of samples is the same class of falsehood as a verifier that +// reports green because it checked nothing. So every window reports its COVERAGE -- how much of the window +// actually has samples -- alongside the percentage, and `since` says when history begins. A reader can then +// tell "99.9% over a full month" from "99.9% over the two days we have been recording". +// +// ── UNMEASURED TIME IS NOT UPTIME, AND NOT DOWNTIME ────────────────────────────────────────────────── +// An `unknown` sample means the alerting source could not be read on that tick. It is excluded from the +// percentage entirely rather than counted either way: counting it as up inflates the figure exactly when we +// were blind, and counting it as down invents an outage nobody observed. It is reported as its own number so +// the blindness is visible instead of averaged away. + +/** The status vocabulary the live board publishes, mirrored here because samples store it verbatim. */ +export type SampledStatus = "operational" | "degraded" | "outage" | "unknown"; + +export type StatusSample = { status: SampledStatus; sampledAt: string }; + +/** A contiguous run of non-operational, non-unknown samples: an incident, derived rather than posted. + * `endedAt` is null while the run reaches the newest sample -- the incident is still open. */ +export type StatusIncident = { status: "degraded" | "outage"; startedAt: string; endedAt: string | null }; + +export type UptimeWindow = { + windowDays: number; + /** Share of MEASURED samples that were operational, 0..1. Null when nothing was measured -- no data must + * render as no claim, never as 0% (which reads as a total outage) or 100% (which reads as perfect). */ + uptime: number | null; + /** Samples that carried a real reading. The denominator of `uptime`. */ + measured: number; + /** Samples where the source could not be read. Reported, never folded into `uptime`. */ + unmeasured: number; + /** Oldest sample in the window, or null when the window is empty. What makes a partial window legible. */ + since: string | null; + /** True when history does not reach back across the whole window, so the figure covers less than it says. */ + partial: boolean; +}; + +/** PURE. Is this a state a reader should see as an incident? `unknown` is deliberately NOT one -- we did not + * observe a problem, we failed to look. */ +export function isIncidentStatus(status: SampledStatus): status is "degraded" | "outage" { + return status === "degraded" || status === "outage"; +} + +/** + * PURE. Uptime over a trailing window, with the honesty fields that make it readable. + * + * `samples` may be in any order and may include rows outside the window; both are filtered here so the + * caller's query does not have to be the thing that is correct. + */ +export function computeUptimeWindow(samples: readonly StatusSample[], windowDays: number, nowMs: number): UptimeWindow { + const startMs = nowMs - windowDays * 86_400_000; + const inWindow = samples.filter((sample) => { + const at = Date.parse(sample.sampledAt); + return Number.isFinite(at) && at >= startMs && at <= nowMs; + }); + const measuredSamples = inWindow.filter((sample) => sample.status !== "unknown"); + const operational = measuredSamples.filter((sample) => sample.status === "operational").length; + const since = inWindow.length === 0 ? null : inWindow.reduce((oldest, s) => (s.sampledAt < oldest ? s.sampledAt : oldest), inWindow[0]!.sampledAt); + // Partiality is decided by the oldest sample OVERALL, not the oldest one inside the window. `inWindow` is + // filtered to `>= startMs`, so an in-window `since` can never predate the window start -- deriving + // partiality from it would mark every window partial forever, including ones with years of history behind + // them. What actually answers "does history reach back across this window" is whether recording began + // before the window did. + const oldestOverall = samples + .map((entry) => Date.parse(entry.sampledAt)) + .filter((ms) => Number.isFinite(ms)) + .reduce((oldest, ms) => (oldest === null || ms < oldest ? ms : oldest), null); + return { + windowDays, + // Guard the denominator, else null -- the same discipline every other published rate here uses. + uptime: measuredSamples.length > 0 ? operational / measuredSamples.length : null, + measured: measuredSamples.length, + unmeasured: inWindow.length - measuredSamples.length, + since, + // An empty history is partial too: it covers none of the period it names. + partial: oldestOverall === null || oldestOverall > startMs, + }; +} + +/** + * PURE. Incidents as contiguous runs of non-operational, non-unknown samples. + * + * DERIVED, NEVER POSTED -- that is #9747's "incidents appear without manual posting". An `unknown` sample + * does NOT close an incident and does not start one: a tick where we could not read the source is no evidence + * either way, and letting it split one outage into two would manufacture incidents out of our own blindness. + * A run's severity is the WORST status within it, so a degradation that escalates reports as an outage rather + * than as two adjacent incidents. + */ +export function computeIncidents(samples: readonly StatusSample[], windowDays: number, nowMs: number): StatusIncident[] { + const startMs = nowMs - windowDays * 86_400_000; + const ordered = samples + .filter((sample) => { + const at = Date.parse(sample.sampledAt); + return Number.isFinite(at) && at >= startMs && at <= nowMs; + }) + .sort((a, b) => a.sampledAt.localeCompare(b.sampledAt)); + + const incidents: StatusIncident[] = []; + let open: StatusIncident | null = null; + for (const sample of ordered) { + if (isIncidentStatus(sample.status)) { + if (open === null) open = { status: sample.status, startedAt: sample.sampledAt, endedAt: null }; + else if (sample.status === "outage") open.status = "outage"; + continue; + } + // `unknown` is not evidence of recovery, so only an `operational` sample closes a run. + if (sample.status === "operational" && open !== null) { + open.endedAt = sample.sampledAt; + incidents.push(open); + open = null; + } + } + // Still open at the newest sample: reported with a null end rather than closed at "now", which would + // assert a recovery that was never observed. + if (open !== null) incidents.push(open); + return incidents; +} + +/** The windows the public board reports. Fixed rather than caller-supplied: a status page whose windows vary + * per request is not comparable to itself. */ +export const UPTIME_WINDOW_DAYS: readonly number[] = [1, 7, 30]; + +export type ComponentHistory = { uptime: UptimeWindow[]; incidents: StatusIncident[] }; + +/** PURE. Everything the board publishes about one component's past. */ +export function buildComponentHistory(samples: readonly StatusSample[], nowMs: number): ComponentHistory { + return { + uptime: UPTIME_WINDOW_DAYS.map((days) => computeUptimeWindow(samples, days, nowMs)), + // Over the widest window, so the incident list and the longest uptime figure describe the same period. + incidents: computeIncidents(samples, Math.max(...UPTIME_WINDOW_DAYS), nowMs), + }; +} + +// ── PERSISTENCE ─────────────────────────────────────────────────────────────────────────────────────── +// Kept in this module rather than a sibling so the write shape and the read shape are one file apart: the +// sampler stores exactly the vocabulary the aggregation above reads, and a change to one is visibly a change +// to the other. + +/** BEST EFFORT. Record one sample per component. A telemetry write must never fail the tick that carries it, + * and a missed sample is already representable -- it simply widens the gap the coverage fields report. */ +export async function recordServiceStatusSamples( + env: Env, + components: readonly { component: string; status: SampledStatus }[], + now: Date = new Date(), +): Promise { + const sampledAt = now.toISOString(); + // try/catch around the whole loop, not `.catch()` on the promise: `prepare()` throws SYNCHRONOUSLY when the + // binding is unusable, so a promise-only guard lets that escape and fail the tick this write is riding on -- + // exactly what "best effort" is supposed to prevent. Caught once around the loop rather than per row because + // a binding that fails for one component fails for all of them, and a partial sample set is worse than none: + // it would look like a real reading of only some components. + try { + for (const entry of components) { + await env.DB.prepare(`INSERT INTO service_status_samples (component, status, sampled_at) VALUES (?, ?, ?)`) + .bind(entry.component, entry.status, sampledAt) + .run(); + } + } catch { + // A missed sample is already representable: it simply widens the gap the coverage fields report. + } +} + +/** + * Load one component's samples over the widest reported window. + * + * FAIL-SAFE, like every other read behind this public surface: a read error yields no samples, which the + * aggregation reports as an empty, PARTIAL window with a null uptime -- "we cannot say", never a fabricated + * figure and never a thrown public endpoint. + */ +export async function loadServiceStatusSamples(env: Env, component: string, now: Date = new Date()): Promise { + const sinceIso = new Date(now.getTime() - Math.max(...UPTIME_WINDOW_DAYS) * 86_400_000).toISOString(); + try { + const result = await env.DB.prepare( + `SELECT status, sampled_at FROM service_status_samples WHERE component = ? AND sampled_at >= ? ORDER BY sampled_at`, + ) + .bind(component, sinceIso) + .all<{ status: string; sampled_at: string }>(); + return (result.results ?? []).map((row) => ({ status: row.status as SampledStatus, sampledAt: row.sampled_at })); + } catch { + return []; + } +} diff --git a/src/types.ts b/src/types.ts index 9ec45be661..43a8892a79 100644 --- a/src/types.ts +++ b/src/types.ts @@ -64,6 +64,14 @@ export type JobMessage = requestedBy: "schedule" | "test"; isHourly: boolean; } + | { + // #9985 (slice 2 of #9747): one status sample per component per tick, so the public board can answer + // "has it been healthy?" as well as "is it now". A single INSERT per component -- no fan-out, no + // GitHub budget -- which is why it rides every tick rather than the hourly clock: coarser sampling + // would make short incidents invisible to the derived history. + type: "sample-service-status"; + requestedBy: "schedule" | "test"; + } | { type: "sync-brokered-installed-repos"; requestedBy: "schedule" | "api" | "test"; diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index a9caac5793..576ee83870 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -19,7 +19,10 @@ describe("worker entrypoint", () => { it("delegates fetch requests to the Hono app", async () => { const env = createTestEnv(); - const response = await worker.fetch(new Request("https://loopover.test/health"), env); + const response = await worker.fetch( + new Request("https://loopover.test/health"), + env, + ); expect(response.status).toBe(200); }); @@ -32,7 +35,12 @@ describe("worker entrypoint", () => { messages: [ { id: "dlq-msg-1", - body: { type: "github-webhook", deliveryId: "d-dlq", eventName: "pull_request", payload: {} }, + body: { + type: "github-webhook", + deliveryId: "d-dlq", + eventName: "pull_request", + payload: {}, + }, ack: () => acked.push("dlq-msg-1"), retry: () => retried.push("dlq-msg-1"), }, @@ -54,7 +62,13 @@ describe("worker entrypoint", () => { messages: [ { id: "wh-dlq-1", - body: { type: "github-webhook", deliveryId: "d-wh-dlq", eventName: "pull_request", payload: {}, redriven: true }, + body: { + type: "github-webhook", + deliveryId: "d-wh-dlq", + eventName: "pull_request", + payload: {}, + redriven: true, + }, ack: () => acked.push("wh-dlq-1"), retry: () => retried.push("wh-dlq-1"), }, @@ -71,14 +85,22 @@ describe("worker entrypoint", () => { const env = createTestEnv(); delete env.SELFHOST_TRANSIENT_CACHE; const sent: import("../../src/types").JobMessage[] = []; - env.WEBHOOKS = { send: async (message: import("../../src/types").JobMessage) => void sent.push(message) } as unknown as Queue; + env.WEBHOOKS = { + send: async (message: import("../../src/types").JobMessage) => + void sent.push(message), + } as unknown as Queue; const acked: string[] = []; const batch = { queue: "loopover-webhooks-dlq", messages: [ { id: "wh-dlq-broker-only", - body: { type: "github-webhook", deliveryId: "d-broker-only", eventName: "pull_request", payload: {} }, + body: { + type: "github-webhook", + deliveryId: "d-broker-only", + eventName: "pull_request", + payload: {}, + }, ack: () => acked.push("wh-dlq-broker-only"), retry: () => undefined, }, @@ -94,14 +116,21 @@ describe("worker entrypoint", () => { it("acks and ignores stale review-execution jobs from a broker-only Cloudflare runtime", async () => { const env = createTestEnv(); delete env.SELFHOST_TRANSIENT_CACHE; - const warned = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const warned = vi + .spyOn(console, "warn") + .mockImplementation(() => undefined); const acked: string[] = []; const retried: string[] = []; const batch = { messages: [ { id: "hosted-review-job", - body: { type: "github-webhook", deliveryId: "d-hosted-review", eventName: "pull_request", payload: {} }, + body: { + type: "github-webhook", + deliveryId: "d-hosted-review", + eventName: "pull_request", + payload: {}, + }, ack: () => acked.push("hosted-review-job"), retry: () => retried.push("hosted-review-job"), }, @@ -120,7 +149,10 @@ describe("worker entrypoint", () => { it("acks successful queue messages and retries failed messages", async () => { const env = createTestEnv(); - vi.stubGlobal("fetch", async () => new Response("missing", { status: 404 })); + vi.stubGlobal( + "fetch", + async () => new Response("missing", { status: 404 }), + ); const acked: string[] = []; const retried: string[] = []; const batch = { @@ -149,8 +181,20 @@ describe("worker entrypoint", () => { vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(new Date("2026-06-24T12:00:00.000Z")); const env = createTestEnv(); - await recordGitHubRateLimitObservation(env, { repoFullName: "owner/repo", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 5, resetAt: "2026-06-24T12:30:00.000Z", observedAt: "2026-06-24T12:00:00.000Z" }); - vi.stubGlobal("fetch", async () => new Response("missing", { status: 404 })); + await recordGitHubRateLimitObservation(env, { + repoFullName: "owner/repo", + resource: "rest", + path: "/x", + statusCode: 200, + limitValue: 5000, + remaining: 5, + resetAt: "2026-06-24T12:30:00.000Z", + observedAt: "2026-06-24T12:00:00.000Z", + }); + vi.stubGlobal( + "fetch", + async () => new Response("missing", { status: 404 }), + ); const retries: Array<{ delaySeconds?: number } | undefined> = []; const batch = { messages: [ @@ -175,20 +219,47 @@ describe("worker entrypoint", () => { vi.setSystemTime(new Date("2026-06-24T12:00:00.000Z")); const env = createTestEnv(); // Scoped to this job's own installation bucket (#audit-rate-scoping) — installationId 123 below. - await recordGitHubRateLimitObservation(env, { repoFullName: "owner/repo", admissionKey: "installation:123", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 120, resetAt: "2026-06-24T12:10:00.000Z", observedAt: "2026-06-24T12:00:00.000Z" }); + await recordGitHubRateLimitObservation(env, { + repoFullName: "owner/repo", + admissionKey: "installation:123", + resource: "rest", + path: "/x", + statusCode: 200, + limitValue: 5000, + remaining: 120, + resetAt: "2026-06-24T12:10:00.000Z", + observedAt: "2026-06-24T12:00:00.000Z", + }); const acked: string[] = []; const retries: Array<{ delaySeconds?: number } | undefined> = []; - const requeued: Array<{ message: import("../../src/types").JobMessage; delaySeconds?: number }> = []; + const requeued: Array<{ + message: import("../../src/types").JobMessage; + delaySeconds?: number; + }> = []; env.JOBS = { - async send(message: import("../../src/types").JobMessage, options?: { delaySeconds?: number }) { - requeued.push({ message, ...(options?.delaySeconds === undefined ? {} : { delaySeconds: options.delaySeconds }) }); + async send( + message: import("../../src/types").JobMessage, + options?: { delaySeconds?: number }, + ) { + requeued.push({ + message, + ...(options?.delaySeconds === undefined + ? {} + : { delaySeconds: options.delaySeconds }), + }); }, } as unknown as Queue; const batch = { messages: [ { id: "background-regate", - body: { type: "agent-regate-pr", deliveryId: "sweep:owner/repo#7", repoFullName: "owner/repo", prNumber: 7, installationId: 123 }, + body: { + type: "agent-regate-pr", + deliveryId: "sweep:owner/repo#7", + repoFullName: "owner/repo", + prNumber: 7, + installationId: 123, + }, ack: () => acked.push("background-regate"), retry: (options?: { delaySeconds?: number }) => retries.push(options), }, @@ -247,13 +318,33 @@ describe("worker entrypoint", () => { const env = createTestEnv(); // reconcile-open-prs has no per-installation field, so it draws from the SAME shared (no-admissionKey) // observation refresh-registry's own equivalent test above uses. - await recordGitHubRateLimitObservation(env, { repoFullName: "owner/repo", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 5, resetAt: "2026-06-24T12:30:00.000Z", observedAt: "2026-06-24T12:00:00.000Z" }); + await recordGitHubRateLimitObservation(env, { + repoFullName: "owner/repo", + resource: "rest", + path: "/x", + statusCode: 200, + limitValue: 5000, + remaining: 5, + resetAt: "2026-06-24T12:30:00.000Z", + observedAt: "2026-06-24T12:00:00.000Z", + }); const acked: string[] = []; const retries: Array<{ delaySeconds?: number } | undefined> = []; - const requeued: Array<{ message: import("../../src/types").JobMessage; delaySeconds?: number }> = []; + const requeued: Array<{ + message: import("../../src/types").JobMessage; + delaySeconds?: number; + }> = []; env.JOBS = { - async send(message: import("../../src/types").JobMessage, options?: { delaySeconds?: number }) { - requeued.push({ message, ...(options?.delaySeconds === undefined ? {} : { delaySeconds: options.delaySeconds }) }); + async send( + message: import("../../src/types").JobMessage, + options?: { delaySeconds?: number }, + ) { + requeued.push({ + message, + ...(options?.delaySeconds === undefined + ? {} + : { delaySeconds: options.delaySeconds }), + }); }, } as unknown as Queue; const batch = { @@ -274,7 +365,12 @@ describe("worker entrypoint", () => { // budget would have been silently ignored and runOpenPrReconciliation would have run immediately. expect(acked).toEqual(["reconcile-tick"]); expect(retries).toEqual([]); - expect(requeued).toEqual([{ message: { type: "reconcile-open-prs", requestedBy: "schedule" }, delaySeconds: 900 }]); // delayUntil clamps to [30, 900] + expect(requeued).toEqual([ + { + message: { type: "reconcile-open-prs", requestedBy: "schedule" }, + delaySeconds: 900, + }, + ]); // delayUntil clamps to [30, 900] vi.useRealTimers(); }); @@ -285,7 +381,17 @@ describe("worker entrypoint", () => { // refresh-installation-health carries no installationId (it loops over every installation internally), so // githubRateLimitAdmissionKeyForJob resolves it to `null` -- its dequeue-time check reads the unscoped // (admissionKey: undefined) latest-observations window, which any recent exhausted REST observation trips. - await recordGitHubRateLimitObservation(env, { repoFullName: null, admissionKey: "installation:1", resource: "rest", path: "/app/installations/1", statusCode: 200, limitValue: 5000, remaining: 5, resetAt: "2026-06-24T12:30:00.000Z", observedAt: "2026-06-24T12:00:00.000Z" }); + await recordGitHubRateLimitObservation(env, { + repoFullName: null, + admissionKey: "installation:1", + resource: "rest", + path: "/app/installations/1", + statusCode: 200, + limitValue: 5000, + remaining: 5, + resetAt: "2026-06-24T12:30:00.000Z", + observedAt: "2026-06-24T12:00:00.000Z", + }); const fetchCalls: string[] = []; vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { fetchCalls.push(input.toString()); @@ -293,17 +399,31 @@ describe("worker entrypoint", () => { }); const acked: string[] = []; const retries: Array<{ delaySeconds?: number } | undefined> = []; - const requeued: Array<{ message: import("../../src/types").JobMessage; delaySeconds?: number }> = []; + const requeued: Array<{ + message: import("../../src/types").JobMessage; + delaySeconds?: number; + }> = []; env.JOBS = { - async send(message: import("../../src/types").JobMessage, options?: { delaySeconds?: number }) { - requeued.push({ message, ...(options?.delaySeconds === undefined ? {} : { delaySeconds: options.delaySeconds }) }); + async send( + message: import("../../src/types").JobMessage, + options?: { delaySeconds?: number }, + ) { + requeued.push({ + message, + ...(options?.delaySeconds === undefined + ? {} + : { delaySeconds: options.delaySeconds }), + }); }, } as unknown as Queue; const batch = { messages: [ { id: "installation-health-tick", - body: { type: "refresh-installation-health", requestedBy: "schedule" }, + body: { + type: "refresh-installation-health", + requestedBy: "schedule", + }, ack: () => acked.push("installation-health-tick"), retry: (options?: { delaySeconds?: number }) => retries.push(options), }, @@ -318,7 +438,15 @@ describe("worker entrypoint", () => { expect(fetchCalls).toEqual([]); expect(acked).toEqual(["installation-health-tick"]); expect(retries).toEqual([]); - expect(requeued).toEqual([{ message: { type: "refresh-installation-health", requestedBy: "schedule" }, delaySeconds: 900 }]); // delayUntil clamps to [30, 900] + expect(requeued).toEqual([ + { + message: { + type: "refresh-installation-health", + requestedBy: "schedule", + }, + delaySeconds: 900, + }, + ]); // delayUntil clamps to [30, 900] vi.useRealTimers(); }); @@ -327,22 +455,22 @@ describe("worker entrypoint", () => { vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { const url = input.toString(); if (url.includes("master_repositories.json")) return Response.json({}); - if (url.includes("api.gittensor.io") || url.includes("mirror.gittensor.io")) return new Response("missing", { status: 404 }); + if ( + url.includes("api.gittensor.io") || + url.includes("mirror.gittensor.io") + ) + return new Response("missing", { status: 404 }); return Response.json([]); }); const waitUntil: Promise[] = []; - await worker.scheduled( - {} as ScheduledController, - env, - { - waitUntil: (promise: Promise) => { - waitUntil.push(promise); - }, - passThroughOnException: () => {}, - exports: {}, - props: {}, - } as unknown as ExecutionContext, - ); + await worker.scheduled({} as ScheduledController, env, { + waitUntil: (promise: Promise) => { + waitUntil.push(promise); + }, + passThroughOnException: () => {}, + exports: {}, + props: {}, + } as unknown as ExecutionContext); await Promise.allSettled(waitUntil); expect(waitUntil).toHaveLength(1); }); @@ -358,12 +486,24 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // A regular */2 tick (not :00, not :30) enqueues ONLY the light auto-maintain sweep — the heavier sync/health // jobs are gated to :00/:30, so the tight cadence stays cheap while merges/closes fire promptly. - expect(sent).toEqual([{ type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }, { type: "agent-regate-sweep", requestedBy: "schedule" }]); + expect(sent).toEqual([ + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, + { type: "agent-regate-sweep", requestedBy: "schedule" }, + ]); }); it("enqueues the APR repo-transfer poll on an hourly tick only when LOOPOVER_APR_TRANSFER_POLL is set (#7741)", async () => { @@ -378,10 +518,17 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); - expect(captured).toContainEqual({ type: "poll-apr-repo-transfers", requestedBy: "schedule" }); + expect(captured).toContainEqual({ + type: "poll-apr-repo-transfers", + requestedBy: "schedule", + }); }); it("keeps enqueueing scheduled sweeps while prior per-PR regate jobs are queued (#2119)", async () => { @@ -398,20 +545,35 @@ describe("worker entrypoint", () => { snapshotCalled = true; return { totals: { pending: 2, processing: 0, dead: 0, due: 2 }, - byType: [{ type: "agent-regate-pr", status: "pending", count: 2, due: 2 }], + byType: [ + { type: "agent-regate-pr", status: "pending", count: 2, due: 2 }, + ], }; }, } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:30:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); expect(sent).toEqual([ - { type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }, + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, { type: "agent-regate-sweep", requestedBy: "schedule" }, - { type: "backfill-registered-repos", requestedBy: "schedule", mode: "light" }, + { + type: "backfill-registered-repos", + requestedBy: "schedule", + mode: "light", + }, { type: "repair-data-fidelity", requestedBy: "schedule" }, { type: "refresh-installation-health", requestedBy: "schedule" }, { type: "backlog-convergence-sweep", requestedBy: "schedule" }, @@ -428,20 +590,40 @@ describe("worker entrypoint", () => { }, snapshot: async () => ({ totals: { pending: 0, processing: 1, dead: 0, due: 0 }, - byType: [{ type: "agent-regate-sweep", status: "processing", count: 1, due: 0 }], + byType: [ + { + type: "agent-regate-sweep", + status: "processing", + count: 1, + due: 0, + }, + ], }), } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:30:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // No SECOND "agent-regate-sweep" trigger is enqueued behind the one already in flight; the other :30 jobs // are unaffected since they never depended on the (removed, broad) backlog check. expect(sent).toEqual([ - { type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }, - { type: "backfill-registered-repos", requestedBy: "schedule", mode: "light" }, + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, + { + type: "backfill-registered-repos", + requestedBy: "schedule", + mode: "light", + }, { type: "repair-data-fidelity", requestedBy: "schedule" }, { type: "refresh-installation-health", requestedBy: "schedule" }, { type: "backlog-convergence-sweep", requestedBy: "schedule" }, @@ -457,21 +639,41 @@ describe("worker entrypoint", () => { }, snapshot: async () => ({ totals: { pending: 0, processing: 1, dead: 0, due: 0 }, - byType: [{ type: "backlog-convergence-sweep", status: "processing", count: 1, due: 0 }], + byType: [ + { + type: "backlog-convergence-sweep", + status: "processing", + count: 1, + due: 0, + }, + ], }), } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:30:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // No SECOND "backlog-convergence-sweep" trigger is enqueued behind the one already in flight; the other :30 // jobs (including agent-regate-sweep, whose OWN backlog is unaffected) still fire normally. expect(sent).toEqual([ - { type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }, + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, { type: "agent-regate-sweep", requestedBy: "schedule" }, - { type: "backfill-registered-repos", requestedBy: "schedule", mode: "light" }, + { + type: "backfill-registered-repos", + requestedBy: "schedule", + mode: "light", + }, { type: "repair-data-fidelity", requestedBy: "schedule" }, { type: "refresh-installation-health", requestedBy: "schedule" }, ]); @@ -492,13 +694,27 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // Fails OPEN on a broken snapshot binding: the sweep still enqueues, and the failure is surfaced (not // silently swallowed) so an operator can see the introspection is unavailable. - expect(sent).toEqual([{ type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }, { type: "agent-regate-sweep", requestedBy: "schedule" }]); - expect(warn).toHaveBeenCalledWith(expect.stringContaining("selfhost_queue_snapshot_failed")); + expect(sent).toEqual([ + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, + { type: "agent-regate-sweep", requestedBy: "schedule" }, + ]); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("selfhost_queue_snapshot_failed"), + ); }); it("does not enqueue review sweeps from a broker-only Cloudflare runtime (anchoring still runs -- it is not a review sweep, #9274)", async () => { @@ -513,10 +729,21 @@ describe("worker entrypoint", () => { delete env.SELFHOST_TRANSIENT_CACHE; const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); - expect(sent).toEqual([{ type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }]); + expect(sent).toEqual([ + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, + ]); }); it("keeps broker-only Cloudflare maintenance cheap on :30 ticks", async () => { @@ -531,11 +758,20 @@ describe("worker entrypoint", () => { delete env.SELFHOST_TRANSIENT_CACHE; const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:30:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); expect(sent).toEqual([ - { type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: false }, + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: false, + }, + { type: "sample-service-status", requestedBy: "schedule" }, { type: "repair-data-fidelity", requestedBy: "schedule" }, { type: "refresh-installation-health", requestedBy: "schedule" }, ]); @@ -544,12 +780,29 @@ describe("worker entrypoint", () => { it("THROTTLES the sweep when the GitHub REST budget is at/below the maintenance headroom (#6 backpressure)", async () => { const sent: Array = []; const env = createTestEnv({ - JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, }); // Seed a low REST observation (remaining 50 <= MAINTENANCE_RESERVED_HEADROOM=150) with a future reset. - await recordGitHubRateLimitObservation(env, { repoFullName: "owner/repo", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 50, resetAt: new Date(Date.now() + 600_000).toISOString(), observedAt: new Date().toISOString() }); + await recordGitHubRateLimitObservation(env, { + repoFullName: "owner/repo", + resource: "rest", + path: "/x", + statusCode: 200, + limitValue: 5000, + remaining: 50, + resetAt: new Date(Date.now() + 600_000).toISOString(), + observedAt: new Date().toISOString(), + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // The sweep is NOT enqueued this tick — the shared budget is reserved for webhooks; the next tick retries. expect(sent.find((m) => m.type === "agent-regate-sweep")).toBeUndefined(); @@ -558,30 +811,68 @@ describe("worker entrypoint", () => { it("THROTTLES the open-data backfill too when the REST budget is at/below the maintenance headroom, keeping the cheap health jobs (#audit-rate-headroom)", async () => { const sent: Array = []; const env = createTestEnv({ - JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, }); // Low REST budget (remaining 50 <= MAINTENANCE_RESERVED_HEADROOM=150) with a future reset. - await recordGitHubRateLimitObservation(env, { repoFullName: "owner/repo", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 50, resetAt: new Date(Date.now() + 600_000).toISOString(), observedAt: new Date().toISOString() }); + await recordGitHubRateLimitObservation(env, { + repoFullName: "owner/repo", + resource: "rest", + path: "/x", + statusCode: 200, + limitValue: 5000, + remaining: 50, + resetAt: new Date(Date.now() + 600_000).toISOString(), + observedAt: new Date().toISOString(), + }); const waitUntil: Promise[] = []; // A :30 tick is when the backfill would normally enqueue — assert it does NOT while the budget is reserved. - await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:30:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // Backfill (+ sweep) yield the budget to webhooks; the cheap single-call health jobs still run. - expect(sent.some((m) => m.type === "backfill-registered-repos")).toBe(false); + expect(sent.some((m) => m.type === "backfill-registered-repos")).toBe( + false, + ); expect(sent.some((m) => m.type === "agent-regate-sweep")).toBe(false); expect(sent.some((m) => m.type === "repair-data-fidelity")).toBe(true); - expect(sent.some((m) => m.type === "refresh-installation-health")).toBe(true); + expect(sent.some((m) => m.type === "refresh-installation-health")).toBe( + true, + ); }); it("enqueues the open-data backfill on a :30 tick when there is REST headroom (#audit-rate-headroom)", async () => { const sent: Array = []; const env = createTestEnv({ - JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, }); // Ample REST budget (remaining 4000 > MAINTENANCE_RESERVED_HEADROOM=150) → the throttle does NOT engage. - await recordGitHubRateLimitObservation(env, { repoFullName: "owner/repo", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 4000, resetAt: new Date(Date.now() + 600_000).toISOString(), observedAt: new Date().toISOString() }); + await recordGitHubRateLimitObservation(env, { + repoFullName: "owner/repo", + resource: "rest", + path: "/x", + statusCode: 200, + limitValue: 5000, + remaining: 4000, + resetAt: new Date(Date.now() + 600_000).toISOString(), + observedAt: new Date().toISOString(), + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:30:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); expect(sent.some((m) => m.type === "backfill-registered-repos")).toBe(true); expect(sent.some((m) => m.type === "agent-regate-sweep")).toBe(true); @@ -598,16 +889,29 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // refresh-registry is absent: a fresh self-host instance (this test's default createTestEnv) has no repo // opted into the experimental gittensor plugin, so the job is never enqueued (#experimental-gittensor-plugin) // — see "re-includes refresh-registry once a repo opts into the experimental gittensor plugin" below. expect(sent).toEqual([ - { type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: true }, + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: true, + }, + { type: "sample-service-status", requestedBy: "schedule" }, { type: "agent-regate-sweep", requestedBy: "schedule" }, - { type: "backfill-registered-repos", requestedBy: "schedule", mode: "light" }, + { + type: "backfill-registered-repos", + requestedBy: "schedule", + mode: "light", + }, { type: "repair-data-fidelity", requestedBy: "schedule" }, { type: "refresh-installation-health", requestedBy: "schedule" }, { type: "backlog-convergence-sweep", requestedBy: "schedule" }, @@ -631,17 +935,35 @@ describe("worker entrypoint", () => { }, } as unknown as Queue, }); - await (env.DB as unknown as { prepare: (s: string) => { bind: (...v: unknown[]) => { run: () => Promise } } }) - .prepare("INSERT INTO repositories (full_name, owner, name, is_installed, is_registered) VALUES (?, ?, ?, 1, 0)") + await ( + env.DB as unknown as { + prepare: (s: string) => { + bind: (...v: unknown[]) => { run: () => Promise }; + }; + } + ) + .prepare( + "INSERT INTO repositories (full_name, owner, name, is_installed, is_registered) VALUES (?, ?, ?, 1, 0)", + ) .bind("JSONbored/loopover", "JSONbored", "loopover") .run(); - await (await import("../../src/signals/focus-manifest-loader")).upsertRepoFocusManifest(env, "JSONbored/loopover", { experimental: { gittensor: true } }); + await ( + await import("../../src/signals/focus-manifest-loader") + ).upsertRepoFocusManifest(env, "JSONbored/loopover", { + experimental: { gittensor: true }, + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); - expect(sent.some((message) => message.type === "refresh-registry")).toBe(true); + expect(sent.some((message) => message.type === "refresh-registry")).toBe( + true, + ); }); it("always enqueues refresh-registry on a cloud (non-self-hosted) runtime, regardless of gittensor opt-in (#experimental-gittensor-plugin)", async () => { @@ -656,12 +978,18 @@ describe("worker entrypoint", () => { delete env.SELFHOST_TRANSIENT_CACHE; const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // Cloud never consults gittensorEnabledRepoFullNames at all — the `!selfHostedReviews` short-circuit // fires before the opted-in check, exactly like before this narrowing existed. - expect(sent.some((message) => message.type === "refresh-registry")).toBe(true); + expect(sent.some((message) => message.type === "refresh-registry")).toBe( + true, + ); }); it("enqueues full-sync scheduled work every six hours", async () => { @@ -675,14 +1003,27 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T06:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T06:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // refresh-registry is absent here too — see the comment on "enqueues hourly refreshes..." above. expect(sent).toEqual([ - { type: "anchor-decision-ledger", requestedBy: "schedule", isHourly: true }, + { + type: "anchor-decision-ledger", + requestedBy: "schedule", + isHourly: true, + }, + { type: "sample-service-status", requestedBy: "schedule" }, { type: "agent-regate-sweep", requestedBy: "schedule" }, - { type: "backfill-registered-repos", requestedBy: "schedule", mode: "full" }, + { + type: "backfill-registered-repos", + requestedBy: "schedule", + mode: "full", + }, { type: "repair-data-fidelity", requestedBy: "schedule" }, { type: "refresh-installation-health", requestedBy: "schedule" }, { type: "backlog-convergence-sweep", requestedBy: "schedule" }, @@ -721,13 +1062,18 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T06:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T06:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); // The enqueued SET is unchanged — jitter only spreads run_after timing, never which jobs are sent. // refresh-registry is absent here too — see the comment on "enqueues hourly refreshes..." above. expect(sent.map((s) => s.message.type)).toEqual([ "anchor-decision-ledger", + "sample-service-status", "agent-regate-sweep", "backfill-registered-repos", "repair-data-fidelity", @@ -752,12 +1098,16 @@ describe("worker entrypoint", () => { } // The priority sweep specifically stays immediate; at least one periodic job is actually deferred, so a // top-of-6h tick no longer fires every heavy fan-out parent in the same instant. - expect(sent.find((s) => s.message.type === "agent-regate-sweep")?.delaySeconds).toBeUndefined(); + expect( + sent.find((s) => s.message.type === "agent-regate-sweep")?.delaySeconds, + ).toBeUndefined(); expect(sent.some((s) => (s.delaySeconds ?? 0) > 0)).toBe(true); }); it("enqueues the ops-alerts job hourly ONLY when LOOPOVER_REVIEW_OPS is ON (flag-OFF is byte-identical)", async () => { - const sentFor = async (opsFlag?: string): Promise> => { + const sentFor = async ( + opsFlag?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ ...(opsFlag === undefined ? {} : { LOOPOVER_REVIEW_OPS: opsFlag }), @@ -768,17 +1118,25 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // Flag OFF (default) → no ops-alerts job; the enqueued set is unchanged from today. expect((await sentFor()).some((m) => m.type === "ops-alerts")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "ops-alerts")).toBe(false); + expect((await sentFor("false")).some((m) => m.type === "ops-alerts")).toBe( + false, + ); // Flag ON → exactly one ops-alerts job, enqueued in the hourly window. const on = await sentFor("true"); - expect(on.filter((m) => m.type === "ops-alerts")).toEqual([{ type: "ops-alerts", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "ops-alerts")).toEqual([ + { type: "ops-alerts", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue ops-alerts outside the hourly window even when LOOPOVER_REVIEW_OPS is ON", async () => { @@ -792,7 +1150,11 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:15:00.000Z"), env, executionContext(waitUntil)); // non-hourly + await worker.scheduled( + controllerFor("2026-05-25T05:15:00.000Z"), + env, + executionContext(waitUntil), + ); // non-hourly await Promise.all(waitUntil); expect(sent.some((m) => m.type === "ops-alerts")).toBe(false); }); @@ -807,11 +1169,19 @@ describe("worker entrypoint", () => { } as unknown as Queue, LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover", }); - await upsertRepoFocusManifest(env, "JSONbored/loopover", { ops: { enabled: true } }); + await upsertRepoFocusManifest(env, "JSONbored/loopover", { + ops: { enabled: true }, + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); - expect(sent.filter((m) => m.type === "ops-alerts")).toEqual([{ type: "ops-alerts", requestedBy: "schedule" }]); + expect(sent.filter((m) => m.type === "ops-alerts")).toEqual([ + { type: "ops-alerts", requestedBy: "schedule" }, + ]); }); it("a present ops manifest override suppresses ops-alerts hourly even when LOOPOVER_REVIEW_OPS is ON (#6275)", async () => { @@ -825,18 +1195,28 @@ describe("worker entrypoint", () => { } as unknown as Queue, LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover", }); - await upsertRepoFocusManifest(env, "JSONbored/loopover", { ops: { enabled: false } }); + await upsertRepoFocusManifest(env, "JSONbored/loopover", { + ops: { enabled: false }, + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); expect(sent.some((m) => m.type === "ops-alerts")).toBe(false); }); it("enqueues sync-brokered-installed-repos hourly ONLY in broker mode (ORB_ENROLLMENT_SECRET set, flag-OFF is byte-identical)", async () => { - const sentFor = async (enrollmentSecret?: string): Promise> => { + const sentFor = async ( + enrollmentSecret?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ - ...(enrollmentSecret === undefined ? {} : { ORB_ENROLLMENT_SECRET: enrollmentSecret }), + ...(enrollmentSecret === undefined + ? {} + : { ORB_ENROLLMENT_SECRET: enrollmentSecret }), JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); @@ -844,16 +1224,26 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // Non-brokered (default) → no sync job; the enqueued set is unchanged from today. - expect((await sentFor()).some((m) => m.type === "sync-brokered-installed-repos")).toBe(false); + expect( + (await sentFor()).some((m) => m.type === "sync-brokered-installed-repos"), + ).toBe(false); // Brokered (an enrollment secret is configured) → exactly one sync job, enqueued in the hourly window. const brokered = await sentFor("orbsec_x"); - expect(brokered.filter((m) => m.type === "sync-brokered-installed-repos")).toEqual([{ type: "sync-brokered-installed-repos", requestedBy: "schedule" }]); + expect( + brokered.filter((m) => m.type === "sync-brokered-installed-repos"), + ).toEqual([ + { type: "sync-brokered-installed-repos", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue sync-brokered-installed-repos outside the hourly window even in broker mode", async () => { @@ -867,16 +1257,26 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:15:00.000Z"), env, executionContext(waitUntil)); // non-hourly + await worker.scheduled( + controllerFor("2026-05-25T05:15:00.000Z"), + env, + executionContext(waitUntil), + ); // non-hourly await Promise.all(waitUntil); - expect(sent.some((m) => m.type === "sync-brokered-installed-repos")).toBe(false); + expect(sent.some((m) => m.type === "sync-brokered-installed-repos")).toBe( + false, + ); }); it("enqueues the sweep-liveness-watchdog job hourly ONLY when LOOPOVER_SWEEP_WATCHDOG is ON (flag-OFF is byte-identical)", async () => { - const sentFor = async (watchdogFlag?: string): Promise> => { + const sentFor = async ( + watchdogFlag?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ - ...(watchdogFlag === undefined ? {} : { LOOPOVER_SWEEP_WATCHDOG: watchdogFlag }), + ...(watchdogFlag === undefined + ? {} + : { LOOPOVER_SWEEP_WATCHDOG: watchdogFlag }), JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); @@ -884,21 +1284,35 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // Flag OFF (default) → no sweep-liveness-watchdog job; the enqueued set is unchanged from today. - expect((await sentFor()).some((m) => m.type === "sweep-liveness-watchdog")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "sweep-liveness-watchdog")).toBe(false); + expect( + (await sentFor()).some((m) => m.type === "sweep-liveness-watchdog"), + ).toBe(false); + expect( + (await sentFor("false")).some( + (m) => m.type === "sweep-liveness-watchdog", + ), + ).toBe(false); // Flag ON → exactly one sweep-liveness-watchdog job, enqueued in the hourly window. const on = await sentFor("true"); - expect(on.filter((m) => m.type === "sweep-liveness-watchdog")).toEqual([{ type: "sweep-liveness-watchdog", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "sweep-liveness-watchdog")).toEqual([ + { type: "sweep-liveness-watchdog", requestedBy: "schedule" }, + ]); }); it("enqueues the loop-escalation-sweep job hourly ONLY when LOOPOVER_LOOP_ESCALATION is ON (flag-OFF is byte-identical)", async () => { - const sentFor = async (flag?: string): Promise> => { + const sentFor = async ( + flag?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ ...(flag === undefined ? {} : { LOOPOVER_LOOP_ESCALATION: flag }), @@ -909,15 +1323,25 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; - expect((await sentFor()).some((m) => m.type === "loop-escalation-sweep")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "loop-escalation-sweep")).toBe(false); + expect( + (await sentFor()).some((m) => m.type === "loop-escalation-sweep"), + ).toBe(false); + expect( + (await sentFor("false")).some((m) => m.type === "loop-escalation-sweep"), + ).toBe(false); const on = await sentFor("true"); - expect(on.filter((m) => m.type === "loop-escalation-sweep")).toEqual([{ type: "loop-escalation-sweep", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "loop-escalation-sweep")).toEqual([ + { type: "loop-escalation-sweep", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue sweep-liveness-watchdog outside the hourly window even when LOOPOVER_SWEEP_WATCHDOG is ON", async () => { @@ -931,13 +1355,20 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:15:00.000Z"), env, executionContext(waitUntil)); // non-hourly + await worker.scheduled( + controllerFor("2026-05-25T05:15:00.000Z"), + env, + executionContext(waitUntil), + ); // non-hourly await Promise.all(waitUntil); expect(sent.some((m) => m.type === "sweep-liveness-watchdog")).toBe(false); }); it("enqueues the reconcile-open-prs job every 10 minutes ONLY when LOOPOVER_PR_RECONCILIATION is ON (flag-OFF is byte-identical)", async () => { - const sentFor = async (flag?: string, isoTime = "2026-05-25T05:10:00.000Z"): Promise> => { + const sentFor = async ( + flag?: string, + isoTime = "2026-05-25T05:10:00.000Z", + ): Promise> => { const sent: Array = []; const env = createTestEnv({ ...(flag === undefined ? {} : { LOOPOVER_PR_RECONCILIATION: flag }), @@ -948,17 +1379,27 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor(isoTime), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor(isoTime), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // Flag OFF (default) → no reconcile-open-prs job; the enqueued set is unchanged from today. - expect((await sentFor()).some((m) => m.type === "reconcile-open-prs")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "reconcile-open-prs")).toBe(false); + expect((await sentFor()).some((m) => m.type === "reconcile-open-prs")).toBe( + false, + ); + expect( + (await sentFor("false")).some((m) => m.type === "reconcile-open-prs"), + ).toBe(false); // Flag ON, on a 10-minute boundary → exactly one reconcile-open-prs job. const on = await sentFor("true"); - expect(on.filter((m) => m.type === "reconcile-open-prs")).toEqual([{ type: "reconcile-open-prs", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "reconcile-open-prs")).toEqual([ + { type: "reconcile-open-prs", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue reconcile-open-prs outside the 10-minute window even when LOOPOVER_PR_RECONCILIATION is ON", async () => { @@ -972,16 +1413,25 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); // not a 10-minute boundary + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); // not a 10-minute boundary await Promise.all(waitUntil); expect(sent.some((m) => m.type === "reconcile-open-prs")).toBe(false); }); it("enqueues the reconcile-active-review-tracking job every 10 minutes ONLY when LOOPOVER_ACTIVE_REVIEW_RECONCILIATION is ON (flag-OFF is byte-identical)", async () => { - const sentFor = async (flag?: string, isoTime = "2026-05-25T05:10:00.000Z"): Promise> => { + const sentFor = async ( + flag?: string, + isoTime = "2026-05-25T05:10:00.000Z", + ): Promise> => { const sent: Array = []; const env = createTestEnv({ - ...(flag === undefined ? {} : { LOOPOVER_ACTIVE_REVIEW_RECONCILIATION: flag }), + ...(flag === undefined + ? {} + : { LOOPOVER_ACTIVE_REVIEW_RECONCILIATION: flag }), JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); @@ -989,17 +1439,33 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor(isoTime), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor(isoTime), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // Flag OFF (default) → no reconcile-active-review-tracking job; the enqueued set is unchanged from today. - expect((await sentFor()).some((m) => m.type === "reconcile-active-review-tracking")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "reconcile-active-review-tracking")).toBe(false); + expect( + (await sentFor()).some( + (m) => m.type === "reconcile-active-review-tracking", + ), + ).toBe(false); + expect( + (await sentFor("false")).some( + (m) => m.type === "reconcile-active-review-tracking", + ), + ).toBe(false); // Flag ON, on a 10-minute boundary → exactly one reconcile-active-review-tracking job. const on = await sentFor("true"); - expect(on.filter((m) => m.type === "reconcile-active-review-tracking")).toEqual([{ type: "reconcile-active-review-tracking", requestedBy: "schedule" }]); + expect( + on.filter((m) => m.type === "reconcile-active-review-tracking"), + ).toEqual([ + { type: "reconcile-active-review-tracking", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue reconcile-active-review-tracking outside the 10-minute window even when LOOPOVER_ACTIVE_REVIEW_RECONCILIATION is ON", async () => { @@ -1013,38 +1479,80 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); // not a 10-minute boundary + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); // not a 10-minute boundary await Promise.all(waitUntil); - expect(sent.some((m) => m.type === "reconcile-active-review-tracking")).toBe(false); + expect( + sent.some((m) => m.type === "reconcile-active-review-tracking"), + ).toBe(false); }); it("enqueues federated-peer-sync every 10 minutes ONLY when the loopover self-repo manifest opts in (#9148) — absent-manifest default is byte-identical", async () => { - const sentFor = async (enabled?: boolean, isoTime = "2026-05-25T05:10:00.000Z"): Promise> => { + const sentFor = async ( + enabled?: boolean, + isoTime = "2026-05-25T05:10:00.000Z", + ): Promise> => { clearFederatedIntelligenceManifestOverrideCacheForTest(); // each sub-case below is a fresh env; don't let the prior case's cached override leak in const sent: Array = []; - const env = createTestEnv({ LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover", JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue }); - if (enabled !== undefined) await upsertRepoFocusManifest(env, "JSONbored/loopover", { federatedIntelligence: { enabled } }); + const env = createTestEnv({ + LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover", + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, + }); + if (enabled !== undefined) + await upsertRepoFocusManifest(env, "JSONbored/loopover", { + federatedIntelligence: { enabled }, + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor(isoTime), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor(isoTime), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // No manifest block at all (absent federatedIntelligence, the fleet-wide default) → never enqueued. - expect((await sentFor(undefined)).some((m) => m.type === "federated-peer-sync")).toBe(false); + expect( + (await sentFor(undefined)).some((m) => m.type === "federated-peer-sync"), + ).toBe(false); // Manifest present but explicitly disabled → still never enqueued. - expect((await sentFor(false)).some((m) => m.type === "federated-peer-sync")).toBe(false); + expect( + (await sentFor(false)).some((m) => m.type === "federated-peer-sync"), + ).toBe(false); // Opted in, on a 10-minute boundary → exactly one federated-peer-sync job. const on = await sentFor(true); - expect(on.filter((m) => m.type === "federated-peer-sync")).toEqual([{ type: "federated-peer-sync", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "federated-peer-sync")).toEqual([ + { type: "federated-peer-sync", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue federated-peer-sync outside the 10-minute window even when opted in", async () => { const sent: Array = []; - const env = createTestEnv({ LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover", JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue }); - await upsertRepoFocusManifest(env, "JSONbored/loopover", { federatedIntelligence: { enabled: true } }); + const env = createTestEnv({ + LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover", + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, + }); + await upsertRepoFocusManifest(env, "JSONbored/loopover", { + federatedIntelligence: { enabled: true }, + }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); // not a 10-minute boundary + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); // not a 10-minute boundary await Promise.all(waitUntil); expect(sent.some((m) => m.type === "federated-peer-sync")).toBe(false); }); @@ -1078,16 +1586,20 @@ describe("worker entrypoint", () => { expect((await sentFor("false")).some((m) => m.type === "selftune")).toBe( false, ); - expect((await sentFor("true")).filter((m) => m.type === "selftune")).toEqual([ - { type: "selftune", requestedBy: "schedule" }, - ]); + expect( + (await sentFor("true")).filter((m) => m.type === "selftune"), + ).toEqual([{ type: "selftune", requestedBy: "schedule" }]); }); it("enqueues the satisfaction-floor-loosening tick hourly only when SATISFACTION_FLOOR_AUTOTUNE_ENABLED is ON (#8158)", async () => { - const sentFor = async (flag?: string): Promise> => { + const sentFor = async ( + flag?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ - ...(flag === undefined ? {} : ({ SATISFACTION_FLOOR_AUTOTUNE_ENABLED: flag } as Partial)), + ...(flag === undefined + ? {} + : ({ SATISFACTION_FLOOR_AUTOTUNE_ENABLED: flag } as Partial)), JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); @@ -1095,30 +1607,52 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); return sent; }; // createTestEnv defaults the flag to "false" — the undefined case exercises exactly that shipped default. - expect((await sentFor()).some((m) => m.type === "satisfaction-floor-loosening")).toBe(false); - expect((await sentFor("true")).filter((m) => m.type === "satisfaction-floor-loosening")).toEqual([ + expect( + (await sentFor()).some((m) => m.type === "satisfaction-floor-loosening"), + ).toBe(false); + expect( + (await sentFor("true")).filter( + (m) => m.type === "satisfaction-floor-loosening", + ), + ).toEqual([ { type: "satisfaction-floor-loosening", requestedBy: "schedule" }, ]); // Off the hourly boundary, flag-ON still enqueues nothing. const sent: Array = []; const env = createTestEnv({ SATISFACTION_FLOOR_AUTOTUNE_ENABLED: "true" as never, - JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-05-25T05:14:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); - expect(sent.some((m) => m.type === "satisfaction-floor-loosening")).toBe(false); + expect(sent.some((m) => m.type === "satisfaction-floor-loosening")).toBe( + false, + ); }); it("enqueues the rag-index-repo fan-out in the full-sync window ONLY when LOOPOVER_REVIEW_RAG is ON (flag-OFF is byte-identical)", async () => { - const sentFor = async (ragFlag?: string): Promise> => { + const sentFor = async ( + ragFlag?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ ...(ragFlag === undefined ? {} : { LOOPOVER_REVIEW_RAG: ragFlag }), @@ -1129,17 +1663,27 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T06:00:00.000Z"), env, executionContext(waitUntil)); // full-sync window + await worker.scheduled( + controllerFor("2026-05-25T06:00:00.000Z"), + env, + executionContext(waitUntil), + ); // full-sync window await Promise.all(waitUntil); return sent; }; // Flag OFF (default) → no rag-index-repo job; the enqueued set is unchanged from today. - expect((await sentFor()).some((m) => m.type === "rag-index-repo")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "rag-index-repo")).toBe(false); + expect((await sentFor()).some((m) => m.type === "rag-index-repo")).toBe( + false, + ); + expect( + (await sentFor("false")).some((m) => m.type === "rag-index-repo"), + ).toBe(false); // Flag ON → exactly one rag-index-repo fan-out job, enqueued in the full-sync window. const on = await sentFor("true"); - expect(on.filter((m) => m.type === "rag-index-repo")).toEqual([{ type: "rag-index-repo", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "rag-index-repo")).toEqual([ + { type: "rag-index-repo", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue rag-index-repo outside the full-sync window even when LOOPOVER_REVIEW_RAG is ON", async () => { @@ -1153,7 +1697,11 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); // hourly but NOT full-sync + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); // hourly but NOT full-sync await Promise.all(waitUntil); expect(sent.some((m) => m.type === "rag-index-repo")).toBe(false); }); @@ -1169,10 +1717,18 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-06-01T09:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-06-01T09:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); - expect(sent).toEqual(expect.arrayContaining([{ type: "repo-doc-refresh-sweep", requestedBy: "schedule" }])); + expect(sent).toEqual( + expect.arrayContaining([ + { type: "repo-doc-refresh-sweep", requestedBy: "schedule" }, + ]), + ); }); it("does NOT enqueue the repo-doc refresh sweep outside the 09:00 UTC window", async () => { @@ -1186,7 +1742,11 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-06-01T10:00:00.000Z"), env, executionContext(waitUntil)); // hourly but not 09:00 + await worker.scheduled( + controllerFor("2026-06-01T10:00:00.000Z"), + env, + executionContext(waitUntil), + ); // hourly but not 09:00 await Promise.all(waitUntil); expect(sent.some((m) => m.type === "repo-doc-refresh-sweep")).toBe(false); @@ -1196,37 +1756,111 @@ describe("worker entrypoint", () => { const send = (over: Record, when: string) => { const sent: Array = []; const env = createTestEnv({ - JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, ...over, }); const waitUntil: Promise[] = []; - return worker.scheduled(controllerFor(when), env, executionContext(waitUntil)).then(() => Promise.all(waitUntil)).then(() => sent); + return worker + .scheduled(controllerFor(when), env, executionContext(waitUntil)) + .then(() => Promise.all(waitUntil)) + .then(() => sent); + }; + const selfhost = { + SELFHOST_TRANSIENT_CACHE: {} as never, + LOOPOVER_DECISION_AUDIT: "true", }; - const selfhost = { SELFHOST_TRANSIENT_CACHE: {} as never, LOOPOVER_DECISION_AUDIT: "true" }; // 2026-06-02 is a Tuesday. - expect((await send(selfhost, "2026-06-02T08:00:00.000Z")).some((m) => m.type === "decision-audit-sample")).toBe(true); + expect( + (await send(selfhost, "2026-06-02T08:00:00.000Z")).some( + (m) => m.type === "decision-audit-sample", + ), + ).toBe(true); // Wrong hour, wrong day, flag off, and cloud runtime each suppress the enqueue. - expect((await send(selfhost, "2026-06-02T09:00:00.000Z")).some((m) => m.type === "decision-audit-sample")).toBe(false); - expect((await send(selfhost, "2026-06-01T08:00:00.000Z")).some((m) => m.type === "decision-audit-sample")).toBe(false); - expect((await send({ ...selfhost, LOOPOVER_DECISION_AUDIT: "false" }, "2026-06-02T08:00:00.000Z")).some((m) => m.type === "decision-audit-sample")).toBe(false); - expect((await send({ LOOPOVER_DECISION_AUDIT: "true", SELFHOST_TRANSIENT_CACHE: undefined }, "2026-06-02T08:00:00.000Z")).some((m) => m.type === "decision-audit-sample")).toBe(false); + expect( + (await send(selfhost, "2026-06-02T09:00:00.000Z")).some( + (m) => m.type === "decision-audit-sample", + ), + ).toBe(false); + expect( + (await send(selfhost, "2026-06-01T08:00:00.000Z")).some( + (m) => m.type === "decision-audit-sample", + ), + ).toBe(false); + expect( + ( + await send( + { ...selfhost, LOOPOVER_DECISION_AUDIT: "false" }, + "2026-06-02T08:00:00.000Z", + ) + ).some((m) => m.type === "decision-audit-sample"), + ).toBe(false); + expect( + ( + await send( + { + LOOPOVER_DECISION_AUDIT: "true", + SELFHOST_TRANSIENT_CACHE: undefined, + }, + "2026-06-02T08:00:00.000Z", + ) + ).some((m) => m.type === "decision-audit-sample"), + ).toBe(false); }); it("#8835: enqueues the risk-control recalibration daily at 07:00 UTC, flag-ON, on a self-host", async () => { const send = (over: Record, when: string) => { const sent: Array = []; const env = createTestEnv({ - JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, ...over, }); const waitUntil: Promise[] = []; - return worker.scheduled(controllerFor(when), env, executionContext(waitUntil)).then(() => Promise.all(waitUntil)).then(() => sent); + return worker + .scheduled(controllerFor(when), env, executionContext(waitUntil)) + .then(() => Promise.all(waitUntil)) + .then(() => sent); }; - const selfhost = { SELFHOST_TRANSIENT_CACHE: {} as never, LOOPOVER_RISK_CONTROL: "true" }; - expect((await send(selfhost, "2026-06-03T07:00:00.000Z")).some((m) => m.type === "risk-control-recalibrate")).toBe(true); - expect((await send(selfhost, "2026-06-03T08:00:00.000Z")).some((m) => m.type === "risk-control-recalibrate")).toBe(false); - expect((await send({ ...selfhost, LOOPOVER_RISK_CONTROL: "0" }, "2026-06-03T07:00:00.000Z")).some((m) => m.type === "risk-control-recalibrate")).toBe(false); - expect((await send({ LOOPOVER_RISK_CONTROL: "true", SELFHOST_TRANSIENT_CACHE: undefined }, "2026-06-03T07:00:00.000Z")).some((m) => m.type === "risk-control-recalibrate")).toBe(false); + const selfhost = { + SELFHOST_TRANSIENT_CACHE: {} as never, + LOOPOVER_RISK_CONTROL: "true", + }; + expect( + (await send(selfhost, "2026-06-03T07:00:00.000Z")).some( + (m) => m.type === "risk-control-recalibrate", + ), + ).toBe(true); + expect( + (await send(selfhost, "2026-06-03T08:00:00.000Z")).some( + (m) => m.type === "risk-control-recalibrate", + ), + ).toBe(false); + expect( + ( + await send( + { ...selfhost, LOOPOVER_RISK_CONTROL: "0" }, + "2026-06-03T07:00:00.000Z", + ) + ).some((m) => m.type === "risk-control-recalibrate"), + ).toBe(false); + expect( + ( + await send( + { + LOOPOVER_RISK_CONTROL: "true", + SELFHOST_TRANSIENT_CACHE: undefined, + }, + "2026-06-03T07:00:00.000Z", + ) + ).some((m) => m.type === "risk-control-recalibrate"), + ).toBe(false); }); it("enqueues weekly value report generation during the Monday report window", async () => { @@ -1240,22 +1874,35 @@ describe("worker entrypoint", () => { }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-06-01T12:00:00.000Z"), env, executionContext(waitUntil)); + await worker.scheduled( + controllerFor("2026-06-01T12:00:00.000Z"), + env, + executionContext(waitUntil), + ); await Promise.all(waitUntil); expect(sent).toEqual( expect.arrayContaining([ { type: "rollup-product-usage", requestedBy: "schedule", days: 7 }, - { type: "generate-weekly-value-report", requestedBy: "schedule", variant: "operator", days: 7 }, + { + type: "generate-weekly-value-report", + requestedBy: "schedule", + variant: "operator", + days: 7, + }, ]), ); }); it("enqueues the maintainer recap digest on the default weekly cadence (Monday 14:00 UTC) ONLY when LOOPOVER_MAINTAINER_RECAP is ON (#2248, flag-OFF is byte-identical)", async () => { - const sentFor = async (recapFlag?: string): Promise> => { + const sentFor = async ( + recapFlag?: string, + ): Promise> => { const sent: Array = []; const env = createTestEnv({ - ...(recapFlag === undefined ? {} : { LOOPOVER_MAINTAINER_RECAP: recapFlag }), + ...(recapFlag === undefined + ? {} + : { LOOPOVER_MAINTAINER_RECAP: recapFlag }), JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); @@ -1263,17 +1910,29 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-06-01T14:00:00.000Z"), env, executionContext(waitUntil)); // Monday, 14:00 UTC + await worker.scheduled( + controllerFor("2026-06-01T14:00:00.000Z"), + env, + executionContext(waitUntil), + ); // Monday, 14:00 UTC await Promise.all(waitUntil); return sent; }; // Flag OFF (default) → no recap job; the enqueued set is unchanged from today. - expect((await sentFor()).some((m) => m.type === "generate-maintainer-recap")).toBe(false); - expect((await sentFor("false")).some((m) => m.type === "generate-maintainer-recap")).toBe(false); + expect( + (await sentFor()).some((m) => m.type === "generate-maintainer-recap"), + ).toBe(false); + expect( + (await sentFor("false")).some( + (m) => m.type === "generate-maintainer-recap", + ), + ).toBe(false); // Flag ON, on the default weekly cadence tick → exactly one recap job. const on = await sentFor("true"); - expect(on.filter((m) => m.type === "generate-maintainer-recap")).toEqual([{ type: "generate-maintainer-recap", requestedBy: "schedule" }]); + expect(on.filter((m) => m.type === "generate-maintainer-recap")).toEqual([ + { type: "generate-maintainer-recap", requestedBy: "schedule" }, + ]); }); it("does NOT enqueue the maintainer recap digest outside its configured cadence even when LOOPOVER_MAINTAINER_RECAP is ON", async () => { @@ -1287,9 +1946,15 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-06-02T14:00:00.000Z"), env, executionContext(waitUntil)); // Tuesday, not the weekly default day + await worker.scheduled( + controllerFor("2026-06-02T14:00:00.000Z"), + env, + executionContext(waitUntil), + ); // Tuesday, not the weekly default day await Promise.all(waitUntil); - expect(sent.some((m) => m.type === "generate-maintainer-recap")).toBe(false); + expect(sent.some((m) => m.type === "generate-maintainer-recap")).toBe( + false, + ); }); it("honors a custom LOOPOVER_RECAP_CADENCE=daily, firing every day at the configured hour", async () => { @@ -1304,9 +1969,15 @@ describe("worker entrypoint", () => { } as unknown as Queue, }); const waitUntil: Promise[] = []; - await worker.scheduled(controllerFor("2026-06-02T14:00:00.000Z"), env, executionContext(waitUntil)); // Tuesday — not the weekly default day + await worker.scheduled( + controllerFor("2026-06-02T14:00:00.000Z"), + env, + executionContext(waitUntil), + ); // Tuesday — not the weekly default day await Promise.all(waitUntil); - expect(sent.filter((m) => m.type === "generate-maintainer-recap")).toEqual([{ type: "generate-maintainer-recap", requestedBy: "schedule" }]); + expect(sent.filter((m) => m.type === "generate-maintainer-recap")).toEqual([ + { type: "generate-maintainer-recap", requestedBy: "schedule" }, + ]); }); }); diff --git a/test/unit/service-status-history.test.ts b/test/unit/service-status-history.test.ts new file mode 100644 index 0000000000..34a2eef18e --- /dev/null +++ b/test/unit/service-status-history.test.ts @@ -0,0 +1,235 @@ +import { describe, expect, it } from "vitest"; + +import { createApp } from "../../src/api/routes"; +import { RETENTION_POLICY, retentionDaysForTable } from "../../src/db/retention"; +import { loadServiceStatusSamples, recordServiceStatusSamples } from "../../src/selfhost/service-status-history"; +import { createTestEnv } from "../helpers/d1"; +import { + buildComponentHistory, + computeIncidents, + computeUptimeWindow, + isIncidentStatus, + UPTIME_WINDOW_DAYS, + type StatusSample, +} from "../../src/selfhost/service-status-history"; + +// #9985: uptime and incidents derived from status samples. +// +// Every test here is about a way a plausible implementation would publish something untrue: +// +// • a 30-day figure computed over three days of data, presented as if it covered a month; +// • `unknown` ticks — time we were BLIND — counted as uptime (inflates exactly when we could not see) or as +// downtime (invents an outage nobody observed); +// • an empty window rendering as 0% (reads as a total outage) or 100% (reads as perfect); +// • an `unknown` tick splitting one outage into two incidents, manufacturing incidents from our blindness; +// • an open incident silently closed at "now", asserting a recovery that was never observed. + +const NOW = Date.parse("2026-07-31T12:00:00.000Z"); +const at = (hoursAgo: number): string => new Date(NOW - hoursAgo * 3_600_000).toISOString(); +const sample = (status: StatusSample["status"], hoursAgo: number): StatusSample => ({ status, sampledAt: at(hoursAgo) }); + +describe("isIncidentStatus", () => { + it("counts degraded and outage, and deliberately not unknown", () => { + expect(isIncidentStatus("degraded")).toBe(true); + expect(isIncidentStatus("outage")).toBe(true); + // We did not observe a problem — we failed to look. + expect(isIncidentStatus("unknown")).toBe(false); + expect(isIncidentStatus("operational")).toBe(false); + }); +}); + +describe("computeUptimeWindow", () => { + it("is the operational share of MEASURED samples", () => { + const window = computeUptimeWindow([sample("operational", 1), sample("operational", 2), sample("outage", 3), sample("operational", 4)], 7, NOW); + expect(window.uptime).toBe(0.75); + expect(window.measured).toBe(4); + expect(window.unmeasured).toBe(0); + }); + + it("REGRESSION: unknown ticks are excluded from the percentage and reported separately", () => { + // Two operational, two blind. Counting the blind ticks as up gives 100% — a claim of perfect health over + // a period half of which we could not see. Counting them as down gives 50%, inventing an outage. + const window = computeUptimeWindow([sample("operational", 1), sample("unknown", 2), sample("unknown", 3), sample("operational", 4)], 7, NOW); + expect(window.uptime).toBe(1); + expect(window.measured).toBe(2); + expect(window.unmeasured).toBe(2); + }); + + it("REGRESSION: an empty window is null, never 0% or 100%", () => { + // 0% reads as a total outage and 100% reads as perfect. Both are claims; no data is not a claim. + const window = computeUptimeWindow([], 30, NOW); + expect(window.uptime).toBeNull(); + expect(window.measured).toBe(0); + expect(window.since).toBeNull(); + }); + + it("is null, not 100%, when every sample in the window was unmeasured", () => { + const window = computeUptimeWindow([sample("unknown", 1), sample("unknown", 2)], 7, NOW); + expect(window.uptime).toBeNull(); + expect(window.unmeasured).toBe(2); + }); + + it("REGRESSION: reports a window as PARTIAL when history does not reach back across it", () => { + // The headline honesty field. Three days of samples cannot support a 30-day claim, and the reader has to + // be able to tell. + const samples = [sample("operational", 1), sample("operational", 48)]; + expect(computeUptimeWindow(samples, 30, NOW).partial).toBe(true); + expect(computeUptimeWindow(samples, 30, NOW).since).toBe(at(48)); + // The SAME samples fully cover a 1-day window, because recording began 48h before now — well before that + // window starts. This is the case a naive implementation gets wrong: partiality read off the oldest + // IN-WINDOW sample marks every window partial forever, since in-window samples can never predate the + // window start. + const oneDay = computeUptimeWindow(samples, 1, NOW); + expect(oneDay.partial).toBe(false); + expect(oneDay.since).toBe(at(1)); + }); + + it("treats an empty window as partial — it covers none of the period it names", () => { + expect(computeUptimeWindow([], 7, NOW).partial).toBe(true); + }); + + it("ignores samples outside the window and unparseable timestamps", () => { + const window = computeUptimeWindow( + [sample("operational", 1), sample("outage", 24 * 40), { status: "outage", sampledAt: "not-a-date" }], + 7, + NOW, + ); + expect(window.measured).toBe(1); + expect(window.uptime).toBe(1); + }); +}); + +describe("computeIncidents", () => { + it("derives a closed incident from a contiguous run, with the recovery as its end", () => { + const incidents = computeIncidents([sample("operational", 5), sample("outage", 4), sample("outage", 3), sample("operational", 2)], 30, NOW); + expect(incidents).toEqual([{ status: "outage", startedAt: at(4), endedAt: at(2) }]); + }); + + it("REGRESSION: leaves an ongoing incident OPEN rather than closing it at now", () => { + // Closing it at `now` would assert a recovery nobody observed. + const incidents = computeIncidents([sample("operational", 5), sample("outage", 4), sample("outage", 1)], 30, NOW); + expect(incidents).toEqual([{ status: "outage", startedAt: at(4), endedAt: null }]); + }); + + it("REGRESSION: an unknown tick does not split one incident into two", () => { + // Our blindness is not a recovery. Splitting here would manufacture a second incident out of a failed + // read, and double the incident count on any night the alerting source was flaky. + const incidents = computeIncidents([sample("outage", 5), sample("unknown", 4), sample("outage", 3), sample("operational", 2)], 30, NOW); + expect(incidents).toHaveLength(1); + expect(incidents[0]).toMatchObject({ startedAt: at(5), endedAt: at(2) }); + }); + + it("an unknown tick alone never starts an incident", () => { + expect(computeIncidents([sample("operational", 3), sample("unknown", 2), sample("operational", 1)], 30, NOW)).toEqual([]); + }); + + it("takes the WORST status in a run, so an escalation is one incident and not two", () => { + const incidents = computeIncidents([sample("degraded", 4), sample("outage", 3), sample("operational", 2)], 30, NOW); + expect(incidents).toHaveLength(1); + expect(incidents[0]?.status).toBe("outage"); + }); + + it("separates genuinely distinct incidents", () => { + const incidents = computeIncidents( + [sample("outage", 8), sample("operational", 7), sample("degraded", 4), sample("operational", 3)], + 30, + NOW, + ); + expect(incidents.map((incident) => incident.status)).toEqual(["outage", "degraded"]); + }); + + it("sorts unordered input before folding, so row order cannot change the answer", () => { + const shuffled = [sample("operational", 2), sample("outage", 4), sample("operational", 5), sample("outage", 3)]; + expect(computeIncidents(shuffled, 30, NOW)).toEqual([{ status: "outage", startedAt: at(4), endedAt: at(2) }]); + }); + + it("returns nothing for an all-operational history", () => { + expect(computeIncidents([sample("operational", 2), sample("operational", 1)], 30, NOW)).toEqual([]); + }); +}); + +describe("buildComponentHistory", () => { + it("reports every declared window, and incidents over the widest one", () => { + const history = buildComponentHistory([sample("operational", 1), sample("outage", 24 * 20), sample("operational", 24 * 19)], NOW); + expect(history.uptime.map((w) => w.windowDays)).toEqual([...UPTIME_WINDOW_DAYS]); + // The 20-day-old outage is outside 1d and 7d but inside 30d, so only the widest window sees it — and the + // incident list is computed over that same widest window, so the two describe the same period. + expect(history.uptime.find((w) => w.windowDays === 1)?.measured).toBe(1); + expect(history.incidents).toHaveLength(1); + }); +}); + + +describe("persistence, over a real migrated D1", () => { + it("round-trips a sample per component", async () => { + const env = createTestEnv(); + const now = new Date(NOW); + await recordServiceStatusSamples(env, [{ component: "review", status: "operational" }, { component: "testing", status: "outage" }], now); + + const review = await loadServiceStatusSamples(env, "review", now); + expect(review).toEqual([{ status: "operational", sampledAt: now.toISOString() }]); + // Scoped per component: the testing outage must not appear in the review history. + expect((await loadServiceStatusSamples(env, "testing", now))[0]?.status).toBe("outage"); + }); + + it("stores `unknown` rather than skipping it, so blindness stays visible as unmeasured time", async () => { + // Dropping these rows would silently shrink the window and let a period we could not see read as uptime. + const env = createTestEnv(); + const now = new Date(NOW); + await recordServiceStatusSamples(env, [{ component: "review", status: "unknown" }], now); + const history = buildComponentHistory(await loadServiceStatusSamples(env, "review", now), NOW); + const day = history.uptime.find((window) => window.windowDays === 1)!; + expect(day.unmeasured).toBe(1); + expect(day.uptime).toBeNull(); + }); + + it("BEST EFFORT: a write failure never throws, because telemetry must not fail the tick carrying it", async () => { + const broken = createTestEnv(); + broken.DB = { prepare: () => { throw new Error("boom"); } } as never; + await expect(recordServiceStatusSamples(broken, [{ component: "review", status: "operational" }])).resolves.toBeUndefined(); + }); + + it("FAIL-SAFE: a read failure yields no samples, which reports as unmeasured rather than as healthy", async () => { + const broken = createTestEnv(); + broken.DB = { prepare: () => { throw new Error("boom"); } } as never; + expect(await loadServiceStatusSamples(broken, "review")).toEqual([]); + expect(buildComponentHistory([], NOW).uptime.every((window) => window.uptime === null && window.partial)).toBe(true); + }); + + it("excludes samples older than the widest reported window", async () => { + const env = createTestEnv(); + const now = new Date(NOW); + await recordServiceStatusSamples(env, [{ component: "review", status: "outage" }], new Date(NOW - 40 * 86_400_000)); + await recordServiceStatusSamples(env, [{ component: "review", status: "operational" }], now); + expect(await loadServiceStatusSamples(env, "review", now)).toHaveLength(1); + }); +}); + +describe("retention", () => { + it("INVARIANT: retention EXCEEDS the widest reported window, so the figure never loses its own tail", () => { + // A retention equal to the window would make the oldest day of every month quietly unmeasured. This is + // the inequality that lets the 30-day figure stay whole without a rollup -- and widening + // UPTIME_WINDOW_DAYS past it would break it silently, so it is asserted rather than commented. + const days = retentionDaysForTable("service_status_samples"); + expect(days).not.toBeNull(); + expect(days!).toBeGreaterThan(Math.max(...UPTIME_WINDOW_DAYS)); + }); + + it("is registered in the policy at all", () => { + expect(RETENTION_POLICY.some((rule) => rule.table === "service_status_samples")).toBe(true); + }); +}); + +describe("GET /v1/public/service-status with history", () => { + it("attaches uptime and incidents per component", async () => { + const env = createTestEnv({ LOOPOVER_ALERTMANAGER_URL: "http://alertmanager:9093" } as Partial); + await recordServiceStatusSamples(env, [{ component: "review", status: "operational" }], new Date()); + + const response = await createApp().request("/v1/public/service-status", {}, env); + expect(response.status).toBe(200); + const body = (await response.json()) as { components: { component: string; uptime: unknown[]; incidents: unknown[] }[] }; + const review = body.components.find((entry) => entry.component === "review"); + expect(review?.uptime).toHaveLength(UPTIME_WINDOW_DAYS.length); + expect(Array.isArray(review?.incidents)).toBe(true); + }); +});