diff --git a/src/review/ops-wire.ts b/src/review/ops-wire.ts index 44dc72f85a..97dc988658 100644 --- a/src/review/ops-wire.ts +++ b/src/review/ops-wire.ts @@ -136,6 +136,46 @@ const REVIEW_FAILURE_BURST_THRESHOLD = 3; * same-tick anomaly to alert on. */ const BYOK_USAGE_WINDOW_HOURS = 24; +// ── Previous-tick anomaly state (#8903) — so a cleared anomaly can auto-resolve its PagerDuty incident ────── + +/** system_flags (migration 0054) key for "did `repoFullName` have anomalies on the PREVIOUS runOpsAlerts + * tick?" -- reuses the SAME generic key/value table outcomes-wire.ts's readUntrustworthyRuleCodes / + * writeUntrustworthyRuleCodes already do for an identical "cheap cron-tick state, no new migration" shape, + * and the SAME `:` key convention as that table's holdonly:/closehold: families. */ +function opsAnomalyActiveFlagKey(repoFullName: string): string { + return `ops_anomaly_active:${repoFullName}`; +} + +/** Did `repoFullName` have anomalies on the PREVIOUS tick? FAIL-SAFE: a read error or missing row degrades + * to `false` ("not previously active") so a D1 blip can never spuriously fire a PagerDuty resolve for an + * incident that was never actually triggered. */ +async function readOpsAnomalyWasActive(env: Env, repoFullName: string): Promise { + try { + const row = await env.DB.prepare("SELECT value FROM system_flags WHERE key = ?").bind(opsAnomalyActiveFlagKey(repoFullName)).first<{ value: string }>(); + return row?.value === "1"; + } catch { + return false; + } +} + +/** Persist whether `repoFullName` has anomalies on THIS tick, read back by {@link readOpsAnomalyWasActive} on + * the next one. Best-effort (mirrors writeUntrustworthyRuleCodes): a write failure is swallowed -- the next + * tick's read simply serves the last successfully-written value, and this tick's write is retried then. */ +async function writeOpsAnomalyActive(env: Env, repoFullName: string, active: boolean): Promise { + const key = opsAnomalyActiveFlagKey(repoFullName); + if (active) { + await env.DB.prepare("INSERT OR REPLACE INTO system_flags (key, value, updated_at) VALUES (?, '1', CURRENT_TIMESTAMP)") + .bind(key) + .run() + .catch(() => undefined); + } else { + await env.DB.prepare("DELETE FROM system_flags WHERE key = ?") + .bind(key) + .run() + .catch(() => undefined); + } +} + /** One repo's outcome reports + the repo it covers — the input to the pure anomaly detector. `reviewBurst` and * `reviewFailureBurst` are optional so existing snapshot-fixture tests need not be touched; absent/null means * "not computed", not "healthy" -- the caller (runOpsAlerts/computeOpsStats) always populates both today. */ @@ -282,7 +322,18 @@ export async function runOpsAlerts(env: Env): Promise> findHottestInconclusiveReviewTargetForRepo(env, repoFullName, reviewBurstSinceIso), ]); const anomalies = detectOutcomeAnomalies({ repoFullName, gatePrecision, calibration, reviewBurst, reviewFailureBurst }); - if (anomalies.length === 0) continue; + if (anomalies.length === 0) { + // #8903: the repo is healthy THIS tick -- if it had anomalies LAST tick, close the loop by sending a + // matching PagerDuty resolve for the same dedupKey (triggerPagerDutyIncident itself is the no-op + // when PagerDuty isn't configured, exactly like the trigger call below). State tracking runs + // unconditionally (not gated on the PagerDuty flag) so enabling/disabling PagerDuty mid-incident can + // never desync from the actual anomaly history. + if (await readOpsAnomalyWasActive(env, repoFullName)) { + await writeOpsAnomalyActive(env, repoFullName, false); + await triggerPagerDutyIncident(env, { repoFullName, dedupKey: `ops_anomaly:${repoFullName}`, eventAction: "resolve" }); + } + continue; + } found[repoFullName] = anomalies; // Structured log = loopover's notify path (no Discord/operator webhook exists) AND the Sentry path // (level:"error" + an `event` field reaches forwardStructuredLogToSentry). One line per repo. @@ -301,9 +352,9 @@ export async function runOpsAlerts(env: Env): Promise> // comment for why alert fatigue needed both controls, not just PagerDuty's own dedup_key. Awaited (not // fire-and-forget) so a page failure is captured within THIS tick's own error handling, not orphaned // after runOpsAlerts has already returned -- triggerPagerDutyIncident itself never throws and bounds - // its own HTTP call to a 5s timeout, so this cannot hang the scan. This does not yet send a matching - // "resolve" event once anomalies clear (would need tracking previous-tick state) -- an operator - // currently resolves the incident manually once the underlying condition is fixed. + // its own HTTP call to a 5s timeout, so this cannot hang the scan. #8903: a matching "resolve" event + // fires automatically on whichever future tick finds this repo healthy again -- see the + // readOpsAnomalyWasActive branch above -- so an operator no longer resolves the incident by hand. const worst = worstAnomaly(anomalies); await triggerPagerDutyIncident(env, { repoFullName, @@ -312,6 +363,7 @@ export async function runOpsAlerts(env: Env): Promise> dedupKey: `ops_anomaly:${repoFullName}`, customDetails: { anomalies }, }); + await writeOpsAnomalyActive(env, repoFullName, true); // #ops-anomaly-metric: Prometheus counterpart to the log line above so a self-host operator can alert on // /metrics instead of grepping Workers Logs. Scoped to reviewBurst/reviewFailureBurst -- the two anomalies // this module exists to catch fast (#orb-ci-stuck-repeat / #review-burst-blind-spot) -- rather than every diff --git a/src/services/notify-pagerduty.ts b/src/services/notify-pagerduty.ts index 073aeaf4a0..4b6ba032a0 100644 --- a/src/services/notify-pagerduty.ts +++ b/src/services/notify-pagerduty.ts @@ -94,6 +94,26 @@ export function resolvePagerDutyRoutingKey(env: Env, repoFullName: string): Page * concept (#5119) instead of a PagerDuty-only copy. */ export type PagerDutySeverity = LoopoverSeverity; +/** {@link triggerPagerDutyIncident}'s params. A discriminated union on `eventAction` (mirrors this file's + * own {@link PagerDutyRoutingResolution} union style): a `trigger` (the default, so every existing call + * site needs no change) carries the incident's summary/severity/details; a `resolve` (#8903 -- closing + * the incident once the underlying anomaly clears) carries neither -- PagerDuty's Events API v2 only + * requires `payload` for `trigger`, and there is no "worst anomaly" to summarize once anomalies are gone. */ +export type PagerDutyIncidentParams = + | { + repoFullName: string; + dedupKey: string; + eventAction?: "trigger"; + summary: string; + severity: PagerDutySeverity; + customDetails?: Record | undefined; + } + | { + repoFullName: string; + dedupKey: string; + eventAction: "resolve"; + }; + /** Resolve the minimum severity that pages for `repoFullName`: per-repo map entry, else the global override, * else {@link DEFAULT_MIN_SEVERITY} — the quietest safe default, so an operator who never touches these vars * still only gets paged for active-incident-grade conditions, never routine calibration nudges. Delegates to @@ -148,16 +168,8 @@ async function auditPagerDutyNotification( * a below-threshold/cooldown-suppressed page are audited as `denied` so they're discoverable, while the * common "not opted in" case stays silent (no audit-log noise for every repo that never configured * PagerDuty). */ -export async function triggerPagerDutyIncident( - env: Env, - params: { - repoFullName: string; - summary: string; - severity: PagerDutySeverity; - dedupKey: string; - customDetails?: Record | undefined; - }, -): Promise { +export async function triggerPagerDutyIncident(env: Env, params: PagerDutyIncidentParams): Promise { + const eventAction = params.eventAction ?? "trigger"; const resolution = resolvePagerDutyRoutingKey(env, params.repoFullName); if (resolution.status === "disabled") { if (resolution.reason !== "flag_off") { @@ -166,29 +178,35 @@ export async function triggerPagerDutyIncident( return; } - const minSeverity = resolvePagerDutyMinSeverity(env, params.repoFullName); - if (!meetsSeverityThreshold(params.severity, minSeverity)) { - await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "denied", "below_min_severity", { - severity: params.severity, - minSeverity, - }); - return; - } + // The two alert-fatigue controls (min-severity floor, repeat-page cooldown) exist only to stop a PAGE from + // crying wolf -- neither applies to a resolve. Gating a resolve behind "was a page sent for this dedupKey + // recently" would be actively wrong: it would suppress the exact resolve meant to follow the trigger that + // just primed the cooldown window. + if (params.eventAction !== "resolve") { + const minSeverity = resolvePagerDutyMinSeverity(env, params.repoFullName); + if (!meetsSeverityThreshold(params.severity, minSeverity)) { + await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "denied", "below_min_severity", { + severity: params.severity, + minSeverity, + }); + return; + } - const cooldownMinutes = resolvePagerDutyCooldownMinutes(env, params.repoFullName); - const cooldownSinceIso = new Date(Date.now() - cooldownMinutes * 60 * 1000).toISOString(); - // Also count the pre-rebrand "gittensory" actor: a page recorded under the OLD actor value just before this - // rebrand deployed must still suppress a duplicate page after it, for as long as the configured cooldown - // window can reach back across the deploy boundary. Querying both actors costs one extra indexed count and - // removes the whole risk category rather than requiring a precisely-timed follow-up cleanup. - const [recentPagesNewActor, recentPagesLegacyActor] = await Promise.all([ - countRecentAuditEventsForActorAndTarget(env, "loopover", "external_notification.pagerduty", params.dedupKey, cooldownSinceIso), - countRecentAuditEventsForActorAndTarget(env, "gittensory", "external_notification.pagerduty", params.dedupKey, cooldownSinceIso), - ]); - const recentPages = recentPagesNewActor + recentPagesLegacyActor; - if (recentPages > 0) { - await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "denied", "cooldown_active", { cooldownMinutes }); - return; + const cooldownMinutes = resolvePagerDutyCooldownMinutes(env, params.repoFullName); + const cooldownSinceIso = new Date(Date.now() - cooldownMinutes * 60 * 1000).toISOString(); + // Also count the pre-rebrand "gittensory" actor: a page recorded under the OLD actor value just before this + // rebrand deployed must still suppress a duplicate page after it, for as long as the configured cooldown + // window can reach back across the deploy boundary. Querying both actors costs one extra indexed count and + // removes the whole risk category rather than requiring a precisely-timed follow-up cleanup. + const [recentPagesNewActor, recentPagesLegacyActor] = await Promise.all([ + countRecentAuditEventsForActorAndTarget(env, "loopover", "external_notification.pagerduty", params.dedupKey, cooldownSinceIso), + countRecentAuditEventsForActorAndTarget(env, "gittensory", "external_notification.pagerduty", params.dedupKey, cooldownSinceIso), + ]); + const recentPages = recentPagesNewActor + recentPagesLegacyActor; + if (recentPages > 0) { + await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "denied", "cooldown_active", { cooldownMinutes }); + return; + } } try { @@ -197,21 +215,26 @@ export async function triggerPagerDutyIncident( headers: { "content-type": "application/json" }, body: JSON.stringify({ routing_key: resolution.routingKey, - event_action: "trigger", + event_action: eventAction, dedup_key: params.dedupKey, - payload: { - summary: params.summary.slice(0, 1024), - source: "loopover", - severity: params.severity, - timestamp: new Date().toISOString(), - component: params.repoFullName, - custom_details: params.customDetails, - }, + payload: + params.eventAction === "resolve" + ? undefined + : { + summary: params.summary.slice(0, 1024), + source: "loopover", + severity: params.severity, + timestamp: new Date().toISOString(), + component: params.repoFullName, + custom_details: params.customDetails, + }, }), signal: AbortSignal.timeout(5000), }); if (!response.ok) throw new Error(`pagerduty_events_http_${response.status}`); - await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "completed", "triggered", { source: resolution.source }); + await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "completed", eventAction === "resolve" ? "resolved" : "triggered", { + source: resolution.source, + }); } catch (error) { const message = errorMessage(error); console.warn(JSON.stringify({ event: "pagerduty_trigger_failed", repo: params.repoFullName, message: message.slice(0, 200) })); diff --git a/test/unit/notify-pagerduty.test.ts b/test/unit/notify-pagerduty.test.ts index 9b01efc361..be2179a401 100644 --- a/test/unit/notify-pagerduty.test.ts +++ b/test/unit/notify-pagerduty.test.ts @@ -53,6 +53,15 @@ function trigger(env: Env, over: Partial<{ repoFullName: string; summary: string }); } +function resolve(env: Env, over: Partial<{ repoFullName: string; dedupKey: string }> = {}): Promise { + return triggerPagerDutyIncident(env, { + repoFullName: "acme/widgets", + dedupKey: "ops_anomaly:acme/widgets", + eventAction: "resolve", + ...over, + }); +} + describe("isPagerDutyEnabled", () => { it("accepts the codebase-standard truthy strings, case-insensitively", () => { for (const value of ["1", "true", "YES", "On"]) expect(isPagerDutyEnabled({ LOOPOVER_ENABLE_PAGERDUTY: value })).toBe(true); @@ -284,3 +293,54 @@ describe("triggerPagerDutyIncident — HTTP delivery", () => { warn.mockRestore(); }); }); + +// #8903: `eventAction: "resolve"` closes the incident once the underlying anomaly clears. +describe("triggerPagerDutyIncident — eventAction: resolve (#8903)", () => { + it("still respects the flag/routing gate: flag off → no fetch and no audit row", async () => { + const calls = stubFetch(); + const env = withEnv(); + await resolve(env); + expect(calls).toEqual([]); + expect(await pagerDutyAudit(env)).toEqual([]); + }); + + it("still respects the flag/routing gate: flag on, no routing key resolves → no fetch, audited denied", async () => { + const calls = stubFetch(); + const env = withEnv({ LOOPOVER_ENABLE_PAGERDUTY: "1" }); + await resolve(env); + expect(calls).toEqual([]); + expect(await pagerDutyAudit(env)).toEqual([expect.objectContaining({ outcome: "denied", detail: "missing_global_key" })]); + }); + + it("bypasses the min-severity gate entirely — fires even below the default error floor (there is no severity to gate on)", async () => { + const calls = stubFetch(); + await resolve(enabledEnv()); + expect(calls).toHaveLength(1); + }); + + it("bypasses the cooldown gate entirely — fires right after the trigger that primed the SAME dedupKey's cooldown window", async () => { + const calls = stubFetch(); + const env = enabledEnv(); + await trigger(env); + await resolve(env); + expect(calls).toHaveLength(2); + expect(calls[1]?.body.event_action).toBe("resolve"); + }); + + it("posts event_action: resolve with no payload key, and audits detail: resolved on success", async () => { + const calls = stubFetch(202); + const env = enabledEnv(); + await resolve(env); + expect(calls).toHaveLength(1); + expect(calls[0]?.body).toEqual({ routing_key: VALID_KEY, event_action: "resolve", dedup_key: "ops_anomaly:acme/widgets" }); + expect(Object.prototype.hasOwnProperty.call(calls[0]!.body, "payload")).toBe(false); + expect(await pagerDutyAudit(env)).toEqual([expect.objectContaining({ outcome: "completed", detail: "resolved" })]); + }); + + it("a non-ok response is audited as an error and never throws", async () => { + stubFetch(500); + const env = enabledEnv(); + await expect(resolve(env)).resolves.toBeUndefined(); + expect(await pagerDutyAudit(env)).toEqual([expect.objectContaining({ outcome: "error" })]); + }); +}); diff --git a/test/unit/ops-wire.test.ts b/test/unit/ops-wire.test.ts index be20a58d84..6c7d7b4f3a 100644 --- a/test/unit/ops-wire.test.ts +++ b/test/unit/ops-wire.test.ts @@ -28,6 +28,18 @@ function poisonDbPrepare(env: Env, pattern: RegExp): void { }) as typeof env.DB.prepare; } +// Wrap env.DB.prepare so a statement matching `pattern` prepares and binds normally but its .run() REJECTS -- +// exercises a write-path fail-safe `.catch()` (unlike poisonDbPrepare's synchronous throw, which never reaches +// a handler chained onto .run()'s promise because it happens before .bind()/.run() are even called). +function poisonDbRun(env: Env, pattern: RegExp): void { + const realPrepare = env.DB.prepare.bind(env.DB); + env.DB.prepare = ((sql: string) => { + const statement = realPrepare(sql); + if (pattern.test(sql)) (statement as unknown as { run: () => Promise }).run = () => Promise.reject(new Error("poisoned run")); + return statement; + }) as typeof env.DB.prepare; +} + // ── Pure detector fixtures ──────────────────────────────────────────────────────────────────────────────── const healthySnapshot: RepoOutcomeSnapshot = { @@ -335,6 +347,7 @@ async function seedGateFalsePositiveAnomaly(env: Env, repoFullName: string): Pro describe("runOpsAlerts — cron path over gittensory's outcome data", () => { afterEach(() => { vi.restoreAllMocks(); + vi.useRealTimers(); resetMetrics(); setSelfHostedMetricsMode(false); }); @@ -556,8 +569,8 @@ describe("runOpsAlerts — cron path over gittensory's outcome data", () => { // ── Experimental PagerDuty paging (#4937): fatigue-controlled wiring on top of the anomaly scan ────────── const PD_KEY = "a".repeat(32); - function stubPagerDutyFetch(status = 202): Array<{ body: { dedup_key: string; payload: { summary: string; severity: string } } }> { - const calls: Array<{ body: { dedup_key: string; payload: { summary: string; severity: string } } }> = []; + function stubPagerDutyFetch(status = 202): Array<{ body: { dedup_key: string; event_action: string; payload?: { summary: string; severity: string } } }> { + const calls: Array<{ body: { dedup_key: string; event_action: string; payload?: { summary: string; severity: string } } }> = []; vi.stubGlobal("fetch", async (_url: RequestInfo | URL, init?: RequestInit) => { calls.push({ body: JSON.parse(String(init?.body)) }); return new Response(null, { status }); @@ -579,8 +592,8 @@ describe("runOpsAlerts — cron path over gittensory's outcome data", () => { await runOpsAlerts(env); expect(calls).toHaveLength(1); - expect(calls[0]?.body.payload.severity).toBe("error"); - expect(calls[0]?.body.payload.summary).toMatch(/review burst/); + expect(calls[0]?.body.payload?.severity).toBe("error"); + expect(calls[0]?.body.payload?.summary).toMatch(/review burst/); expect(calls[0]?.body.dedup_key).toBe("ops_anomaly:owner/repo"); }); @@ -611,6 +624,130 @@ describe("runOpsAlerts — cron path over gittensory's outcome data", () => { expect(found["owner/repo"]?.some((a) => /review burst/.test(a))).toBe(true); expect(calls).toEqual([]); }); + + // ── #8903: auto-resolve the PagerDuty incident once a previously-anomalous repo clears ───────────────────── + + it("two consecutive ticks: pages on the anomalous tick, then sends a matching PagerDuty resolve once the review burst ages out of its window", async () => { + const calls = stubPagerDutyFetch(); + const env = createTestEnv({ LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: PD_KEY }); + await seedRegisteredRepo(env, "owner/repo"); + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-01T00:00:00.000Z")); + for (let i = 0; i < 7; i += 1) { + await recordAuditEvent(env, { eventType: "github_app.pr_public_surface_published", actor: "contributor", targetKey: "owner/repo#99", outcome: "completed" }); + } + vi.spyOn(console, "error").mockImplementation(() => {}); + + const tick1 = await runOpsAlerts(env); + expect(tick1["owner/repo"]?.some((a) => /review burst/.test(a))).toBe(true); + expect(calls).toHaveLength(1); + expect(calls[0]?.body).toMatchObject({ dedup_key: "ops_anomaly:owner/repo", event_action: "trigger" }); + + // Advance past REVIEW_BURST_WINDOW_HOURS (2h) -- the same 7 publish events now fall outside the burst + // window, so this tick's own detector genuinely finds the repo healthy (no seeded-data mutation needed). + vi.setSystemTime(new Date("2026-07-01T03:00:00.000Z")); + const tick2 = await runOpsAlerts(env); + expect(tick2["owner/repo"]).toBeUndefined(); + expect(calls).toHaveLength(2); + expect(calls[1]?.body).toMatchObject({ dedup_key: "ops_anomaly:owner/repo", event_action: "resolve" }); + expect(calls[1]?.body.payload).toBeUndefined(); + }); + + it("a healthy repo with no prior anomaly never sends a resolve (nothing to close)", async () => { + const calls = stubPagerDutyFetch(); + const env = createTestEnv({ LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: PD_KEY }); + await seedRegisteredRepo(env, "owner/clean"); + vi.spyOn(console, "error").mockImplementation(() => {}); + + const found = await runOpsAlerts(env); + + expect(found["owner/clean"]).toBeUndefined(); + expect(calls).toEqual([]); + }); + + it("a repo that stays anomalous across two ticks pages (trigger) each tick and never sends a resolve", async () => { + const calls = stubPagerDutyFetch(); + const env = createTestEnv({ LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: PD_KEY }); + await seedRegisteredRepo(env, "owner/repo"); + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-01T00:00:00.000Z")); + // Error-grade (meets the default min-severity floor) so it actually pages, unlike a bare gate/calibration + // nudge (see the "does NOT page for a routine calibration nudge" test above). + for (let i = 0; i < 7; i += 1) { + await recordAuditEvent(env, { eventType: "github_app.pr_public_surface_published", actor: "contributor", targetKey: "owner/repo#99", outcome: "completed" }); + } + vi.spyOn(console, "error").mockImplementation(() => {}); + + await runOpsAlerts(env); // tick 1 + + // +61 minutes: past the default 60-minute PagerDuty cooldown (so tick 2's page isn't cooldown-suppressed) + // but still well inside the 2h review-burst window (so the SAME 7 events still read as anomalous). + vi.setSystemTime(new Date("2026-07-01T01:01:00.000Z")); + await runOpsAlerts(env); // tick 2 -- still anomalous + + expect(calls).toHaveLength(2); + expect(calls.every((c) => c.body.event_action === "trigger")).toBe(true); + }); + + it("fails safe: a system_flags READ error degrades to 'not previously active' -- never spuriously resolves, never crashes the scan", async () => { + const calls = stubPagerDutyFetch(); + const env = createTestEnv({ LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: PD_KEY }); + await seedRegisteredRepo(env, "owner/repo"); + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-01T00:00:00.000Z")); + for (let i = 0; i < 7; i += 1) { + await recordAuditEvent(env, { eventType: "github_app.pr_public_surface_published", actor: "contributor", targetKey: "owner/repo#99", outcome: "completed" }); + } + vi.spyOn(console, "error").mockImplementation(() => {}); + await runOpsAlerts(env); // tick 1 (unpoisoned): pages and sets the flag active + expect(calls).toHaveLength(1); + + vi.setSystemTime(new Date("2026-07-01T03:00:00.000Z")); // burst ages out -> tick 2's OWN detector is healthy + poisonDbPrepare(env, /SELECT value FROM system_flags/i); + const tick2 = await runOpsAlerts(env); // resolves (never throws) despite the poisoned read + + expect(tick2["owner/repo"]).toBeUndefined(); + // The read failing safe to "false" means tick2 believes there was nothing to resolve -- no second call. + expect(calls).toHaveLength(1); + }); + + it("fails safe: a poisoned system_flags INSERT (marking the flag active on a page) never breaks the scan", async () => { + const calls = stubPagerDutyFetch(); + const env = createTestEnv({ LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: PD_KEY }); + await seedRegisteredRepo(env, "owner/repo"); + for (let i = 0; i < 7; i += 1) { + await recordAuditEvent(env, { eventType: "github_app.pr_public_surface_published", actor: "contributor", targetKey: "owner/repo#99", outcome: "completed" }); + } + poisonDbRun(env, /INSERT OR REPLACE INTO system_flags/i); + vi.spyOn(console, "error").mockImplementation(() => {}); + + const found = await runOpsAlerts(env); // resolves (never throws) despite the poisoned write + + expect(found["owner/repo"]?.some((a) => /review burst/.test(a))).toBe(true); + expect(calls).toHaveLength(1); // the trigger page itself still fires -- only the flag persistence is poisoned + expect(calls[0]?.body.event_action).toBe("trigger"); + }); + + it("fails safe: a poisoned system_flags DELETE (clearing the flag on resolve) never breaks the scan", async () => { + const calls = stubPagerDutyFetch(); + const env = createTestEnv({ LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: PD_KEY }); + await seedRegisteredRepo(env, "owner/repo"); + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-01T00:00:00.000Z")); + for (let i = 0; i < 7; i += 1) { + await recordAuditEvent(env, { eventType: "github_app.pr_public_surface_published", actor: "contributor", targetKey: "owner/repo#99", outcome: "completed" }); + } + vi.spyOn(console, "error").mockImplementation(() => {}); + await runOpsAlerts(env); // tick 1 (unpoisoned): sets the flag active + + vi.setSystemTime(new Date("2026-07-01T03:00:00.000Z")); // burst ages out -> tick 2 is healthy + poisonDbRun(env, /DELETE FROM system_flags/i); + const tick2 = await runOpsAlerts(env); // resolves (never throws) despite the poisoned DELETE + + expect(tick2["owner/repo"]).toBeUndefined(); + expect(calls).toHaveLength(2); // the resolve page itself still fires -- only the flag persistence is poisoned + expect(calls[1]?.body.event_action).toBe("resolve"); + }); }); describe("computeOpsStats — cross-repo outcome aggregate", () => {