diff --git a/client/package-lock.json b/client/package-lock.json index 7462d95d71..b0c1acfb46 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -83,6 +83,7 @@ "node_modules/@babel/core": { "version": "7.29.0", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -353,6 +354,7 @@ "node_modules/@emotion/react": { "version": "11.14.0", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -390,6 +392,7 @@ "node_modules/@emotion/styled": { "version": "11.14.1", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -1159,6 +1162,7 @@ "node_modules/@mui/material": { "version": "7.3.7", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.28.4", "@mui/core-downloads-tracker": "^7.3.7", @@ -1263,6 +1267,7 @@ "node_modules/@mui/system": { "version": "7.3.7", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.28.4", "@mui/private-theming": "^7.3.7", @@ -2303,6 +2308,7 @@ "version": "24.5.2", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.12.0" } @@ -2318,6 +2324,7 @@ "node_modules/@types/react": { "version": "18.3.27", "license": "MIT", + "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -2439,6 +2446,7 @@ "version": "8.15.0", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2763,6 +2771,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -3194,7 +3203,8 @@ }, "node_modules/dayjs": { "version": "1.11.13", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/debug": { "version": "4.4.3", @@ -3576,6 +3586,7 @@ "version": "8.57.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -4320,6 +4331,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.27.6" }, @@ -5103,6 +5115,7 @@ "resolved": "https://registry.npmjs.org/maplibre-gl/-/maplibre-gl-5.19.0.tgz", "integrity": "sha512-REhYUN8gNP3HlcIZS6QU2uy8iovl31cXsrNDkCcqWSQbCkcpdYLczqDz5PVIwNH42UQNyvukjes/RoHPDrOUmQ==", "license": "BSD-3-Clause", + "peer": true, "dependencies": { "@mapbox/geojson-rewind": "^0.5.2", "@mapbox/jsonlint-lines-primitives": "^2.0.2", @@ -5750,6 +5763,7 @@ "node_modules/react-dom": { "version": "18.3.1", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -5761,6 +5775,7 @@ "node_modules/react-hook-form": { "version": "7.71.1", "license": "MIT", + "peer": true, "engines": { "node": ">=18.0.0" }, @@ -5960,7 +5975,8 @@ }, "node_modules/redux": { "version": "5.0.1", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/redux-persist": { "version": "6.0.0", @@ -6110,6 +6126,7 @@ "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==", "license": "MIT", + "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -6898,6 +6915,7 @@ "version": "5.9.3", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -7033,6 +7051,7 @@ "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz", "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", diff --git a/client/src/Pages/CreateMonitor/EscalationRulesPanel.tsx b/client/src/Pages/CreateMonitor/EscalationRulesPanel.tsx new file mode 100644 index 0000000000..211c5a86cf --- /dev/null +++ b/client/src/Pages/CreateMonitor/EscalationRulesPanel.tsx @@ -0,0 +1,84 @@ +import Stack from "@mui/material/Stack"; +import FormControl from "@mui/material/FormControl"; +import FormControlLabel from "@mui/material/FormControlLabel"; +import FormHelperText from "@mui/material/FormHelperText"; +import MenuItem from "@mui/material/MenuItem"; +import Switch from "@mui/material/Switch"; + +import { TextField, Select } from "@/Components/inputs"; + +type EscalationChannel = { + _id: string; + name: string; +}; + +type EscalationErrors = { + delayMinutes?: string; + channelId?: string; +}; + +type EscalationRulesPanelProps = { + enabled: boolean; + delayMinutes: string; + channelId: string; + channels: EscalationChannel[]; + errors: EscalationErrors; + onToggle: (enabled: boolean) => void; + onDelayMinutesChange: (value: string) => void; + onChannelChange: (value: string) => void; +}; + +export const EscalationRulesPanel = ({ + enabled, + delayMinutes, + channelId, + channels, + errors, + onToggle, + onDelayMinutesChange, + onChannelChange, +}: EscalationRulesPanelProps) => { + return ( + + onToggle(event.target.checked)} + /> + } + label="Enable escalation" + /> + + {enabled && ( + <> + onDelayMinutesChange(event.target.value)} + inputProps={{ min: 1, step: 1 }} + error={Boolean(errors.delayMinutes)} + helperText={errors.delayMinutes || ""} + /> + + + + {errors.channelId && {errors.channelId}} + + + )} + + ); +}; diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 15b76eab36..f4692cab4d 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -32,6 +32,7 @@ import { import { SPACING, LAYOUT } from "@/Utils/Theme/constants"; import { useGet, usePost, usePatch, useDelete } from "@/Hooks/UseApi"; import { useMonitorForm } from "@/Hooks/useMonitorForm"; +import { EscalationRulesPanel } from "./EscalationRulesPanel"; import { type Monitor, type MonitorType, @@ -212,6 +213,13 @@ const CreateMonitorPage = () => { const watchedUseAdvancedMatching = watch("useAdvancedMatching") as boolean; const watchGeoCheckEnabled = watch("geoCheckEnabled") as boolean; + const [escalationEnabled, setEscalationEnabled] = useState(false); + const [escalationDelayMinutes, setEscalationDelayMinutes] = useState("15"); + const [escalationChannelId, setEscalationChannelId] = useState(""); + const [escalationErrors, setEscalationErrors] = useState<{ + delayMinutes?: string; + channelId?: string; + }>({}); useEffect(() => { clearErrors(); @@ -222,8 +230,27 @@ const CreateMonitorPage = () => { [watchedType, t] ); - const { post, loading: isCreating } = usePost(); - const { patch, loading: isUpdating } = usePatch(); + const escalationChannels = useMemo( + () => + (notifications ?? []).map((channel) => ({ + _id: channel.id, + name: channel.notificationName, + })), + [notifications] + ); + + const { post, loading: isCreating } = usePost< + MonitorFormData & { + escalation?: { delayMinutes: number; channelId: string }; + }, + Monitor + >(); + const { patch, loading: isUpdating } = usePatch< + MonitorFormData & { + escalation?: { delayMinutes: number; channelId: string }; + }, + Monitor + >(); const isSubmitting = isCreating || isUpdating; // Delete functionality const [isDeleteDialogOpen, setIsDeleteDialogOpen] = useState(false); @@ -251,12 +278,69 @@ const CreateMonitorPage = () => { setIsDeleteDialogOpen(false); }; + useEffect(() => { + const escalationData = (existingMonitor as Monitor & { + escalation?: { delayMinutes: number; channelId: string }; + })?.escalation; + if (!escalationData) { + setEscalationEnabled(false); + setEscalationDelayMinutes("15"); + setEscalationChannelId(""); + setEscalationErrors({}); + return; + } + + setEscalationEnabled(true); + setEscalationDelayMinutes(String(escalationData.delayMinutes)); + setEscalationChannelId(escalationData.channelId); + setEscalationErrors({}); + }, [existingMonitor]); + + const validateEscalation = () => { + if (!escalationEnabled) { + setEscalationErrors({}); + return true; + } + + const nextErrors: { delayMinutes?: string; channelId?: string } = {}; + const delay = Number(escalationDelayMinutes); + + if (!Number.isInteger(delay) || delay < 1) { + nextErrors.delayMinutes = "Delay must be a positive integer"; + } + + if (!escalationChannelId) { + nextErrors.channelId = "Please select an escalation channel"; + } + + setEscalationErrors(nextErrors); + return Object.keys(nextErrors).length === 0; + }; + const onSubmit = async (data: MonitorFormData) => { + if (!validateEscalation()) { + return; + } + + const payload: MonitorFormData & { + escalation?: { delayMinutes: number; channelId: string }; + } = { + ...data, + ...(escalationEnabled + ? { + escalation: { + delayMinutes: Number(escalationDelayMinutes), + channelId: escalationChannelId, + }, + } + : {}), + }; + let result; if (isEditMode && monitorId) { - result = await patch(`/monitors/${monitorId}`, data); + result = await patch(`/monitors/${monitorId}`, payload); } else { - result = await post("/monitors", data); + result = await post("/monitors", payload); } if (result?.success) { @@ -1044,6 +1128,48 @@ const CreateMonitorPage = () => { /> )} + { + setEscalationEnabled(enabled); + if (!enabled) { + setEscalationErrors({}); + } + }} + onDelayMinutesChange={(value) => { + setEscalationDelayMinutes(value); + if (escalationEnabled) { + const delay = Number(value); + setEscalationErrors((prev) => ({ + ...prev, + delayMinutes: + Number.isInteger(delay) && delay > 0 + ? undefined + : "Delay must be a positive integer", + })); + } + }} + onChannelChange={(value) => { + setEscalationChannelId(value); + if (escalationEnabled) { + setEscalationErrors((prev) => ({ + ...prev, + channelId: value ? undefined : "Please select an escalation channel", + })); + } + }} + /> + } + /> + =18" }, @@ -1546,6 +1548,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" } @@ -4337,6 +4340,7 @@ "integrity": "sha512-wGA0NX93b19/dZC1J18tKWVIYWyyF2ZjT9vin/NRu0qzzvfVzWjs04iq2rQ3H65vCTQYlRqs3YHfY7zjdV+9Kw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", @@ -4497,6 +4501,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-24.9.1.tgz", "integrity": "sha512-QoiaXANRkSXK6p0Duvt56W208du4P9Uye9hWLWgGMDTEoKPhuenzNcC4vGUmrNkiOKTlIrBoyNQYNpSwfEZXSg==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -4734,6 +4739,7 @@ "integrity": "sha512-klQbnPAAiGYFyI02+znpBRLyjL4/BrBd0nyWkdC0s/6xFLkXYQ8OoRrSkqacS1ddVxf/LDyODIKbQ5TgKAf/Fg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.1", "@typescript-eslint/types": "8.56.1", @@ -5347,6 +5353,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5925,6 +5932,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.19", "caniuse-lite": "^1.0.30001751", @@ -6867,6 +6875,7 @@ "resolved": "https://registry.npmjs.org/cssnano/-/cssnano-7.1.1.tgz", "integrity": "sha512-fm4D8ti0dQmFPeF8DXSAA//btEmqCOgAc/9Oa3C1LW94h5usNrJEfrON7b4FkPZgnDEn6OUs5NdxiJZmAtGOpQ==", "license": "MIT", + "peer": true, "dependencies": { "cssnano-preset-default": "^7.0.9", "lilconfig": "^3.1.3" @@ -7634,6 +7643,7 @@ "integrity": "sha512-t5aPOpmtJcZcz5UJyY2GbvpDlsK5E8JqRqoKtfiKE3cNh437KIqfJr3A3AKf5k64NPx6d0G3dno6XDY05PqPtw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -7995,6 +8005,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", @@ -9639,6 +9650,7 @@ "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "30.2.0", "@jest/types": "30.2.0", @@ -12509,6 +12521,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -14604,6 +14617,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -14772,6 +14786,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -14934,6 +14949,7 @@ "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/server/src/config/redis.ts b/server/src/config/redis.ts new file mode 100644 index 0000000000..ce4cc96079 --- /dev/null +++ b/server/src/config/redis.ts @@ -0,0 +1,4 @@ +export const bullmqRedisConnection = { + host: process.env.REDIS_HOST ?? "localhost", + port: Number(process.env.REDIS_PORT ?? 6379), +}; diff --git a/server/src/db/migration/0006_addMonitorEscalationField.ts b/server/src/db/migration/0006_addMonitorEscalationField.ts new file mode 100644 index 0000000000..496b6b2b51 --- /dev/null +++ b/server/src/db/migration/0006_addMonitorEscalationField.ts @@ -0,0 +1,29 @@ +import { MonitorModel } from "../models/Monitor.js"; +import { logger } from "@/utils/logger.js"; + +/** + * Backfills missing monitor escalation field to null for consistency. + * MongoDB does not require structural migration for optional fields, + * but we record this migration for release traceability. + */ +export async function addMonitorEscalationField(): Promise { + const SERVICE_NAME = "Migration:AddMonitorEscalationField"; + + try { + logger.info({ service: SERVICE_NAME, message: "Starting migration" }); + + const result = await MonitorModel.updateMany( + { escalation: { $exists: false } }, + { $set: { escalation: null } } + ); + + logger.info({ + service: SERVICE_NAME, + message: `Migration completed. Updated ${result.modifiedCount} monitors.`, + }); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.error({ service: SERVICE_NAME, message: `Migration failed: ${errorMessage}` }); + throw error; + } +} diff --git a/server/src/db/migration/0007_fixCorruptedMonitorNotifications.ts b/server/src/db/migration/0007_fixCorruptedMonitorNotifications.ts new file mode 100644 index 0000000000..c25b7d16bb --- /dev/null +++ b/server/src/db/migration/0007_fixCorruptedMonitorNotifications.ts @@ -0,0 +1,21 @@ +import mongoose from "mongoose"; +import { MonitorModel } from "../models/Monitor.js"; +import { logger } from "@/utils/logger.js"; + +/** + * Repairs monitor notifications field where a stringified object was saved + * instead of ObjectId values. + */ +export async function fixCorruptedMonitorNotifications(): Promise { + const SERVICE_NAME = "Migration:FixCorruptedMonitorNotifications"; + + await MonitorModel.updateOne( + { _id: new mongoose.Types.ObjectId("69d6d089cd86851fff8bc9b5") }, + { $set: { notifications: [new mongoose.Types.ObjectId("69d6d030cd86851fff8bc9ae")] } } + ); + + logger.info({ + service: SERVICE_NAME, + message: "Patched monitor 69d6d089cd86851fff8bc9b5 notifications field", + }); +} diff --git a/server/src/db/migration/index.ts b/server/src/db/migration/index.ts index f1a672af9a..6d319cf142 100644 --- a/server/src/db/migration/index.ts +++ b/server/src/db/migration/index.ts @@ -4,6 +4,7 @@ import { cleanupDuplicateMonitorStats } from "./0003_cleanupDuplicateMonitorStat import { fixInfrastructureThresholds } from "./0004_fixInfrastructureThresholds.js"; import MigrationModel from "../models/Migration.js"; import { migrateStatusPageTypeToArray } from "./0005_migrateStatusPageTypeToArray.js"; +import { fixCorruptedMonitorNotifications } from "./0007_fixCorruptedMonitorNotifications.js"; import type { ILogger } from "@/utils/logger.js"; type MigrationEntry = { @@ -17,6 +18,7 @@ const migrations: MigrationEntry[] = [ { name: "0003_cleanupDuplicateMonitorStats", execute: cleanupDuplicateMonitorStats }, { name: "0004_fixInfrastructureThresholds", execute: fixInfrastructureThresholds }, { name: "0005_migrateStatusPageTypeToArray", execute: migrateStatusPageTypeToArray }, + { name: "0007_fixCorruptedMonitorNotifications", execute: fixCorruptedMonitorNotifications }, ]; const runMigrations = async (logger?: ILogger) => { diff --git a/server/src/db/models/Incident.ts b/server/src/db/models/Incident.ts index 82e2b5eb2b..228dcc2908 100644 --- a/server/src/db/models/Incident.ts +++ b/server/src/db/models/Incident.ts @@ -31,6 +31,10 @@ const IncidentSchema = new Schema( immutable: true, index: true, }, + escalationJobId: { + type: String, + default: null, + }, startTime: { type: Date, immutable: true, diff --git a/server/src/db/models/Monitor.ts b/server/src/db/models/Monitor.ts index 036aeadad6..253d0eeea0 100644 --- a/server/src/db/models/Monitor.ts +++ b/server/src/db/models/Monitor.ts @@ -16,15 +16,21 @@ import type { type CheckSnapshotDocument = Omit & { createdAt: Date }; +type EscalationConfigDocument = { + delayMinutes: number; + channelId: Types.ObjectId; +}; + type MonitorDocumentBase = Omit< Monitor, - "id" | "userId" | "teamId" | "notifications" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt" + "id" | "userId" | "teamId" | "notifications" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt" | "escalation" > & { statusWindow: boolean[]; recentChecks: CheckSnapshotDocument[]; notifications: Types.ObjectId[]; selectedDisks: string[]; matchMethod?: MonitorMatchMethod; + escalation?: EscalationConfigDocument; }; interface MonitorDocument extends MonitorDocumentBase { @@ -198,6 +204,14 @@ const checkSnapshotSchema = new Schema( { _id: false } ); +const escalationConfigSchema = new Schema( + { + delayMinutes: { type: Number, min: 1, required: true }, + channelId: { type: Schema.Types.ObjectId, ref: "Notification", required: true }, + }, + { _id: false } +); + const MonitorSchema = new Schema( { userId: { @@ -284,6 +298,10 @@ const MonitorSchema = new Schema( ref: "Notification", }, ], + escalation: { + type: escalationConfigSchema, + required: false, + }, secret: { type: String, }, diff --git a/server/src/repositories/incidents/MongoIncidentRepository.ts b/server/src/repositories/incidents/MongoIncidentRepository.ts index 096ba3d37b..a6670ca05d 100644 --- a/server/src/repositories/incidents/MongoIncidentRepository.ts +++ b/server/src/repositories/incidents/MongoIncidentRepository.ts @@ -51,6 +51,7 @@ class MongoIncidentRepository implements IIncidentsRepository { id: this.toStringId(doc._id), monitorId: this.toStringId(doc.monitorId), teamId: this.toStringId(doc.teamId), + escalationJobId: doc.escalationJobId ?? null, startTime: this.toDateString(doc.startTime), endTime: doc.endTime ? this.toDateString(doc.endTime) : null, status: doc.status, diff --git a/server/src/repositories/monitors/MongoMonitorsRepository.ts b/server/src/repositories/monitors/MongoMonitorsRepository.ts index b2d7594483..ba8e74f6fd 100644 --- a/server/src/repositories/monitors/MongoMonitorsRepository.ts +++ b/server/src/repositories/monitors/MongoMonitorsRepository.ts @@ -338,6 +338,28 @@ class MongoMonitorsRepository implements IMonitorsRepository { return documents.map((doc) => this.toEntity(doc)); }; + private toNotificationIdString = (notification: unknown): string => { + if (notification instanceof mongoose.Types.ObjectId) { + return notification.toString(); + } + + if (typeof notification === "string") { + return notification; + } + + if (notification && typeof notification === "object" && "notificationId" in notification) { + const maybeNotificationId = (notification as { notificationId?: unknown }).notificationId; + if (maybeNotificationId instanceof mongoose.Types.ObjectId) { + return maybeNotificationId.toString(); + } + if (typeof maybeNotificationId === "string") { + return maybeNotificationId; + } + } + + return ""; + }; + private toEntity = (doc: MonitorDocument): Monitor => { const toStringId = (value: unknown): string => { if (value instanceof mongoose.Types.ObjectId) { @@ -350,7 +372,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { return value instanceof Date ? value.toISOString() : value; }; - const notificationIds = (doc.notifications ?? []).map((notification) => toStringId(notification)); + const notificationIds = (doc.notifications ?? []).map((notification) => this.toNotificationIdString(notification)).filter(Boolean); return { id: toStringId(doc._id), @@ -387,6 +409,12 @@ class MongoMonitorsRepository implements IMonitorsRepository { gameId: doc.gameId ?? undefined, grpcServiceName: doc.grpcServiceName ?? undefined, group: doc.group ?? null, + escalation: doc.escalation + ? { + delayMinutes: doc.escalation.delayMinutes, + channelId: toStringId(doc.escalation.channelId), + } + : undefined, recentChecks: (doc.recentChecks ?? []).map((check: CheckSnapshotDocument) => this.toCheckSnapshot(check)), geoCheckEnabled: doc.geoCheckEnabled ?? false, geoCheckLocations: doc.geoCheckLocations ?? [], @@ -409,7 +437,9 @@ class MongoMonitorsRepository implements IMonitorsRepository { return value instanceof Date ? value.toISOString() : value; }; - const notificationIds = (doc.notifications ?? []).map((notification: unknown) => toStringId(notification)); + const notificationIds = (doc.notifications ?? []) + .map((notification: unknown) => this.toNotificationIdString(notification)) + .filter(Boolean); return { id: toStringId(doc._id), @@ -446,6 +476,12 @@ class MongoMonitorsRepository implements IMonitorsRepository { gameId: doc.gameId ?? undefined, grpcServiceName: doc.grpcServiceName ?? undefined, group: doc.group ?? null, + escalation: doc.escalation + ? { + delayMinutes: doc.escalation.delayMinutes, + channelId: toStringId(doc.escalation.channelId), + } + : undefined, recentChecks: (doc.recentChecks ?? []).map((check: CheckSnapshotDocument) => this.toCheckSnapshot(check)), geoCheckEnabled: doc.geoCheckEnabled ?? false, geoCheckLocations: doc.geoCheckLocations ?? [], diff --git a/server/src/service/business/incidentService.ts b/server/src/service/business/incidentService.ts index 4790f9aacc..7e377b0381 100644 --- a/server/src/service/business/incidentService.ts +++ b/server/src/service/business/incidentService.ts @@ -8,6 +8,8 @@ import type { Incident, IncidentSummary, User } from "@/types/index.js"; import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import type { INotificationMessageBuilder } from "@/service/infrastructure/notificationMessageBuilder.js"; import type { ILogger } from "@/utils/logger.js"; +import { Queue } from "bullmq"; +import { bullmqRedisConnection } from "@/config/redis.js"; export interface IIncidentService { handleIncident( @@ -39,6 +41,7 @@ export class IncidentService implements IIncidentService { private monitorsRepository: IMonitorsRepository; private usersRepository: IUsersRepository; private notificationMessageBuilder: INotificationMessageBuilder; + private escalationQueue: Queue; constructor( logger: ILogger, @@ -52,6 +55,7 @@ export class IncidentService implements IIncidentService { this.monitorsRepository = monitorsRepository; this.usersRepository = usersRepository; this.notificationMessageBuilder = notificationMessageBuilder; + this.escalationQueue = new Queue("escalation", { connection: bullmqRedisConnection }); } get serviceName() { @@ -86,6 +90,7 @@ export class IncidentService implements IIncidentService { const incident = { monitorId: monitor.id, teamId: monitor.teamId, + escalationJobId: monitor.escalation ? `escalation-${monitor.id}` : null, startTime: Date.now().toString(), status: true, statusCode, @@ -146,8 +151,17 @@ export class IncidentService implements IIncidentService { throw new AppError({ message: "Incident is already resolved", service: SERVICE_NAME, method: "resolveIncident" }); } + if (incident.escalationJobId) { + const pendingEscalationJob = await this.escalationQueue.getJob(incident.escalationJobId); + // job may already be null if escalation fired before manual resolution — safe to skip + if (pendingEscalationJob) { + await pendingEscalationJob.remove(); + } + } + incident.resolutionType = "manual"; incident.status = false; + incident.escalationJobId = null; incident.resolvedBy = userId; incident.resolvedByEmail = userEmail || null; incident.comment = comment || null; diff --git a/server/src/service/business/monitorService.ts b/server/src/service/business/monitorService.ts index 71c9d9d906..20344b77d4 100644 --- a/server/src/service/business/monitorService.ts +++ b/server/src/service/business/monitorService.ts @@ -437,7 +437,11 @@ export class MonitorService implements IMonitorService { }; editMonitor = async ({ teamId, monitorId, body }: { teamId: string; monitorId: string; body: Partial }) => { - const editedMonitor = await this.monitorsRepository.updateById(monitorId, teamId, body); + const updatePayload: Partial = { + ...body, + escalation: body.escalation ?? null, + }; + const editedMonitor = await this.monitorsRepository.updateById(monitorId, teamId, updatePayload); await this.jobQueue.updateJob(editedMonitor); return editedMonitor; }; diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index b6908127b2..e154b80825 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -1,6 +1,13 @@ const SERVICE_NAME = "JobQueueHelper"; import type { Monitor } from "@/types/monitor.js"; import { supportsGeoCheck } from "@/types/monitor.js"; +import { Queue, Worker } from "bullmq"; + +type EscalationJobData = { + monitorId: string; + channelId: string; + teamId: string; +}; import { AppError } from "@/utils/AppError.js"; import { ICheckService, @@ -11,7 +18,7 @@ import { IncidentService, type IGeoChecksService, } from "@/service/index.js"; -import { CHECK_TTL_SENTINEL, type MaintenanceWindow, type StatusChangeResult } from "@/types/index.js"; +import { CHECK_TTL_SENTINEL, type MaintenanceWindow, type MonitorStatusResponse, type StatusChangeResult } from "@/types/index.js"; import { IMaintenanceWindowsRepository, IMonitorsRepository, @@ -23,6 +30,7 @@ import { } from "@/repositories/index.js"; import { ILogger } from "@/utils/logger.js"; import { IBufferService } from "@/service/index.js"; +import { bullmqRedisConnection } from "@/config/redis.js"; export interface ISuperSimpleQueueHelper { readonly serviceName: string; @@ -31,6 +39,7 @@ export interface ISuperSimpleQueueHelper { getCleanupOrphanedJob(): () => Promise; getCleanupRetentionJob(): () => Promise; isInMaintenanceWindow(monitorId: string, teamId: string): Promise; + removeEscalationJob(monitorId: string): Promise; } export interface MonitorActionDecision { @@ -66,6 +75,8 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { private incidentsRepository: IIncidentsRepository; private geoChecksService: IGeoChecksService; private geoChecksRepository: IGeoChecksRepository; + private escalationQueue: Queue; + private escalationWorker: Worker; constructor( logger: ILogger, @@ -101,6 +112,95 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { this.incidentsRepository = incidentsRepository; this.geoChecksService = geoChecksService; this.geoChecksRepository = geoChecksRepository; + + this.escalationQueue = new Queue("escalation", { connection: bullmqRedisConnection }); + + this.escalationWorker = new Worker( + "escalation", + async (job) => { + const { monitorId, channelId, teamId } = job.data; + this.logger.info({ + message: `Escalation worker received job ${job.id ?? "unknown"} for monitor ${monitorId}`, + service: SERVICE_NAME, + method: "escalationWorker", + details: { monitorId, channelId, teamId, jobId: job.id }, + }); + + // Bail early if the incident resolved before the delay elapsed + const activeIncident = await this.incidentsRepository.findActiveByMonitorId(monitorId, teamId); + if (!activeIncident) { + this.logger.info({ + message: `Escalation fired but incident already resolved for monitor ${monitorId}`, + service: SERVICE_NAME, + method: "escalationWorker", + }); + return; + } + + // Fetch monitor — log clearly and abort if it no longer exists + let monitor: Monitor; + try { + monitor = await this.monitorsRepository.findById(monitorId, teamId); + } catch (error: unknown) { + this.logger.error({ + message: `Escalation worker: monitor ${monitorId} not found — cannot send escalation alert`, + service: SERVICE_NAME, + method: "escalationWorker", + stack: error instanceof Error ? error.stack : undefined, + }); + return; + } + + try { + // Validate the escalation channel still exists before dispatching + try { + await this.notificationsService.findById(channelId, teamId); + } catch (error: unknown) { + this.logger.error({ + message: `Escalation worker: notification channel ${channelId} not found for monitor ${monitorId} — cannot send escalation alert`, + service: SERVICE_NAME, + method: "escalationWorker", + stack: error instanceof Error ? error.stack : undefined, + }); + return; + } + + // Use the same notification path as real monitor-down alerts. + // Override notifications so only the escalation channel receives this alert. + const escalationMonitor: Monitor = { ...monitor, notifications: [channelId] }; + const syntheticResponse: MonitorStatusResponse = { + monitorId, + teamId, + type: monitor.type, + status: false, + code: 0, + message: "Escalation alert: monitor has been down beyond the configured threshold", + }; + const decision: MonitorActionDecision = { + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: true, + incidentReason: "status_down", + notificationReason: "status_change", + }; + + await this.notificationsService.handleNotifications(escalationMonitor, syntheticResponse, decision); + this.logger.info({ + message: `Escalation notification sent for monitor ${monitorId}`, + service: SERVICE_NAME, + method: "escalationWorker", + }); + } catch (error: unknown) { + this.logger.error({ + message: `Escalation worker: failed to send escalation alert for monitor ${monitorId}: ${error instanceof Error ? error.message : "Unknown error"}`, + service: SERVICE_NAME, + method: "escalationWorker", + stack: error instanceof Error ? error.stack : undefined, + }); + } + }, + { connection: bullmqRedisConnection } + ); } get serviceName() { @@ -177,6 +277,37 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { stack: error instanceof Error ? error.stack : undefined, }); }); + + // Step 8. Schedule or cancel escalation job (best effort) + if (monitor.escalation) { + this.logger.debug({ + message: `Escalation evaluation for monitor ${monitor.id}`, + service: SERVICE_NAME, + method: "getMonitorJob", + details: { + escalation: monitor.escalation, + shouldCreateIncident: decision.shouldCreateIncident, + shouldResolveIncident: decision.shouldResolveIncident, + }, + }); + if (decision.shouldCreateIncident) { + this.scheduleEscalationJob(monitor).catch((error: unknown) => { + this.logger.error({ + message: `Error scheduling escalation job for monitor ${monitor.id}: ${error instanceof Error ? error.message : "Unknown error"}`, + service: SERVICE_NAME, + method: "getMonitorJob", + }); + }); + } else if (decision.shouldResolveIncident) { + this.removeEscalationJob(monitor.id).catch((error: unknown) => { + this.logger.error({ + message: `Error removing escalation job for monitor ${monitor.id}: ${error instanceof Error ? error.message : "Unknown error"}`, + service: SERVICE_NAME, + method: "getMonitorJob", + }); + }); + } + } } catch (error: unknown) { this.logger.warn({ message: error instanceof Error ? error.message : "Unknown error", @@ -189,6 +320,47 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { }; }; + private scheduleEscalationJob = async (monitor: Monitor): Promise => { + if (!monitor.escalation) return; + const delayMs = monitor.escalation.delayMinutes * 60 * 1000; + const jobId = `escalation-${monitor.id}`; + this.logger.info({ + message: `Adding escalation job for monitor ${monitor.id}`, + service: SERVICE_NAME, + method: "scheduleEscalationJob", + details: { delayMs, jobId }, + }); + await this.escalationQueue.add( + "escalation", + { monitorId: monitor.id, channelId: monitor.escalation.channelId, teamId: monitor.teamId }, + { + delay: delayMs, + // Deterministic ID prevents duplicate jobs if the monitor goes down repeatedly + // before the first escalation fires + jobId, + removeOnComplete: true, + removeOnFail: false, + } + ); + this.logger.info({ + message: `Escalation job scheduled for monitor ${monitor.id} (delay: ${monitor.escalation.delayMinutes}m)`, + service: SERVICE_NAME, + method: "scheduleEscalationJob", + }); + }; + + removeEscalationJob = async (monitorId: string): Promise => { + const job = await this.escalationQueue.getJob(`escalation-${monitorId}`); + if (job) { + await job.remove(); + this.logger.info({ + message: `Escalation job removed for monitor ${monitorId}`, + service: SERVICE_NAME, + method: "removeEscalationJob", + }); + } + }; + getCleanupOrphanedJob = () => { return async () => { try { diff --git a/server/src/service/infrastructure/notificationProviders/email.ts b/server/src/service/infrastructure/notificationProviders/email.ts index b3686651cc..5b83303ca3 100644 --- a/server/src/service/infrastructure/notificationProviders/email.ts +++ b/server/src/service/infrastructure/notificationProviders/email.ts @@ -93,8 +93,12 @@ export class EmailProvider implements INotificationProvider { } private async buildEmailFromMessage(message: NotificationMessage): Promise { + // For escalation alerts (critical monitor_down), use "Escalation" as the header + const isEscalation = message.type === "monitor_down" && message.severity === "critical"; + const title = isEscalation ? "Escalation" : message.content.title; + const context = { - title: message.content.title, + title, summary: message.content.summary, monitorName: message.monitor.name, monitorUrl: message.monitor.url, diff --git a/server/src/types/incident.ts b/server/src/types/incident.ts index 6b076ff835..fb2c01f6cd 100644 --- a/server/src/types/incident.ts +++ b/server/src/types/incident.ts @@ -7,6 +7,7 @@ export interface Incident { id: string; monitorId: string; teamId: string; + escalationJobId?: string | null; startTime: string; endTime: string | null; status: boolean; diff --git a/server/src/types/monitor.ts b/server/src/types/monitor.ts index f29ce75d78..71b4f39504 100644 --- a/server/src/types/monitor.ts +++ b/server/src/types/monitor.ts @@ -52,8 +52,10 @@ export interface Monitor { group: string | null; geoCheckEnabled?: boolean; geoCheckLocations?: GeoContinent[]; - geoCheckInterval?: number; - recentChecks: CheckSnapshot[]; + geoCheckInterval?: number; escalation?: { + delayMinutes: number; + channelId: string; + }; recentChecks: CheckSnapshot[]; createdAt: string; updatedAt: string; } diff --git a/server/src/validation/monitorValidation.ts b/server/src/validation/monitorValidation.ts index df000ecef2..af68677b84 100644 --- a/server/src/validation/monitorValidation.ts +++ b/server/src/validation/monitorValidation.ts @@ -107,6 +107,13 @@ export const editMonitorBodyValidation = z.object({ geoCheckEnabled: z.boolean().optional(), geoCheckLocations: z.array(z.enum(GeoContinents)).optional(), geoCheckInterval: z.number().min(300000).optional(), + escalation: z + .object({ + delayMinutes: z.number().int().min(1), + channelId: z.string().min(1), + }) + .nullable() + .optional(), }); export const pauseMonitorParamValidation = z.object({