From 748b77792169e16b104ef9dfd08e1a00fe089aba Mon Sep 17 00:00:00 2001 From: Sophia Pappous Date: Thu, 9 Apr 2026 15:49:26 -0400 Subject: [PATCH] Created Escalate Notifications using GitHub Copilot --- client/src/Hooks/useMonitorForm.ts | 1 + client/src/Pages/CreateMonitor/index.tsx | 74 ++++++++++++++++++- client/src/Types/Monitor.ts | 4 + client/src/Validation/monitor.ts | 6 ++ client/src/locales/en.json | 12 +++ server/src/db/models/Incident.ts | 7 +- server/src/db/models/Monitor.ts | 10 +++ .../monitors/MongoMonitorsRepository.ts | 12 +++ .../src/service/business/incidentService.ts | 7 +- .../SuperSimpleQueueHelper.ts | 24 +++++- .../notificationMessageBuilder.ts | 4 + .../infrastructure/notificationsService.ts | 61 ++++++++++++++- server/src/types/incident.ts | 1 + server/src/types/monitor.ts | 4 + server/src/validation/monitorValidation.ts | 6 ++ 15 files changed, 225 insertions(+), 8 deletions(-) diff --git a/client/src/Hooks/useMonitorForm.ts b/client/src/Hooks/useMonitorForm.ts index 963409fc8a..38c4e26513 100644 --- a/client/src/Hooks/useMonitorForm.ts +++ b/client/src/Hooks/useMonitorForm.ts @@ -17,6 +17,7 @@ const getBaseDefaults = (data?: Monitor | null) => ({ geoCheckEnabled: data?.geoCheckEnabled ?? false, geoCheckLocations: data?.geoCheckLocations || [], geoCheckInterval: data?.geoCheckInterval || 300000, + escalation: data?.escalation, }); export const useMonitorForm = ({ diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 15b76eab36..fb5e6d9886 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -40,6 +40,8 @@ import { } from "@/Types/Monitor"; import type { Notification } from "@/Types/Notification"; import type { MonitorFormData } from "@/Validation/monitor"; +import { parse } from "zod"; +import { es } from "zod/v4/locales"; interface GeneralSettingsConfig { urlLabel: string; @@ -252,11 +254,16 @@ const CreateMonitorPage = () => { }; const onSubmit = async (data: MonitorFormData) => { + const escalation = data.escalation && data.escalation.escalationDelay > 0 && data.escalation.channelID ? data.escalation : undefined; + const payload = { + ...data, + escalation, + } let result; if (isEditMode && monitorId) { - result = await patch(`/monitors/${monitorId}`, data); + result = await patch(`/monitors/${monitorId}`, payload); } else { - result = await post("/monitors", data); + result = await post("/monitors", payload); } if (result?.success) { @@ -765,6 +772,69 @@ const CreateMonitorPage = () => { } /> + { + const notify = (notifications ?? []).map((n) => ({ + ...n, + name: n.notificationName, + })); + const currEscalation = field.value; + const selectedChannel = notify.find( + (n) => n.id === currEscalation?.channelID + ); + return ( + + { + const escalationDelay = parseInt(e.target.value, 10) || 0; + field.onChange({ + escalationDelay, + channelID: currEscalation?.channelID || "", + }); + }} + fullWidth + /> + + + ); + }} + /> + } + /> + {(watchedType === "http" || watchedType === "grpc" || watchedType === "websocket") && ( diff --git a/client/src/Types/Monitor.ts b/client/src/Types/Monitor.ts index 053b517d1d..fd3131f2ff 100644 --- a/client/src/Types/Monitor.ts +++ b/client/src/Types/Monitor.ts @@ -76,6 +76,10 @@ export interface Monitor { geoCheckEnabled?: boolean; geoCheckLocations?: GeoContinent[]; geoCheckInterval?: number; + escalation?: { + escalationDelay: number; + channelID: string; + }; recentChecks: CheckSnapshot[]; createdAt: string; updatedAt: string; diff --git a/client/src/Validation/monitor.ts b/client/src/Validation/monitor.ts index 9acffe6fed..c091484064 100644 --- a/client/src/Validation/monitor.ts +++ b/client/src/Validation/monitor.ts @@ -1,5 +1,6 @@ import { z } from "zod"; import { GeoContinents } from "@/Types/GeoCheck"; +import { es } from "zod/v4/locales"; // URL schema with custom error message const urlSchema = z.url({ message: "Please enter a valid URL" }); @@ -27,6 +28,11 @@ const baseSchema = z.object({ .number() .min(300000, "Interval must be at least 5 minutes") .optional(), + escalation: z.object({ + escalationDelay: z.number().min(0), + channelID: z.string().min(1),}) + .nullable() + .optional(), }); // HTTP monitor schema diff --git a/client/src/locales/en.json b/client/src/locales/en.json index 92a21939f3..84f1384965 100644 --- a/client/src/locales/en.json +++ b/client/src/locales/en.json @@ -543,6 +543,18 @@ "description": "Select the notification channels you want to use", "title": "Notifications" }, + "escalation":{ + "description": "If the monitor stays down for the specified time, notify additional channels.", + "title": "Escalation Rules", + "escalationDelay": { + "label": "Escalate after (minutes)", + "placeholder": "e.g. 3" + }, + "channelID": { + "label": "Escalation notification channels", + "placeholder": "Type to search" + } + }, "type": { "description": "Select the type of check to perform", "optionDockerDescription": "Use Docker to monitor if a container is running.", diff --git a/server/src/db/models/Incident.ts b/server/src/db/models/Incident.ts index 82e2b5eb2b..afd13836e4 100644 --- a/server/src/db/models/Incident.ts +++ b/server/src/db/models/Incident.ts @@ -1,12 +1,13 @@ import { Schema, model, type Types } from "mongoose"; import { IncidentResolutionTypes, type Incident } from "@/types/incident.js"; -type IncidentDocumentBase = Omit & { +type IncidentDocumentBase = Omit & { monitorId: Types.ObjectId; teamId: Types.ObjectId; resolvedBy?: Types.ObjectId | null; startTime: Date; endTime: Date | null; + escalationTime: Date | null; createdAt: Date; updatedAt: Date; }; @@ -72,6 +73,10 @@ const IncidentSchema = new Schema( type: String, default: null, }, + escalationTime:{ + type: Date, + default: null, + } }, { timestamps: true } ); diff --git a/server/src/db/models/Monitor.ts b/server/src/db/models/Monitor.ts index 036aeadad6..7b46524df8 100644 --- a/server/src/db/models/Monitor.ts +++ b/server/src/db/models/Monitor.ts @@ -351,6 +351,16 @@ const MonitorSchema = new Schema( type: Number, default: 300000, }, + escalation: { + escalationDelay:{ + type: Number, + required: true, + }, + channelID:{ + type: String, + required: true, + }, + }, recentChecks: { type: [checkSnapshotSchema], default: [], diff --git a/server/src/repositories/monitors/MongoMonitorsRepository.ts b/server/src/repositories/monitors/MongoMonitorsRepository.ts index b2d7594483..c2ed3f7f00 100644 --- a/server/src/repositories/monitors/MongoMonitorsRepository.ts +++ b/server/src/repositories/monitors/MongoMonitorsRepository.ts @@ -374,6 +374,12 @@ class MongoMonitorsRepository implements IMonitorsRepository { interval: doc.interval, uptimePercentage: doc.uptimePercentage ?? undefined, notifications: notificationIds, + escalation: doc.escalation + ? { + escalationDelay: doc.escalation.escalationDelay, + channelID: toStringId(doc.escalation.channelID), + } + : undefined, secret: doc.secret ?? undefined, cpuAlertThreshold: doc.cpuAlertThreshold, cpuAlertCounter: doc.cpuAlertCounter, @@ -433,6 +439,12 @@ class MongoMonitorsRepository implements IMonitorsRepository { interval: doc.interval, uptimePercentage: doc.uptimePercentage ?? undefined, notifications: notificationIds, + escalation: doc.escalation + ? { + escalationDelay: doc.escalation.escalationDelay, + channelID: toStringId(doc.escalation.channelID), + } + : undefined, secret: doc.secret ?? undefined, cpuAlertThreshold: doc.cpuAlertThreshold, cpuAlertCounter: doc.cpuAlertCounter, diff --git a/server/src/service/business/incidentService.ts b/server/src/service/business/incidentService.ts index 4790f9aacc..e681fa22e5 100644 --- a/server/src/service/business/incidentService.ts +++ b/server/src/service/business/incidentService.ts @@ -64,7 +64,7 @@ export class IncidentService implements IIncidentService { decision: MonitorActionDecision, monitorStatusResponse?: MonitorStatusResponse ): Promise => { - if (!decision.shouldCreateIncident && !decision.shouldResolveIncident) { + if (!decision.shouldCreateIncident && !decision.shouldResolveIncident && !decision.shouldEscalateIncident) { return null; } @@ -105,6 +105,11 @@ export class IncidentService implements IIncidentService { return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident); } + // Handle incident escalation + if(decision.shouldEscalateIncident && activeIncident && !activeIncident.escalationTime) { + activeIncident.escalationTime = new Date().toISOString(); + return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident); + } return null; }; diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index b6908127b2..69f850f46f 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -45,6 +45,8 @@ export interface MonitorActionDecision { disk?: boolean; temp?: boolean; }; + shouldEscalateIncident?: boolean; + escalationChannelID?: string | null; } export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { @@ -156,8 +158,28 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { // Step 5. Get decisions const decision = this.evaluateMonitorAction(statusChangeResult); + if(statusChangeResult.monitor.escalation && statusChangeResult.monitor.status === "down"){ + const activeIncident = await this.incidentsRepository.findActiveByMonitorId(monitorId, teamId); + if(activeIncident){ + const incidentStart = new Date(activeIncident.createdAt).getTime(); + const nowNow = new Date().getTime(); + const incidentDuration = nowNow - incidentStart; + const escalationDelay = statusChangeResult.monitor.escalation.escalationDelay * 60 * 1000; // Convert minutes to milliseconds + + if (incidentDuration >= escalationDelay && !activeIncident.escalationTime) { + decision.shouldEscalateIncident = true; + decision.escalationChannelID = statusChangeResult.monitor.escalation.channelID; + this.logger.debug({ + message: `Incident for monitor ${monitorId} is eligible for escalation`, + service: SERVICE_NAME, + method: "getHeartbeatJob", + }); + } + } + } + // Step 6. Handle notifications (best effort, continue even in event of failure, don't wait) - if (decision.shouldSendNotification) { + if (decision.shouldSendNotification || decision.shouldEscalateIncident) { this.notificationsService.handleNotifications(statusChangeResult.monitor, status, decision).catch((error: unknown) => { this.logger.error({ message: `Error sending notifications for job ${statusChangeResult.monitor.id}: ${error instanceof Error ? error.message : "Unknown error"}`, diff --git a/server/src/service/infrastructure/notificationMessageBuilder.ts b/server/src/service/infrastructure/notificationMessageBuilder.ts index 934163b2a9..a3f1f44cc2 100644 --- a/server/src/service/infrastructure/notificationMessageBuilder.ts +++ b/server/src/service/infrastructure/notificationMessageBuilder.ts @@ -33,6 +33,10 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { const severity = this.determineSeverity(type); const content = this.buildContent(type, monitor, monitorStatusResponse); + if (decision.shouldEscalateIncident) { + content.title = `[ESCALATION] ${content.title}`; + content.summary = `[ESCALATION] ${content.summary}`; + } return { type, severity, diff --git a/server/src/service/infrastructure/notificationsService.ts b/server/src/service/infrastructure/notificationsService.ts index c75477c88c..82b2768027 100644 --- a/server/src/service/infrastructure/notificationsService.ts +++ b/server/src/service/infrastructure/notificationsService.ts @@ -132,13 +132,68 @@ export class NotificationsService implements INotificationsService { return succeeded === notifications.length; }; - handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { - if (!decision.shouldSendNotification) { + private sendNotifyEscalation = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision, escalationChannelID: string | null | undefined) => { + if (!escalationChannelID) { + this.logger.warn({ + message: "Escalation channel ID not provided for escalation notification", + service: SERVICE_NAME, + method: "sendNotifyEscalation", + }); + return false; + } + + const notification = await this.notificationsRepository.findById(escalationChannelID, monitor.teamId); + if (!notification) { + this.logger.warn({ + message: `Escalation notification with ID ${escalationChannelID} not found`, + service: SERVICE_NAME, + method: "sendNotifyEscalation", + }); return false; } + // Build notification message for escalation + const settings = this.settingsService.getSettings(); + const clientHost = settings.clientHost || "Host not defined"; + const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, monitorStatusResponse, decision, clientHost); + + this.logger.debug({ + message: `Sending escalation notification to channel ID ${escalationChannelID}`, + service: SERVICE_NAME, + method: "sendNotifyEscalation", + }); + + const result = await this.send(notification, monitor, monitorStatusResponse, decision, notificationMessage); + + if (result){ + try{ + const activeIncident = await this.monitorsRepository.findById(monitor.id, monitor.teamId); + this.logger.debug({ + message: `Marking incident as escalated for monitor ${monitor.id}`, + service: SERVICE_NAME, + method: "sendNotifyEscalation", + }); + } catch (error: unknown) { + this.logger.warn({ + message: `Failed to mark incident as escalated for monitor ${monitor.id}: ${(error as Error).message}`, + service: SERVICE_NAME, + method: "sendNotifyEscalation", + }); + } + } // Send notifications based on decision - return await this.sendNotifications(monitor, monitorStatusResponse, decision); + return result; + }; + + handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { + if(decision.shouldEscalateIncident && decision.escalationChannelID){ + await this.sendNotifyEscalation(monitor, monitorStatusResponse, decision, decision.escalationChannelID); + } + if (decision.shouldSendNotification) { + return await this.sendNotifications(monitor, monitorStatusResponse, decision); + } + + return false; }; sendTestNotification = async (notification: Partial) => { diff --git a/server/src/types/incident.ts b/server/src/types/incident.ts index 6b076ff835..7d0be56947 100644 --- a/server/src/types/incident.ts +++ b/server/src/types/incident.ts @@ -16,6 +16,7 @@ export interface Incident { resolvedBy?: string | null; resolvedByEmail?: string | null; comment?: string | null; + escalationTime?: string | null; createdAt: string; updatedAt: string; } diff --git a/server/src/types/monitor.ts b/server/src/types/monitor.ts index f29ce75d78..99f30fa762 100644 --- a/server/src/types/monitor.ts +++ b/server/src/types/monitor.ts @@ -52,6 +52,10 @@ export interface Monitor { group: string | null; geoCheckEnabled?: boolean; geoCheckLocations?: GeoContinent[]; + escalation?:{ + escalationDelay: number; + channelID: string; + } geoCheckInterval?: number; recentChecks: CheckSnapshot[]; createdAt: string; diff --git a/server/src/validation/monitorValidation.ts b/server/src/validation/monitorValidation.ts index df000ecef2..eab14e7b74 100644 --- a/server/src/validation/monitorValidation.ts +++ b/server/src/validation/monitorValidation.ts @@ -78,6 +78,9 @@ export const createMonitorBodyValidation = z.object({ geoCheckEnabled: z.boolean().optional(), geoCheckLocations: z.array(z.enum(GeoContinents)).optional(), geoCheckInterval: z.number().min(300000).optional(), + escalation: z.object({ + escalationDelay: z.number().min(0), + channelID: z.string().min(1),}), }); export const editMonitorBodyValidation = z.object({ @@ -107,6 +110,9 @@ export const editMonitorBodyValidation = z.object({ geoCheckEnabled: z.boolean().optional(), geoCheckLocations: z.array(z.enum(GeoContinents)).optional(), geoCheckInterval: z.number().min(300000).optional(), + escalation: z.object({ + escalationDelay: z.number().min(0), + channelID: z.string().min(1),}).optional(), }); export const pauseMonitorParamValidation = z.object({