feat: implement escalated notifications system

client/src/Hooks/useMonitorForm.ts

@@ -12,6 +12,7 @@ const getBaseDefaults = (data?: Monitor | null) => ({
 	description: data?.description || "",
 	interval: data?.interval || 60000,
 	notifications: data?.notifications || [],
+	escalationRules: data?.escalationRules || [],
 	statusWindowSize: data?.statusWindowSize || 5,
 	statusWindowThreshold: data?.statusWindowThreshold || 60,
 	geoCheckEnabled: data?.geoCheckEnabled ?? false,

client/src/Pages/CreateMonitor/index.tsx

@@ -36,6 +36,7 @@ import {
 	type Monitor,
 	type MonitorType,
 	type GamesMap,
+	type EscalationRule,
 	supportsGeoCheck,
 } from "@/Types/Monitor";
 import type { Notification } from "@/Types/Notification";
@@ -765,6 +766,93 @@ const CreateMonitorPage = () => {
 				}
 			/>
 
+			<ConfigBox
+				title="Escalation Rules"
+				subtitle="Configure delayed notifications when monitor stays down"
+				rightContent={
+					<Controller
+						name="escalationRules"
+						control={control}
+						render={({ field }) => {
+							const escalationRules = field.value ?? [];
+							return (
+								<Stack spacing={theme.spacing(LAYOUT.MD)}>
+									{escalationRules.map((rule: EscalationRule, index: number) => (
+										<Stack
+											key={index}
+											direction="row"
+											spacing={theme.spacing(LAYOUT.SM)}
+											alignItems="center"
+										>
+											<TextField
+												label="Delay (minutes)"
+												type="number"
+												value={rule.delayMinutes}
+												onChange={(e) => {
+													const newRules = [...escalationRules];
+													newRules[index] = {
+														...rule,
+														delayMinutes: parseInt(e.target.value) || 1,
+													};
+													field.onChange(newRules);
+												}}
+												size="small"
+												sx={{ width: 120 }}
+											/>
+											<Autocomplete
+												options={(notifications ?? []).map((n) => ({
+													...n,
+													name: n.notificationName,
+												}))}
+												value={
+													notifications?.find((n) => n.id === rule.notificationId) ? {
+														...notifications.find((n) => n.id === rule.notificationId)!,
+														name: notifications.find((n) => n.id === rule.notificationId)!.notificationName,
+													} : null
+												}
+												getOptionLabel={(option) => option.name}
+												onChange={(_, newValue) => {
+													const newRules = [...escalationRules];
+													newRules[index] = {
+														...rule,
+														notificationId: newValue?.id || "",
+													};
+													field.onChange(newRules);
+												}}
+												isOptionEqualToValue={(option, value) => option.id === value.id}
+												sx={{ flexGrow: 1 }}
+												size="small"
+											/>
+											<IconButton
+												size="small"
+												onClick={() => {
+													field.onChange(escalationRules.filter((_, i) => i !== index));
+												}}
+												aria-label="Remove escalation rule"
+											>
+												<Trash2 size={16} />
+											</IconButton>
+										</Stack>
+									))}
+									<Button
+										variant="outlined"
+										size="small"
+										onClick={() => {
+											field.onChange([
+												...escalationRules,
+												{ delayMinutes: 5, notificationId: "" },
+											]);
+										}}
+									>
+										Add Escalation Rule
+									</Button>
+								</Stack>
+							);
+						}}
+					/>
+				}
+			/>
+
 			{(watchedType === "http" ||
 				watchedType === "grpc" ||
 				watchedType === "websocket") && (

client/src/Types/Monitor.ts

@@ -38,6 +38,11 @@ export type MonitorStatus = (typeof MonitorStatuses)[number];
 
 export type MonitorMatchMethod = "equal" | "include" | "regex" | "";
 
+export interface EscalationRule {
+	delayMinutes: number;
+	notificationId: string;
+}
+
 export interface Monitor {
 	id: string;
 	userId: string;
@@ -60,6 +65,7 @@ export interface Monitor {
 	interval: number;
 	uptimePercentage?: number;
 	notifications: string[];
+	escalationRules: EscalationRule[];
 	secret?: string;
 	cpuAlertThreshold: number;
 	cpuAlertCounter: number;

client/src/Validation/monitor.ts

@@ -1,6 +1,12 @@
 import { z } from "zod";
 import { GeoContinents } from "@/Types/GeoCheck";
 
+// Escalation rule schema
+const escalationRuleSchema = z.object({
+	delayMinutes: z.number().min(1, "Delay must be at least 1 minute"),
+	notificationId: z.string().min(1, "Notification is required"),
+});
+
 // URL schema with custom error message
 const urlSchema = z.url({ message: "Please enter a valid URL" });
 
@@ -13,6 +19,7 @@ const baseSchema = z.object({
 	description: z.string().optional(),
 	interval: z.number().min(15000, "Interval must be at least 15 seconds"),
 	notifications: z.array(z.string()),
+	escalationRules: z.array(escalationRuleSchema),
 	statusWindowSize: z
 		.number({ message: "Status window size is required" })
 		.min(1, "Status window size must be at least 1")

server/src/config/services.ts

@@ -9,6 +9,7 @@ import {
 	SuperSimpleQueue,
 	SuperSimpleQueueHelper,
 	NotificationsService,
+	EscalationService,
 	StatusService,
 	NotificationMessageBuilder,
 	MonitorService,
@@ -34,6 +35,7 @@ import {
 	IBufferService,
 	ISuperSimpleQueue,
 	INotificationsService,
+	IEscalationService,
 	IStatusService,
 	IMonitorService,
 	IUserService,
@@ -129,6 +131,7 @@ export type InitializedServices = {
 	incidentService: IIncidentService;
 	logger: ILogger;
 	notificationsService: INotificationsService;
+	escalationService: IEscalationService;
 	statusPageService: IStatusPageService;
 	notificationMessageBuilder: INotificationMessageBuilder;
 
@@ -205,8 +208,6 @@ export const initializeServices = async ({
 
 	const notificationMessageBuilder = new NotificationMessageBuilder();
 
-	const incidentService = new IncidentService(logger, incidentsRepository, monitorsRepository, usersRepository, notificationMessageBuilder);
-
 	const checkService = new CheckService(monitorsRepository, logger, checksRepository);
 
 	const globalPingService = new GlobalPingService(logger);
@@ -246,6 +247,10 @@ export const initializeServices = async ({
 		notificationMessageBuilder
 	);
 
+	const escalationService = new EscalationService(logger, notificationsService);
+
+	const incidentService = new IncidentService(logger, incidentsRepository, monitorsRepository, usersRepository, notificationMessageBuilder, escalationService);
+
 	const superSimpleQueueHelper = new SuperSimpleQueueHelper(
 		logger,
 		networkService,
@@ -326,6 +331,7 @@ export const initializeServices = async ({
 		incidentService,
 		logger,
 		notificationsService,
+		escalationService,
 		statusPageService,
 		notificationMessageBuilder,
 

server/src/db/models/Monitor.ts

@@ -18,11 +18,12 @@ type CheckSnapshotDocument = Omit<CheckSnapshot, "createdAt"> & { createdAt: Dat
 
 type MonitorDocumentBase = Omit<
 	Monitor,
-	"id" | "userId" | "teamId" | "notifications" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt"
+	"id" | "userId" | "teamId" | "notifications" | "escalationRules" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt"
 > & {
 	statusWindow: boolean[];
 	recentChecks: CheckSnapshotDocument[];
 	notifications: Types.ObjectId[];
+	escalationRules: { delayMinutes: number; notificationId: Types.ObjectId }[];
 	selectedDisks: string[];
 	matchMethod?: MonitorMatchMethod;
 };
@@ -198,6 +199,14 @@ const checkSnapshotSchema = new Schema<CheckSnapshotDocument>(
 	{ _id: false }
 );
 
+const escalationRuleSchema = new Schema(
+	{
+		delayMinutes: { type: Number, required: true },
+		notificationId: { type: Schema.Types.ObjectId, ref: "Notification", required: true },
+	},
+	{ _id: false }
+);
+
 const MonitorSchema = new Schema<MonitorDocument>(
 	{
 		userId: {
@@ -284,6 +293,7 @@ const MonitorSchema = new Schema<MonitorDocument>(
 				ref: "Notification",
 			},
 		],
+		escalationRules: [escalationRuleSchema],
 		secret: {
 			type: String,
 		},

server/src/repositories/monitors/MongoMonitorsRepository.ts

@@ -374,6 +374,10 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			interval: doc.interval,
 			uptimePercentage: doc.uptimePercentage ?? undefined,
 			notifications: notificationIds,
+			escalationRules: (doc.escalationRules ?? []).map((rule) => ({
+				delayMinutes: rule.delayMinutes,
+				notificationId: toStringId(rule.notificationId),
+			})),
 			secret: doc.secret ?? undefined,
 			cpuAlertThreshold: doc.cpuAlertThreshold,
 			cpuAlertCounter: doc.cpuAlertCounter,
@@ -433,6 +437,10 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			interval: doc.interval,
 			uptimePercentage: doc.uptimePercentage ?? undefined,
 			notifications: notificationIds,
+			escalationRules: (doc.escalationRules ?? []).map((rule) => ({
+				delayMinutes: rule.delayMinutes,
+				notificationId: toStringId(rule.notificationId),
+			})),
 			secret: doc.secret ?? undefined,
 			cpuAlertThreshold: doc.cpuAlertThreshold,
 			cpuAlertCounter: doc.cpuAlertCounter,

server/src/service/business/incidentService.ts

@@ -7,6 +7,7 @@ import type { IIncidentsRepository, IMonitorsRepository, IUsersRepository } from
 import type { Incident, IncidentSummary, User } from "@/types/index.js";
 import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
 import type { INotificationMessageBuilder } from "@/service/infrastructure/notificationMessageBuilder.js";
+import type { IEscalationService } from "@/service/infrastructure/escalationService.js";
 import type { ILogger } from "@/utils/logger.js";
 
 export interface IIncidentService {
@@ -39,19 +40,22 @@ export class IncidentService implements IIncidentService {
 	private monitorsRepository: IMonitorsRepository;
 	private usersRepository: IUsersRepository;
 	private notificationMessageBuilder: INotificationMessageBuilder;
+	private escalationService: IEscalationService;
 
 	constructor(
 		logger: ILogger,
 		incidentsRepository: IIncidentsRepository,
 		monitorsRepository: IMonitorsRepository,
 		usersRepository: IUsersRepository,
-		notificationMessageBuilder: INotificationMessageBuilder
+		notificationMessageBuilder: INotificationMessageBuilder,
+		escalationService: IEscalationService
 	) {
 		this.logger = logger;
 		this.incidentsRepository = incidentsRepository;
 		this.monitorsRepository = monitorsRepository;
 		this.usersRepository = usersRepository;
 		this.notificationMessageBuilder = notificationMessageBuilder;
+		this.escalationService = escalationService;
 	}
 
 	get serviceName() {
@@ -91,14 +95,23 @@ export class IncidentService implements IIncidentService {
 					statusCode,
 					message,
 				};
-				return await this.incidentsRepository.create(incident);
+				const createdIncident = await this.incidentsRepository.create(incident);
+
+				// Schedule escalation notifications
+				await this.escalationService.scheduleEscalation(monitor, createdIncident.id);
+
+				return createdIncident;
 			}
 		}
 
 		if (decision.shouldResolveIncident) {
 			if (!activeIncident) {
 				return null;
 			}
+
+			// Cancel any scheduled escalations
+			await this.escalationService.cancelEscalation(monitor.id);
+
 			activeIncident.status = false;
 			activeIncident.endTime = Date.now().toString();
 			activeIncident.resolutionType = "automatic";
@@ -153,6 +166,9 @@ export class IncidentService implements IIncidentService {
 			incident.comment = comment || null;
 			incident.endTime = Date.now().toString();
 
+			// Cancel any scheduled escalations
+			await this.escalationService.cancelEscalation(incident.monitorId);
+
 			const resolvedIncident = await this.incidentsRepository.updateById(incident.id, teamId, incident);
 
 			this.logger.debug({

server/src/service/index.ts

@@ -15,6 +15,7 @@ export * from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.
 export * from "@/service/infrastructure/notificationMessageBuilder.js";
 export * from "@/service/infrastructure/bufferService.js";
 export * from "@/service/infrastructure/emailService.js";
+export * from "@/service/infrastructure/escalationService.js";
 export * from "@/service/infrastructure/globalPingService.js";
 export * from "@/service/infrastructure/networkService.js";
 export * from "@/service/infrastructure/notificationsService.js";

server/src/service/infrastructure/emailService.ts

@@ -112,6 +112,16 @@ export class EmailService implements IEmailService {
 			config = transportConfig;
 		} else {
 			config = await this.settingsService.getDBSettings();
+			if (!config.systemEmailHost) {
+				config = {
+					...config,
+					systemEmailHost: process.env.SYSTEM_EMAIL_HOST,
+					systemEmailPort: process.env.SYSTEM_EMAIL_PORT ? Number(process.env.SYSTEM_EMAIL_PORT) : undefined,
+					systemEmailAddress: process.env.SYSTEM_EMAIL_ADDRESS,
+					systemEmailPassword: process.env.SYSTEM_EMAIL_PASSWORD,
+					systemEmailSecure: process.env.SYSTEM_EMAIL_SECURE === "true",
+				};
+			}
 		}
 		const {
 			systemEmailHost,