Console 2039 make metric alert notification fan out at least once with

packages/migrations/src/actions/2026.06.16T00-00-00.metric-alert-channel-health.ts

@@ -0,0 +1,20 @@
+import { type MigrationExecutor } from '../pg-migrator';
+
+export default {
+  name: '2026.06.16T00-00-00.metric-alert-channel-health.ts',
+  run: ({ psql }) => psql`
+-- Per-(rule, channel) record of a dropped delivery: upserted when a channel
+-- exhausts its retries for an event, deleted on the next success. A sibling
+-- table (not a column on "metric_alert_rule_channels") so it survives that
+-- table's DELETE+INSERT churn when a rule's channels are edited.
+CREATE TABLE "metric_alert_channel_health" (
+  "metric_alert_rule_id" uuid NOT NULL REFERENCES "metric_alert_rules"("id") ON DELETE CASCADE,
+  "alert_channel_id"     uuid NOT NULL REFERENCES "alert_channels"("id")    ON DELETE CASCADE,
+  "degraded_at"          timestamptz NOT NULL DEFAULT now(),
+  "last_error"           text,
+  PRIMARY KEY ("metric_alert_rule_id", "alert_channel_id")
+);
+
+CREATE INDEX "idx_metric_alert_channel_health_channel" ON "metric_alert_channel_health" ("alert_channel_id"); -- FK-cascade index
+`,
+} satisfies MigrationExecutor;

packages/migrations/src/run-pg-migrations.ts

@@ -190,5 +190,6 @@ export const runPGMigrations = async (args: { slonik: PostgresDatabasePool; runT
       await import('./actions/2026.05.29T00-00-00.schema-log-target-id-nullability'),
       await import('./actions/2026.05.07T00-00-00.schema-version-promotion'),
       await import('./actions/2026.06.05T00-00-00.metric-alert-filter-shared-only'),
+      await import('./actions/2026.06.16T00-00-00.metric-alert-channel-health'),
     ],
   });

packages/migrations/test/2026.06.16T00-00-00.metric-alert-channel-health.test.ts

@@ -0,0 +1,116 @@
+import assert from 'node:assert';
+import { describe, test } from 'node:test';
+import { z } from 'zod';
+import { psql } from '@hive/postgres';
+import { initMigrationTestingEnvironment } from './utils/testkit';
+
+const idRow = z.object({ id: z.string() });
+const countRow = z.object({ c: z.number() });
+
+await describe('migration: metric-alert-channel-health', async () => {
+  await test('creates the table + index and cascades on rule/channel delete', async () => {
+    const { db, runTo, complete, seed, done } = await initMigrationTestingEnvironment();
+
+    try {
+      // Schema up to (and including) metric-alert-rules; our table has NOT been
+      // created yet.
+      await runTo('2026.04.15T00-00-01.metric-alert-rules.ts');
+
+      const user = await seed.user({ user: { name: 'u1', email: 'u1@test.com' } });
+      const org = await seed.organization({ organization: { name: 'org-1' }, user });
+      const project = await seed.project({
+        project: { name: 'p1', type: 'SINGLE' },
+        organization: org,
+      });
+      const target = await seed.target({ project, target: { name: 't1' } });
+
+      const channel = await db
+        .one(
+          psql`
+            INSERT INTO alert_channels (project_id, type, name, webhook_endpoint)
+            VALUES (${project.id}, 'WEBHOOK', 'wh', 'https://example.test/hook')
+            RETURNING id
+          `,
+        )
+        .then(idRow.parse);
+
+      const rule = await db
+        .one(
+          psql`
+            INSERT INTO metric_alert_rules (
+              organization_id, project_id, target_id, type, time_window_minutes,
+              threshold_type, threshold_value, direction, severity, name, enabled,
+              state, confirmation_minutes
+            ) VALUES (
+              ${org.id}, ${project.id}, ${target.id}, 'TRAFFIC', 60,
+              'FIXED_VALUE', 100, 'ABOVE', 'WARNING', 'rule', true,
+              'NORMAL', 0
+            )
+            RETURNING id
+          `,
+        )
+        .then(idRow.parse);
+
+      // Apply our migration: metric_alert_channel_health is created.
+      await complete();
+
+      // The index exists (FK-cascade index on alert_channel_id).
+      const idx = await db
+        .one(
+          psql`
+            SELECT count(*)::int as c FROM pg_indexes
+            WHERE indexname = 'idx_metric_alert_channel_health_channel'
+          `,
+        )
+        .then(countRow.parse);
+      assert.equal(idx.c, 1);
+
+      await db.query(psql`
+        INSERT INTO metric_alert_channel_health (metric_alert_rule_id, alert_channel_id, last_error)
+        VALUES (${rule.id}, ${channel.id}, 'HTTP 503')
+      `);
+
+      // PK is (rule, channel): a second bare insert for the same pair conflicts.
+      await assert.rejects(
+        db.query(psql`
+          INSERT INTO metric_alert_channel_health (metric_alert_rule_id, alert_channel_id)
+          VALUES (${rule.id}, ${channel.id})
+        `),
+      );
+
+      // Deleting the channel cascades the health row away.
+      await db.query(psql`DELETE FROM alert_channels WHERE id = ${channel.id}`);
+      const afterChannelDelete = await db
+        .one(
+          psql`SELECT count(*)::int as c FROM metric_alert_channel_health WHERE metric_alert_rule_id = ${rule.id}`,
+        )
+        .then(countRow.parse);
+      assert.equal(afterChannelDelete.c, 0);
+
+      // Re-create the pair, then delete the rule — also cascades.
+      const channel2 = await db
+        .one(
+          psql`
+            INSERT INTO alert_channels (project_id, type, name, webhook_endpoint)
+            VALUES (${project.id}, 'WEBHOOK', 'wh2', 'https://example.test/hook2')
+            RETURNING id
+          `,
+        )
+        .then(idRow.parse);
+      await db.query(psql`
+        INSERT INTO metric_alert_channel_health (metric_alert_rule_id, alert_channel_id)
+        VALUES (${rule.id}, ${channel2.id})
+      `);
+      await db.query(psql`DELETE FROM metric_alert_rules WHERE id = ${rule.id}`);
+      const afterRuleDelete = await db
+        .one(psql`SELECT count(*)::int as c FROM metric_alert_channel_health`)
+        .then(countRow.parse);
+      assert.equal(afterRuleDelete.c, 0);
+    } finally {
+      // Close the test-db pool before done() drops the database, otherwise the
+      // lingering session makes DROP DATABASE fail with "session in use".
+      await db.end().catch(() => {});
+      await done();
+    }
+  });
+});

packages/migrations/test/root.ts

@@ -5,3 +5,4 @@ import './2024.01.26T00.00.01.schema-check-purging.test';
 import './2024.07.23T09.36.00.schema-cleanup-tracker.test';
 import './2025.01.09T00-00-00.legacy-member-scopes.test';
 import './2026.06.05T00-00-00.metric-alert-filter-shared-only.test';
+import './2026.06.16T00-00-00.metric-alert-channel-health.test';

packages/services/api/src/modules/alerts/module.graphql.mappers.ts

@@ -14,3 +14,8 @@ export type AlertMapper = Alert;
 export type MetricAlertRuleMapper = MetricAlertRule;
 export type MetricAlertRuleIncidentMapper = MetricAlertIncident;
 export type MetricAlertRuleStateChangeMapper = MetricAlertStateLogEntry;
+export type MetricAlertChannelDeliveryMapper = {
+  channel: AlertChannel;
+  degradedAt: string;
+  lastError: string | null;
+};

packages/services/api/src/modules/alerts/module.graphql.ts

@@ -210,6 +210,11 @@ export default gql`
     Destinations that receive notifications when this rule fires or resolves.
     """
     channels: [AlertChannel!]!
+    """
+    Channels whose most recent notification for this rule failed every delivery
+    attempt within the last 24h (the alert was dropped).
+    """
+    degradedChannels: [MetricAlertChannelDelivery!]!
     timeWindowMinutes: Int!
     metric: MetricAlertRuleMetric
     thresholdType: MetricAlertRuleThresholdType!
@@ -261,6 +266,18 @@ export default gql`
     stateAt(timestamp: DateTime!): MetricAlertRuleState!
   }
 
+  type MetricAlertChannelDelivery {
+    channel: AlertChannel!
+    """
+    When this channel's delivery for the rule was last marked degraded.
+    """
+    degradedAt: DateTime!
+    """
+    Last delivery error recorded, if captured.
+    """
+    lastError: String
+  }
+
   type MetricAlertRuleIncident {
     id: ID!
     startedAt: DateTime!

packages/services/api/src/modules/alerts/providers/metric-alert-rules-storage.ts

@@ -149,6 +149,18 @@ const AlertChannelRowSchema = zod.object({
   webhookEndpoint: zod.string().nullable(),
 });
 
+export type DegradedChannel = {
+  channel: AlertChannel;
+  degradedAt: string;
+  lastError: string | null;
+};
+
+const DegradedChannelRowSchema = AlertChannelRowSchema.extend({
+  ruleId: zod.string(),
+  degradedAt: zod.string(),
+  lastError: zod.string().nullable(),
+});
+
 const METRIC_ALERT_STATE_LOG_SELECT = psql`
   "id"
   , "metric_alert_rule_id" as "metricAlertRuleId"
@@ -225,6 +237,47 @@ export class MetricAlertRulesStorage {
     { cache: true },
   );
 
+  // Joins through metric_alert_rule_channels so a health row for a channel
+  // since detached from the rule doesn't surface (detach doesn't delete it).
+  private degradedChannelsByRuleLoader = new DataLoader<string, DegradedChannel[]>(
+    async ruleIds => {
+      const rows = await this.pool.any(psql`/* batchedDegradedChannelsByRule */
+        SELECT
+          h."metric_alert_rule_id" as "ruleId"
+          , to_json(h."degraded_at") as "degradedAt"
+          , h."last_error" as "lastError"
+          , ac."id" as "id"
+          , ac."project_id" as "projectId"
+          , ac."type" as "type"
+          , ac."name" as "name"
+          , to_json(ac."created_at") as "createdAt"
+          , ac."slack_channel" as "slackChannel"
+          , ac."webhook_endpoint" as "webhookEndpoint"
+        FROM "metric_alert_channel_health" h
+        INNER JOIN "metric_alert_rule_channels" rc
+          ON rc."metric_alert_rule_id" = h."metric_alert_rule_id"
+          AND rc."alert_channel_id" = h."alert_channel_id"
+        INNER JOIN "alert_channels" ac ON ac."id" = h."alert_channel_id"
+        WHERE h."metric_alert_rule_id" = ANY(${psql.array([...ruleIds], 'uuid')})
+          AND h."degraded_at" > now() - interval '24 hours'
+        ORDER BY h."degraded_at" DESC
+      `);
+      const byRule = new Map<string, DegradedChannel[]>();
+      for (const raw of rows) {
+        const { ruleId, degradedAt, lastError, ...channel } = DegradedChannelRowSchema.parse(raw);
+        const entry: DegradedChannel = { channel, degradedAt, lastError };
+        const list = byRule.get(ruleId);
+        if (list) {
+          list.push(entry);
+        } else {
+          byRule.set(ruleId, [entry]);
+        }
+      }
+      return ruleIds.map(id => byRule.get(id) ?? []);
+    },
+    { cache: true },
+  );
+
   // Per-request batcher: the manage-filters list resolves `usedByAlertRulesCount`
   // for each saved filter; this collapses N counts into one grouped query.
   private rulesCountBySavedFilterLoader = new DataLoader<string, number>(
@@ -533,6 +586,10 @@ export class MetricAlertRulesStorage {
     return results.filter((c): c is AlertChannel => c !== null);
   }
 
+  getDegradedChannels(args: { ruleId: string }): Promise<DegradedChannel[]> {
+    return this.degradedChannelsByRuleLoader.load(args.ruleId);
+  }
+
   /**
    * Returns the `project_id` for each existing channel in `channelIds`.
    * Used by mutation resolvers to validate that channels belong to the same

packages/services/api/src/modules/alerts/resolvers/MetricAlertChannelDelivery.ts

@@ -0,0 +1,14 @@
+import type { MetricAlertChannelDeliveryResolvers } from './../../../__generated__/types';
+
+/*
+ * Note: This object type is generated because "MetricAlertChannelDeliveryMapper" is declared. This is to ensure runtime safety.
+ *
+ * When a mapper is used, it is possible to hit runtime errors in some scenarios:
+ * - given a field name, the schema type's field type does not match mapper's field type
+ * - or a schema type's field does not exist in the mapper's fields
+ *
+ * If you want to skip this file generation, remove the mapper or update the pattern in the `resolverGeneration.object` config.
+ */
+export const MetricAlertChannelDelivery: MetricAlertChannelDeliveryResolvers = {
+  /* Implement MetricAlertChannelDelivery resolver logic here */
+};

packages/services/api/src/modules/alerts/resolvers/MetricAlertRule.ts

@@ -22,6 +22,9 @@ export const MetricAlertRule: MetricAlertRuleResolvers = {
     const channelIds = await storage.getRuleChannelIds({ ruleId: rule.id });
     return storage.getAlertChannelsByIds(channelIds);
   },
+  degradedChannels: (rule, _, { injector }) => {
+    return injector.get(MetricAlertRulesStorage).getDegradedChannels({ ruleId: rule.id });
+  },
   savedFilter: async (rule, _, { injector }) => {
     if (!rule.savedFilterId) {
       return null;

packages/services/storage/src/db/types.ts

@@ -160,6 +160,13 @@ export interface graphile_worker_deduplication {
   task_name: string;
 }
 
+export interface metric_alert_channel_health {
+  alert_channel_id: string;
+  degraded_at: Date;
+  last_error: string | null;
+  metric_alert_rule_id: string;
+}
+
 export interface metric_alert_incidents {
   current_value: number;
   id: string;
@@ -614,6 +621,7 @@ export interface DBTables {
   document_preflight_scripts: document_preflight_scripts;
   email_verifications: email_verifications;
   graphile_worker_deduplication: graphile_worker_deduplication;
+  metric_alert_channel_health: metric_alert_channel_health;
   metric_alert_incidents: metric_alert_incidents;
   metric_alert_notifications_sent: metric_alert_notifications_sent;
   metric_alert_rule_channels: metric_alert_rule_channels;

packages/services/workflows/src/lib/metric-alert-evaluator.ts

@@ -613,7 +613,10 @@ async function enqueueChannelNotifications(
           'stateLogId', ${stateLogId}::text,
           'channelId', marc."alert_channel_id"::text
         )
-      )
+      ),
+      -- graphile-worker defaults to 25; after 5 attempts the send task records
+      -- the channel as degraded rather than retrying a dead channel for days.
+      max_attempts => 5
     )
     FROM "metric_alert_rule_channels" marc
     WHERE marc."metric_alert_rule_id" = ${ruleId}

packages/services/workflows/src/tasks/send-metric-alert-channel-notification.ts

@@ -69,7 +69,7 @@ export const task = implementTask(SendMetricAlertChannelNotificationTask, async
       },
     },
     async span => {
-      let outcome: 'sent' | 'deduped' | 'skipped-deleted' | 'failed' = 'failed';
+      let outcome: 'sent' | 'deduped' | 'skipped-deleted' | 'degraded' | 'failed' = 'failed';
       // Hoisted so the finally block can compute breach-to-dispatch lag
       // regardless of whether the dispatch succeeded, failed, or threw. Stays
       // null when the SELECT didn't return a row (already-deduped or
@@ -216,6 +216,12 @@ export const task = implementTask(SendMetricAlertChannelNotificationTask, async
           ON CONFLICT ("state_log_id", "alert_channel_id") DO NOTHING
         `);
 
+        // A successful delivery clears any prior degraded marker for this pair.
+        await context.pg.query(psql`
+          DELETE FROM "metric_alert_channel_health"
+          WHERE "metric_alert_rule_id" = ${row.ruleId} AND "alert_channel_id" = ${channelId}
+        `);
+
         outcome = 'sent';
       } catch (err) {
         // outcome already initialized to 'failed'; record on the span for the
@@ -226,6 +232,31 @@ export const task = implementTask(SendMetricAlertChannelNotificationTask, async
           message: err instanceof Error ? err.message : String(err),
         });
         span.setAttribute('error.type', err instanceof Error ? err.name : 'unknown');
+
+        // Final attempt failed → the delivery is dropped. Record the (rule,
+        // channel) pair as degraded so the rule UI can warn. Best-effort: a
+        // failure here must never mask the original send error.
+        if (helpers.job.attempts >= helpers.job.max_attempts) {
+          outcome = 'degraded';
+          try {
+            const lastError = (err instanceof Error ? err.message : String(err)).slice(0, 500);
+            await context.pg.query(psql`
+              INSERT INTO "metric_alert_channel_health"
+                ("metric_alert_rule_id", "alert_channel_id", "degraded_at", "last_error")
+              SELECT sl."metric_alert_rule_id", ${channelId}, now(), ${lastError}
+              FROM "metric_alert_state_log" sl
+              WHERE sl."id" = ${stateLogId}
+              ON CONFLICT ("metric_alert_rule_id", "alert_channel_id")
+              DO UPDATE SET "degraded_at" = now(), "last_error" = EXCLUDED."last_error"
+            `);
+          } catch (healthErr) {
+            logger.error(
+              { error: healthErr, stateLogId, channelId },
+              'Failed to record metric alert channel health',
+            );
+          }
+        }
+
         throw err;
       } finally {
         span.setAttribute('notification.outcome', outcome);

packages/web/app/.ladle/vite.config.ts

@@ -4,4 +4,6 @@ import tailwindcss from '@tailwindcss/vite';
 
 export default defineConfig({
   plugins: [tsconfigPaths(), tailwindcss()],
+  optimizeDeps: { esbuildOptions: { target: 'esnext' } },
+  esbuild: { target: 'esnext' },
 });