Additional metric alert clickhouse performance improvements

deployment/grafana-dashboards/Metric-Alerts.json

@@ -272,9 +272,135 @@
       "title": "evaluateMetricAlertRules task duration (p99)",
       "type": "timeseries"
     },
+    {
+      "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" },
+      "description": "Enabled query groups (one ClickHouse query each) split by whether a saved filter is attached. Filtered groups read the heavier legacy tables (cost scales with the target's traffic cardinality), so watch this for growth in the more expensive with-filter population.",
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisPlacement": "auto",
+            "axisSoftMin": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "stacking": { "group": "A", "mode": "none" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "mappings": [],
+          "min": 0,
+          "noValue": "0",
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "with filter" },
+            "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }]
+          }
+        ]
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": ["last", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "pluginVersion": "9.3.6",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" },
+          "editorMode": "code",
+          "expr": "hive_metric_alert_rule_groups{filtered=\"true\"}",
+          "legendFormat": "with filter",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" },
+          "editorMode": "code",
+          "expr": "hive_metric_alert_rule_groups{filtered=\"false\"}",
+          "legendFormat": "no filter",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Alert query groups: filtered vs unfiltered",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" },
+      "description": "Enabled alert rules split by whether a saved filter is attached. Companion to the query-groups panel (a group can hold several rules that share one query).",
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisPlacement": "auto",
+            "axisSoftMin": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "stacking": { "group": "A", "mode": "none" },
+            "thresholdsStyle": { "mode": "off" }
+          },
+          "mappings": [],
+          "min": 0,
+          "noValue": "0",
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "with filter" },
+            "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }]
+          }
+        ]
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": ["last", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "pluginVersion": "9.3.6",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" },
+          "editorMode": "code",
+          "expr": "hive_metric_alert_enabled_rules{filtered=\"true\"}",
+          "legendFormat": "with filter",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" },
+          "editorMode": "code",
+          "expr": "hive_metric_alert_enabled_rules{filtered=\"false\"}",
+          "legendFormat": "no filter",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Enabled rules: filtered vs unfiltered",
+      "type": "timeseries"
+    },
     {
       "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 },
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 },
       "id": 100,
       "panels": [],
       "title": "Traces",
@@ -296,7 +422,7 @@
         },
         "overrides": []
       },
-      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 17 },
+      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 25 },
       "id": 101,
       "options": {
         "cellHeight": "sm",
@@ -334,7 +460,7 @@
         },
         "overrides": []
       },
-      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 17 },
+      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 25 },
       "id": 102,
       "options": {
         "cellHeight": "sm",
@@ -389,7 +515,7 @@
           }
         ]
       },
-      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 26 },
+      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 34 },
       "id": 104,
       "options": {
         "legend": {
@@ -443,7 +569,7 @@
         },
         "overrides": []
       },
-      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 26 },
+      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 34 },
       "id": 105,
       "options": {
         "legend": {
@@ -497,7 +623,7 @@
         },
         "overrides": []
       },
-      "gridPos": { "h": 9, "w": 24, "x": 0, "y": 35 },
+      "gridPos": { "h": 9, "w": 24, "x": 0, "y": 43 },
       "id": 106,
       "options": {
         "legend": {
@@ -541,7 +667,7 @@
         },
         "overrides": []
       },
-      "gridPos": { "h": 9, "w": 24, "x": 0, "y": 44 },
+      "gridPos": { "h": 9, "w": 24, "x": 0, "y": 52 },
       "id": 103,
       "options": {
         "cellHeight": "sm",

deployment/index.ts

@@ -201,6 +201,7 @@ deployWorkflows({
   schema,
   redis,
   clickhouse,
+  dbMigrations,
 });
 
 const zendesk = configureZendesk({ environment });

deployment/services/workflows.ts

@@ -3,6 +3,7 @@ import { serviceLocalEndpoint } from '../utils/local-endpoint';
 import { ServiceSecret } from '../utils/secrets';
 import { ServiceDeployment } from '../utils/service-deployment';
 import { Clickhouse } from './clickhouse';
+import { DbMigrations } from './db-migrations';
 import { Docker } from './docker';
 import { Environment } from './environment';
 import { Observability } from './observability';
@@ -29,6 +30,7 @@ export function deployWorkflows({
   schema,
   redis,
   clickhouse,
+  dbMigrations,
 }: {
   postgres: Postgres;
   observability: Observability;
@@ -41,6 +43,7 @@ export function deployWorkflows({
   schema: Schema;
   redis: Redis;
   clickhouse: Clickhouse;
+  dbMigrations: DbMigrations;
 }) {
   const featureFlagsConfig = new pulumi.Config('featureFlags');
   return (
@@ -75,7 +78,10 @@ export function deployWorkflows({
         image,
         replicas: environment.podsConfig.general.replicas,
       },
-      [redis.deployment, redis.service],
+      // Depend on dbMigrations so the ClickHouse migration (which creates
+      // operations_by_target_daily) runs before this service starts routing
+      // long-window queries to it. Mirrors deployGraphQL.
+      [dbMigrations, redis.deployment, redis.service],
     )
       // PG
       .withSecret('POSTGRES_HOST', postgres.pgBouncerSecret, 'host')

packages/migrations/src/clickhouse-actions/019-metric-alert-target-daily-rollup.ts

@@ -0,0 +1,49 @@
+import type { Action } from '../clickhouse';
+
+// Daily by-target rollup for the metric-alert evaluator. Mirrors the
+// minutely/hourly rollups in 018 but bucketed per day, so long-window rules
+// (>= 7 days) read ~60 daily buckets for a 30-day query instead of ~1,440
+// hourly ones. The 90-day TTL covers the 60-day (2x window) scan of a 30-day
+// rule.
+const tableColumns = `
+  target LowCardinality(String) CODEC(ZSTD(1)),
+  timestamp DateTime('UTC') CODEC(DoubleDelta, LZ4),
+  total UInt32 CODEC(T64, ZSTD(1)),
+  total_ok UInt32 CODEC(T64, ZSTD(1)),
+  duration_avg AggregateFunction(avg, UInt64) CODEC(ZSTD(1)),
+  duration_quantiles AggregateFunction(quantilesTDigest(0.75, 0.9, 0.95, 0.99), UInt64) CODEC(ZSTD(1))
+`;
+
+export const action: Action = async exec => {
+  await exec(`
+    CREATE TABLE IF NOT EXISTS default.operations_by_target_daily
+    (
+      ${tableColumns}
+    )
+    ENGINE = SummingMergeTree
+    -- Monthly, not toYYYYMMDD like the hourly rollup: daily buckets would make
+    -- one partition per day. Keep monthly so a 90-day TTL has few parts.
+    PARTITION BY toYYYYMM(timestamp)
+    PRIMARY KEY (target, timestamp)
+    ORDER BY (target, timestamp)
+    TTL timestamp + INTERVAL 90 DAY
+    SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
+  `);
+
+  await exec(`
+    CREATE MATERIALIZED VIEW IF NOT EXISTS default.operations_by_target_daily_mv TO default.operations_by_target_daily
+    AS (
+      SELECT
+        target,
+        toStartOfDay(timestamp) AS timestamp,
+        CAST(count() AS UInt32) AS total,
+        CAST(sum(ok) AS UInt32) AS total_ok,
+        avgState(duration) AS duration_avg,
+        quantilesTDigestState(0.75, 0.9, 0.95, 0.99)(duration) AS duration_quantiles
+      FROM default.operations
+      GROUP BY
+        target,
+        timestamp
+    )
+  `);
+};

packages/migrations/src/clickhouse.ts

@@ -182,6 +182,7 @@ export async function migrateClickHouse(
     import('./clickhouse-actions/016-subgraph-otel-traces-cleanup'),
     import('./clickhouse-actions/017-affected-app-deployments-performance'),
     import('./clickhouse-actions/018-metric-alert-target-rollups'),
+    import('./clickhouse-actions/019-metric-alert-target-daily-rollup'),
   ]);
 
   async function actionRunner(action: Action, index: number) {

packages/services/api/src/modules/alerts/providers/metric-alert-rules-manager.ts

@@ -3,9 +3,11 @@ import type { MetricAlertRule } from '../../../shared/entities';
 import { AccessError } from '../../../shared/errors';
 import { Session } from '../../auth/lib/authz';
 import {
+  METRIC_ALERT_RULE_DAILY_ROLLUP_THRESHOLD_MINUTES,
   METRIC_ALERT_RULE_TIME_WINDOW_MAX_MINUTES,
   METRIC_ALERT_RULE_TIME_WINDOW_MIN_MINUTES,
   METRIC_ALERT_RULES_PER_TARGET_LIMIT,
+  MINUTES_PER_DAY,
 } from '../../commerce/constants';
 import { OrganizationManager } from '../../organization/providers/organization-manager';
 import { Logger } from '../../shared/providers/logger';
@@ -338,6 +340,17 @@ export class MetricAlertRulesManager {
         `Time window must be a whole number of minutes between ${METRIC_ALERT_RULE_TIME_WINDOW_MIN_MINUTES} and ${METRIC_ALERT_RULE_TIME_WINDOW_MAX_MINUTES} (30 days).`,
       );
     }
+    // Windows at/above the daily-rollup threshold read whole-day ClickHouse
+    // buckets, so they must be a whole number of days or the window is silently
+    // rounded. The UI presets already satisfy this; this guards direct API callers.
+    if (
+      timeWindowMinutes >= METRIC_ALERT_RULE_DAILY_ROLLUP_THRESHOLD_MINUTES &&
+      timeWindowMinutes % MINUTES_PER_DAY !== 0
+    ) {
+      throw new MetricAlertRuleValidationError(
+        'Time windows of 7 days or more must be a whole number of days.',
+      );
+    }
   }
 
   private async assertChannelsBelongToProject(

packages/services/api/src/modules/commerce/constants.ts

@@ -51,5 +51,12 @@ export const METRIC_ALERT_RULES_PER_TARGET_LIMIT = 10;
  * reasons; these constants are the absolute bounds the API enforces against
  * any caller (form, seed scripts, customer integrations).
  */
+export const MINUTES_PER_DAY = 24 * 60; // 1440
 export const METRIC_ALERT_RULE_TIME_WINDOW_MIN_MINUTES = 1;
-export const METRIC_ALERT_RULE_TIME_WINDOW_MAX_MINUTES = 30 * 24 * 60; // 43200
+export const METRIC_ALERT_RULE_TIME_WINDOW_MAX_MINUTES = 30 * MINUTES_PER_DAY; // 43200
+
+// Windows at or above this read the daily ClickHouse rollup, whose buckets are
+// whole days. A window this size must therefore be a whole number of days
+// (a multiple of MINUTES_PER_DAY) or the daily aggregate would silently round it.
+// Mirrors DAILY_THRESHOLD_MINUTES in the workflows evaluator; keep the two in sync.
+export const METRIC_ALERT_RULE_DAILY_ROLLUP_THRESHOLD_MINUTES = 7 * MINUTES_PER_DAY; // 10080

packages/services/workflows/package.json

@@ -31,6 +31,7 @@
     "graphql-yoga": "5.13.3",
     "mjml": "4.14.0",
     "nodemailer": "9.0.1",
+    "p-limit": "6.2.0",
     "sendmail": "1.6.1",
     "zod": "3.25.76"
   }