feat(analytics): add route analytics heatmaps

apps/backend/scripts/clickhouse-migrations.ts

@@ -32,11 +32,21 @@ export async function runClickhouseMigrations() {
     client.command({ query: NOTIFICATION_PREFERENCES_TABLE_BASE_SQL }),
     client.command({ query: REFRESH_TOKENS_TABLE_BASE_SQL }),
     client.command({ query: CONNECTED_ACCOUNTS_TABLE_BASE_SQL }),
+    client.command({ query: CLICKMAP_EVENTS_TABLE_SQL }),
   ]);
 
   // Alter events table (must come before views that reference new columns)
   await client.command({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL });
 
+  // Clickmap materialized view depends on the events table existing; create after the ALTER above
+  // so the view sees the replay columns. IF NOT EXISTS makes this idempotent across reboots.
+  await client.command({ query: CLICKMAP_EVENTS_MV_SQL });
+
+  // Backfill historical $click rows that pre-date the MV. Predicate picks rows
+  // older than the earliest MV-captured row, so re-runs are no-ops once the
+  // first backfill completes.
+  await client.command({ query: CLICKMAP_EVENTS_BACKFILL_SQL });
+
   // Create all views in parallel
   await Promise.all([
     client.command({ query: EVENTS_VIEW_SQL }),
@@ -52,6 +62,7 @@ export async function runClickhouseMigrations() {
     client.command({ query: NOTIFICATION_PREFERENCES_VIEW_SQL }),
     client.command({ query: REFRESH_TOKENS_VIEW_SQL }),
     client.command({ query: CONNECTED_ACCOUNTS_VIEW_SQL }),
+    client.command({ query: CLICKMAP_EVENTS_VIEW_SQL }),
   ]);
 
   // Data migrations (mutations)
@@ -66,6 +77,7 @@ export async function runClickhouseMigrations() {
     "events", "users", "contact_channels", "teams", "team_member_profiles",
     "team_permissions", "team_invitations", "email_outboxes",
     "project_permissions", "notification_preferences", "refresh_tokens", "connected_accounts",
+    "clickmap_events",
   ];
   await Promise.all(tables.map(table =>
     client.command({
@@ -648,3 +660,161 @@ WHERE sync_is_deleted = 0;
 const EXTERNAL_ANALYTICS_DB_SQL = `
 CREATE DATABASE IF NOT EXISTS analytics_internal;
 `;
+
+// Clickmap-only physical table (PostHog-style schema). Fed by clickmap_events_mv
+// from analytics_internal.events WHERE event_type='$click'. Backwards compatible
+// with click rows that pre-date elements_chain / scaled coords: the MV derives
+// pointer_* from raw data.x / data.y / data.page_y, and elements_chain falls
+// back to the empty string when the SDK didn't emit one.
+//
+// SCALE_FACTOR = 16 mirrors PostHog: pixel coords are divided at ingest so
+// downstream queries operate on small integers and partitions stay compact.
+//
+// Order key (project_id, branch_id, date, path, viewport_width) matches the
+// hot clickmap query: "all clicks on this path in this date range at these
+// viewport widths".
+const CLICKMAP_EVENTS_TABLE_SQL = `
+CREATE TABLE IF NOT EXISTS analytics_internal.clickmap_events (
+    project_id           String,
+    branch_id            String,
+    event_at             DateTime64(3, 'UTC'),
+    user_id              Nullable(String),
+    session_replay_id    Nullable(String),
+    url                  String,
+    path                 String,
+    viewport_width       UInt16,
+    viewport_height      UInt16,
+    pointer_x            UInt16,
+    pointer_y            UInt16,
+    client_y             UInt16,
+    pointer_relative_x   Float32,
+    pointer_target_fixed UInt8,
+    elements_chain       String,
+    selector             String,
+    elements_text        String,
+    tag_name             LowCardinality(String),
+    href                 Nullable(String)
+)
+ENGINE MergeTree
+PARTITION BY toYYYYMM(event_at)
+ORDER BY (project_id, branch_id, toDate(event_at), path, viewport_width);
+`;
+
+// Materialized view that auto-populates clickmap_events on every $click insert.
+// No POPULATE clause: existing rows are not backfilled (they remain queryable
+// via the existing /api/.../analytics/clickmap route which still reads from
+// analytics_internal.events). New click rows flow into both tables.
+//
+// All field accesses use the toFloat64OrZero(toString(...)) pattern that the
+// existing analytics queries use, so JSON-Variant nullability is handled the
+// same way.
+const CLICKMAP_EVENTS_MV_SQL = `
+CREATE MATERIALIZED VIEW IF NOT EXISTS analytics_internal.clickmap_events_mv
+TO analytics_internal.clickmap_events
+AS
+SELECT
+    project_id,
+    branch_id,
+    event_at,
+    user_id,
+    session_replay_id,
+    toString(data.url) AS url,
+    toString(data.path) AS path,
+    toUInt16(least(65535, greatest(0, toUInt32(toFloat64OrZero(toString(data.viewport_width)))))) AS viewport_width,
+    toUInt16(least(65535, greatest(0, toUInt32(toFloat64OrZero(toString(data.viewport_height)))))) AS viewport_height,
+    toUInt16(least(65535, greatest(0, toUInt32(
+        coalesce(toFloat64OrNull(toString(data.x_scaled)), toFloat64OrZero(toString(data.page_x)) / 16, toFloat64OrZero(toString(data.x)) / 16)
+    )))) AS pointer_x,
+    toUInt16(least(65535, greatest(0, toUInt32(
+        coalesce(toFloat64OrNull(toString(data.y_scaled)), toFloat64OrZero(toString(data.page_y)) / 16, toFloat64OrZero(toString(data.y)) / 16)
+    )))) AS pointer_y,
+    toUInt16(least(65535, greatest(0, toUInt32(
+        coalesce(toFloat64OrNull(toString(data.client_y_scaled)), toFloat64OrZero(toString(data.y)) / 16)
+    )))) AS client_y,
+    toFloat32(coalesce(
+        toFloat64OrNull(toString(data.pointer_relative_x)),
+        if(toFloat64OrZero(toString(data.viewport_width)) > 0,
+           toFloat64OrZero(toString(data.x)) / toFloat64OrZero(toString(data.viewport_width)),
+           0)
+    )) AS pointer_relative_x,
+    toUInt8(coalesce(toUInt8OrNull(toString(data.pointer_target_fixed)), 0)) AS pointer_target_fixed,
+    toString(data.elements_chain) AS elements_chain,
+    toString(data.selector) AS selector,
+    toString(data.text) AS elements_text,
+    toString(data.tag_name) AS tag_name,
+    nullIf(toString(data.href), '') AS href
+FROM analytics_internal.events
+WHERE event_type = '$click';
+`;
+
+// Idempotent backfill: insert pre-MV $click rows into clickmap_events. After
+// the first run, min(event_at) in clickmap_events corresponds to the MV-capture
+// start (or earlier, once historical rows land), so this predicate returns no
+// new rows and the migration is a cheap no-op.
+const CLICKMAP_EVENTS_BACKFILL_SQL = `
+INSERT INTO analytics_internal.clickmap_events
+SELECT
+    project_id,
+    branch_id,
+    event_at,
+    user_id,
+    session_replay_id,
+    toString(data.url) AS url,
+    toString(data.path) AS path,
+    toUInt16(least(65535, greatest(0, toUInt32(toFloat64OrZero(toString(data.viewport_width)))))) AS viewport_width,
+    toUInt16(least(65535, greatest(0, toUInt32(toFloat64OrZero(toString(data.viewport_height)))))) AS viewport_height,
+    toUInt16(least(65535, greatest(0, toUInt32(
+        coalesce(toFloat64OrNull(toString(data.x_scaled)), toFloat64OrZero(toString(data.page_x)) / 16, toFloat64OrZero(toString(data.x)) / 16)
+    )))) AS pointer_x,
+    toUInt16(least(65535, greatest(0, toUInt32(
+        coalesce(toFloat64OrNull(toString(data.y_scaled)), toFloat64OrZero(toString(data.page_y)) / 16, toFloat64OrZero(toString(data.y)) / 16)
+    )))) AS pointer_y,
+    toUInt16(least(65535, greatest(0, toUInt32(
+        coalesce(toFloat64OrNull(toString(data.client_y_scaled)), toFloat64OrZero(toString(data.y)) / 16)
+    )))) AS client_y,
+    toFloat32(coalesce(
+        toFloat64OrNull(toString(data.pointer_relative_x)),
+        if(toFloat64OrZero(toString(data.viewport_width)) > 0,
+           toFloat64OrZero(toString(data.x)) / toFloat64OrZero(toString(data.viewport_width)),
+           0)
+    )) AS pointer_relative_x,
+    toUInt8(coalesce(toUInt8OrNull(toString(data.pointer_target_fixed)), 0)) AS pointer_target_fixed,
+    toString(data.elements_chain) AS elements_chain,
+    toString(data.selector) AS selector,
+    toString(data.text) AS elements_text,
+    toString(data.tag_name) AS tag_name,
+    nullIf(toString(data.href), '') AS href
+FROM analytics_internal.events
+WHERE event_type = '$click'
+  AND event_at < coalesce(
+    (SELECT min(event_at) FROM analytics_internal.clickmap_events),
+    toDateTime64('2999-01-01 00:00:00', 3, 'UTC')
+  );
+`;
+
+const CLICKMAP_EVENTS_VIEW_SQL = `
+CREATE OR REPLACE VIEW default.clickmap_events
+SQL SECURITY DEFINER
+AS
+SELECT
+  project_id,
+  branch_id,
+  event_at,
+  user_id,
+  session_replay_id,
+  url,
+  path,
+  viewport_width,
+  viewport_height,
+  pointer_x,
+  pointer_y,
+  client_y,
+  pointer_relative_x,
+  pointer_target_fixed,
+  elements_chain,
+  selector,
+  elements_text,
+  tag_name,
+  href
+FROM analytics_internal.clickmap_events;
+`;

apps/backend/src/app/api/latest/analytics/clickmap/route.ts

@@ -0,0 +1,121 @@
+import {
+  type ClickmapClicksQueryResult,
+  parseBoundedDateTime,
+  runClickmapClicksQuery,
+  throwClickhouseClickmapError,
+} from "@/lib/analytics-clickmap-query";
+import { verifyAnalyticsClickmapToken } from "@/lib/analytics-clickmap-tokens";
+import { getClickhouseAdminClientForMetrics } from "@/lib/clickhouse";
+import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
+import { AnalyticsClickmapResponseBodySchema, type AnalyticsClickmapResponse } from "@stackframe/stack-shared/dist/interface/admin-metrics";
+import { adaptSchema, clientOrHigherAuthTypeSchema, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";
+import { StatusError } from "@stackframe/stack-shared/dist/utils/errors";
+
+const MAX_WINDOW_DAYS = 31;
+const ONE_DAY_MS = 24 * 60 * 60 * 1000;
+const ROUTE_LIMIT = 50;
+const ELEMENTS_CHAIN_LIMIT = 200;
+
+export const POST = createSmartRouteHandler({
+  metadata: {
+    summary: "Get page clickmap data",
+    description: "Returns aggregated click data for the current browser origin when authorized by a short-lived clickmap token.",
+    tags: ["Analytics"],
+    hidden: true,
+  },
+  request: yupObject({
+    auth: yupObject({
+      type: clientOrHigherAuthTypeSchema.defined(),
+      tenancy: adaptSchema.defined(),
+      user: adaptSchema.optional(),
+    }).defined(),
+    body: yupObject({
+      clickmap_token: yupString().defined(),
+      origin: yupString().defined(),
+      route_path: yupString().optional(),
+      route_regex: yupString().optional(),
+      url_pattern: yupString().optional(),
+      user_id: yupString().optional(),
+      replay_id: yupString().optional(),
+      device: yupString().oneOf(["tv", "widescreen", "desktop", "laptop", "tablet", "mobile"]).optional(),
+      viewport_width_min: yupNumber().integer().min(0).max(65535).optional(),
+      viewport_width_max: yupNumber().integer().min(0).max(65535).optional(),
+      sampling: yupNumber().min(0).max(1).optional(),
+      since: yupString().defined(),
+      until: yupString().defined(),
+    }).defined(),
+  }),
+  response: yupObject({
+    statusCode: yupNumber().oneOf([200]).defined(),
+    bodyType: yupString().oneOf(["json"]).defined(),
+    body: AnalyticsClickmapResponseBodySchema,
+  }),
+  handler: async ({ body }) => {
+    // The dashboard mint path is the feature gate for clickmap overlays. This
+    // public read endpoint is authorized by the short-lived origin-bound token
+    // below, so avoid app/user gates that can disagree with the launching
+    // dashboard or anonymous customer pages.
+
+    const clickmapToken = await verifyAnalyticsClickmapToken({
+      token: body.clickmap_token,
+      origin: body.origin,
+    });
+
+    const since = parseBoundedDateTime(body.since, "since");
+    const until = parseBoundedDateTime(body.until, "until");
+    if (until.getTime() <= since.getTime()) {
+      throw new StatusError(StatusError.BadRequest, "until must be after since");
+    }
+    if (until.getTime() - since.getTime() > MAX_WINDOW_DAYS * ONE_DAY_MS) {
+      throw new StatusError(StatusError.BadRequest, `Clickmap window cannot exceed ${MAX_WINDOW_DAYS} days`);
+    }
+
+    const client = getClickhouseAdminClientForMetrics();
+    let result: ClickmapClicksQueryResult;
+    try {
+      result = await runClickmapClicksQuery(client, {
+        projectId: clickmapToken.project_id,
+        branchId: clickmapToken.branch_id,
+        since,
+        until,
+        origin: clickmapToken.origin,
+        routePath: body.route_path,
+        routeRegex: body.route_regex,
+        urlPattern: body.url_pattern,
+        userId: body.user_id,
+        replayId: body.replay_id,
+        device: body.device,
+        viewportWidthMin: body.viewport_width_min,
+        viewportWidthMax: body.viewport_width_max,
+        sampling: body.sampling,
+        routeLimit: ROUTE_LIMIT,
+        elementsChainLimit: ELEMENTS_CHAIN_LIMIT,
+      });
+    } catch (error) {
+      throwClickhouseClickmapError(error, {
+        captureLabel: "analytics-clickmap-clickhouse-fallback",
+        routeRegex: body.route_regex,
+        context: { projectId: clickmapToken.project_id, branchId: clickmapToken.branch_id },
+      });
+    }
+
+    // The public overlay only consumes routes/selectors/elements; per-user and
+    // per-replay aggregates are intentionally not fetched here (no linkedLimit).
+    const responseBody: AnalyticsClickmapResponse = {
+      kind: "session_replay_clicks",
+      cells: [],
+      sampling: result.samplingPct / 100,
+      routes: result.routes,
+      users: [],
+      replays: [],
+      selectors: result.selectors,
+      elements: result.elements,
+    };
+
+    return {
+      statusCode: 200,
+      bodyType: "json",
+      body: responseBody,
+    };
+  },
+});

apps/backend/src/app/api/latest/internal/analytics/clickmap-token/route.ts

@@ -0,0 +1,34 @@
+import { createAnalyticsClickmapToken } from "@/lib/analytics-clickmap-tokens";
+import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
+import { AnalyticsClickmapTokenResponseBodySchema } from "@stackframe/stack-shared/dist/interface/admin-metrics";
+import { adaptSchema, adminAuthTypeSchema, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";
+
+export const POST = createSmartRouteHandler({
+  metadata: { hidden: true },
+  request: yupObject({
+    auth: yupObject({
+      type: adminAuthTypeSchema.defined(),
+      tenancy: adaptSchema.defined(),
+    }).defined(),
+    body: yupObject({
+      origin: yupString().defined(),
+    }).defined(),
+  }),
+  response: yupObject({
+    statusCode: yupNumber().oneOf([200]).defined(),
+    bodyType: yupString().oneOf(["json"]).defined(),
+    body: AnalyticsClickmapTokenResponseBodySchema,
+  }),
+  handler: async ({ auth, body }) => {
+    const token = await createAnalyticsClickmapToken({ tenancy: auth.tenancy, origin: body.origin });
+    return {
+      statusCode: 200,
+      bodyType: "json",
+      body: {
+        token: token.token,
+        origin: token.origin,
+        expires_at_millis: token.expiresAtMillis,
+      },
+    };
+  },
+});