refactor(web): derive route labels from routes-manifest instead of a hardcoded list

The hardcoded KNOWN_ROUTES set bounded cardinality but rots: new routes
silently degrade to `other` until someone updates the file, and two-segment
truncation collapses distinct routes (everything under /api/ee/* became one
label).

Next's build already emits .next/routes-manifest.json with every defined
route and a matching regex, ordered by the router's own resolution priority.
Load that at startup: exact-match static routes, then first dynamic regex
wins, mirroring how the server actually routes the request. Labels become the
route pattern itself (/browse/[...path], /settings/connections/[id]), so
cardinality is bounded by the number of defined routes plus /_next and
`other`, and the label set tracks the app automatically at build time.

Scanner traffic now lands on the catch-all routes that genuinely serve it
(/[...slug], /api/[...slug]) rather than a synthetic bucket. If the manifest
is missing or unreadable, everything is labelled `other` — granularity lost,
bound kept.

Verified against the production build's manifest (78 static + 18 dynamic
routes) and the running pod: the standalone server chdirs to the app dir, so
the cwd-relative manifest path resolves.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: brendan-kellam

packages/web/src/httpMetrics.integration.test.ts

@@ -1,6 +1,6 @@
 import { afterAll, describe, expect, it } from 'vitest';
 import { createServer, Server } from 'node:http';
-import { startHttpMetrics } from './httpMetrics';
+import { initRouteTable, startHttpMetrics } from './httpMetrics';
 import { registry } from './promClient';
 
 const app = createServer((_req, res) => {
@@ -32,7 +32,14 @@ const countLines = (output: string): string[] => {
 };
 
 describe('httpMetrics', () => {
-    it('records durations per normalized route and ignores the metrics port', async () => {
+    it('records durations per route pattern and ignores the metrics port', async () => {
+        // The table is injected rather than read from disk so the test doesn't
+        // depend on a prior `next build` having produced routes-manifest.json.
+        initRouteTable({
+            staticRoutes: [{ page: '/api/health' }],
+            dynamicRoutes: [{ page: '/browse/[...path]', regex: '^/browse/(.+?)(?:/)?$' }],
+        });
+
         // Both servers take an ephemeral port, then the metrics port is published
         // to env before subscribing, so the test never depends on a fixed port.
         const metricsPort = await listen(metricsServer);
@@ -54,8 +61,8 @@ describe('httpMetrics', () => {
         expect(counts.some(line => line.includes('route="/api/health"'))).toBe(true);
         expect(counts.some(line => line.includes('status="200"'))).toBe(true);
 
-        // Two distinct file paths must collapse to a single /browse series.
-        const browse = counts.filter(line => line.includes('route="/browse"'));
+        // Two distinct file paths must collapse to the single route-pattern series.
+        const browse = counts.filter(line => line.includes('route="/browse/[...path]"'));
         expect(browse).toHaveLength(1);
         expect(browse[0].trim().endsWith('2')).toBe(true);
 

packages/web/src/httpMetrics.test.ts

@@ -1,47 +1,92 @@
 import { describe, expect, it } from 'vitest';
-import { MAX_ROUTE_LABELS, normalizeRoute } from './httpMetrics';
+import { buildRouteTable, normalizeRoute } from './httpMetrics';
+
+// Mirrors the shape and ordering of the real .next/routes-manifest.json:
+// dynamic routes are listed in Next's resolution priority, with catch-alls
+// after specific routes and the root catch-all last.
+const table = buildRouteTable({
+    staticRoutes: [
+        { page: '/' },
+        { page: '/search' },
+        { page: '/repos' },
+        { page: '/api/health' },
+        { page: '/api/commits' },
+    ],
+    dynamicRoutes: [
+        { page: '/api/auth/[...nextauth]', regex: '^/api/auth/(.+?)(?:/)?$' },
+        { page: '/api/repos/[repoId]/image', regex: '^/api/repos/([^/]+?)/image(?:/)?$' },
+        { page: '/api/[...slug]', regex: '^/api/(.+?)(?:/)?$' },
+        { page: '/browse/[...path]', regex: '^/browse/(.+?)(?:/)?$' },
+        { page: '/settings/[...slug]', regex: '^/settings/(.+?)(?:/)?$' },
+        { page: '/[...slug]', regex: '^/(.+?)(?:/)?$' },
+    ],
+});
+
+const maxLabels = table.staticPages.size + table.dynamicRoutes.length + 2; // + '/_next', 'other'
 
 describe('normalizeRoute', () => {
     it('maps the root path', () => {
-        expect(normalizeRoute('/')).toBe('/');
-        expect(normalizeRoute('')).toBe('/');
+        expect(normalizeRoute('/', table)).toBe('/');
+        expect(normalizeRoute('', table)).toBe('/');
+    });
+
+    it('matches static routes exactly', () => {
+        expect(normalizeRoute('/api/health', table)).toBe('/api/health');
+        expect(normalizeRoute('/search', table)).toBe('/search');
     });
 
-    it('keeps two segments for known API routes', () => {
-        expect(normalizeRoute('/api/health')).toBe('/api/health');
-        expect(normalizeRoute('/api/commits')).toBe('/api/commits');
-        expect(normalizeRoute('/api/auth/callback/github')).toBe('/api/auth');
+    it('labels dynamic routes with their route pattern', () => {
+        expect(normalizeRoute('/api/auth/callback/github', table)).toBe('/api/auth/[...nextauth]');
+        expect(normalizeRoute('/api/repos/42/image', table)).toBe('/api/repos/[repoId]/image');
+        expect(normalizeRoute('/settings/connections/42', table)).toBe('/settings/[...slug]');
     });
 
-    it('keeps one segment for known page routes', () => {
-        expect(normalizeRoute('/search')).toBe('/search');
-        expect(normalizeRoute('/settings/connections/42')).toBe('/settings');
+    it('respects manifest ordering: specific routes win over catch-alls', () => {
+        // /api/auth/... must hit [...nextauth], not the /api/[...slug] catch-all.
+        expect(normalizeRoute('/api/auth/session', table)).toBe('/api/auth/[...nextauth]');
+        // Unknown API paths fall through to the catch-all that actually serves them.
+        expect(normalizeRoute('/api/not-a-real-route', table)).toBe('/api/[...slug]');
     });
 
-    it('collapses unbounded repository and file paths', () => {
-        const a = normalizeRoute('/browse/github.com/org/repo/-/blob/src/index.ts');
-        const b = normalizeRoute('/browse/github.com/other/repo/-/blob/lib/other.ts');
+    it('collapses unbounded repository and file paths to one label', () => {
+        const a = normalizeRoute('/browse/github.com/org/repo/-/blob/src/index.ts', table);
+        const b = normalizeRoute('/browse/github.com/other/repo/-/blob/lib/other.ts', table);
 
-        expect(a).toBe('/browse');
+        expect(a).toBe('/browse/[...path]');
         expect(b).toBe(a);
     });
 
     it('is unaffected by trailing or duplicate slashes', () => {
-        expect(normalizeRoute('/search/')).toBe('/search');
-        expect(normalizeRoute('//search//')).toBe('/search');
+        expect(normalizeRoute('/search/', table)).toBe('/search');
+        expect(normalizeRoute('//search//', table)).toBe('/search');
+        expect(normalizeRoute('/api/health/', table)).toBe('/api/health');
+    });
+
+    it('labels asset requests /_next without consulting the table', () => {
+        expect(normalizeRoute('/_next/static/chunks/main.js', table)).toBe('/_next');
+        expect(normalizeRoute('/_next/image', undefined)).toBe('/_next');
+    });
+
+    it('reports everything as other when no table is loaded', () => {
+        expect(normalizeRoute('/api/health', undefined)).toBe('other');
+        expect(normalizeRoute('/search', undefined)).toBe('other');
     });
 
     describe('cardinality bounding', () => {
-        it('reports unknown top-level paths as other', () => {
-            expect(normalizeRoute('/wp-admin')).toBe('other');
-            expect(normalizeRoute('/.env')).toBe('other');
-            expect(normalizeRoute('/phpmyadmin/index.php')).toBe('other');
+        it('routes scanner traffic to catch-alls, not new labels', () => {
+            expect(normalizeRoute('/wp-admin', table)).toBe('/[...slug]');
+            expect(normalizeRoute('/.env', table)).toBe('/[...slug]');
+            expect(normalizeRoute('/api/12345', table)).toBe('/api/[...slug]');
         });
 
-        it('reports unknown API paths as other, despite the [...slug] catch-all', () => {
-            expect(normalizeRoute('/api/not-a-real-route')).toBe('other');
-            expect(normalizeRoute('/api/12345')).toBe('other');
-            expect(normalizeRoute('/api/health-check')).toBe('other');
+        it('reports unmatched paths as other when there is no root catch-all', () => {
+            const noCatchAll = buildRouteTable({
+                staticRoutes: [{ page: '/search' }],
+                dynamicRoutes: [{ page: '/api/[...slug]', regex: '^/api/(.+?)(?:/)?$' }],
+            });
+
+            expect(normalizeRoute('/wp-admin', noCatchAll)).toBe('other');
+            expect(normalizeRoute('/api/anything', noCatchAll)).toBe('/api/[...slug]');
         });
 
         it('stays bounded under scanner traffic', () => {
@@ -52,28 +97,26 @@ describe('normalizeRoute', () => {
                 hostile.push(`/${i}/${i}/${i}`);
             }
 
-            const labels = new Set(hostile.map(normalizeRoute));
+            const labels = new Set(hostile.map(p => normalizeRoute(p, table)));
 
-            // 3000 distinct hostile paths must produce exactly one label.
-            expect(labels).toEqual(new Set(['other']));
+            // 3,000 distinct hostile paths produce exactly the two catch-all labels.
+            expect(labels).toEqual(new Set(['/[...slug]', '/api/[...slug]']));
         });
 
-        it('never exceeds the documented label bound for any input', () => {
+        it('never exceeds the table-derived bound for any input', () => {
             const paths = [
-                '/', '/search', '/repos', '/settings/general', '/browse/a/b/c',
-                '/api/health', '/api/commits', '/api/auth/session', '/_next/static/x.js',
-                '/wp-admin', '/api/bogus', '/random', '/api/9', '/..%2f', '/a/b/c/d/e',
+                '/', '/search', '/repos', '/browse/a/b/c', '/api/health',
+                '/api/commits', '/api/auth/session', '/_next/static/x.js',
+                '/wp-admin', '/api/bogus', '/random', '/..%2f', '/a/b/c/d/e',
             ];
             for (let i = 0; i < 500; i++) {
                 paths.push(`/junk${i}`, `/api/junk${i}`);
             }
 
-            const labels = new Set(paths.map(normalizeRoute));
+            const labels = new Set(paths.map(p => normalizeRoute(p, table)));
 
-            expect(labels.size).toBeLessThanOrEqual(MAX_ROUTE_LABELS);
-            // Known routes still resolve; only the unknown ones collapse.
+            expect(labels.size).toBeLessThanOrEqual(maxLabels);
             expect(labels).toContain('/api/health');
-            expect(labels).toContain('other');
         });
     });
 });

packages/web/src/httpMetrics.ts

@@ -1,93 +1,106 @@
 import { createLogger, env } from '@sourcebot/shared';
 import { subscribe } from 'node:diagnostics_channel';
+import { readFileSync } from 'node:fs';
 import type { IncomingMessage, ServerResponse } from 'node:http';
+import path from 'node:path';
 import { httpRequestDuration } from './promClient';
 
 const logger = createLogger('web-http-metrics');
 
+interface RoutesManifest {
+    staticRoutes: { page: string }[];
+    dynamicRoutes: { page: string; regex: string }[];
+}
+
+interface RouteTable {
+    staticPages: Set<string>;
+    dynamicRoutes: { page: string; regex: RegExp }[];
+}
+
+const OTHER_ROUTE = 'other';
+
 /**
- * Every path that isn't in this set is reported as `other`.
- *
- * Truncating path depth alone does not bound cardinality: the first segment is
- * client-supplied, and `/api/[...slug]` is a catch-all, so `/wp-admin`,
- * `/.env`, and `/api/<anything>` would each mint a new time series. Scanner or
- * bot traffic would then grow the series count without limit. Matching against
- * a known set instead bounds it to this size plus one, whatever gets requested.
- *
- * Adding a route here is deliberate. A missing one is reported as `other`, so
- * new routes lose granularity rather than breaking, and cardinality holds.
+ * Builds a route matcher from Next's routes-manifest. The manifest lists every
+ * defined route with a matching regex, ordered by Next's own resolution
+ * priority (specific routes before catch-alls), so first-match-wins here
+ * agrees with how the server actually routes the request.
  */
-const KNOWN_ROUTES = new Set([
-    '/',
-    '/_next',
-    '/askgh',
-    '/browse',
-    '/chat',
-    '/chats',
-    '/invite',
-    '/login',
-    '/oauth',
-    '/onboard',
-    '/redeem',
-    '/repos',
-    '/search',
-    '/settings',
-    '/signup',
-    '/slow',
-    '/api/auth',
-    '/api/avatar',
-    '/api/blame',
-    '/api/changelog',
-    '/api/chat',
-    '/api/commit',
-    '/api/commits',
-    '/api/diff',
-    '/api/ee',
-    '/api/files',
-    '/api/find_definitions',
-    '/api/find_references',
-    '/api/folder_contents',
-    '/api/health',
-    '/api/minidenticon',
-    '/api/models',
-    '/api/offers',
-    '/api/openapi.json',
-    '/api/repo-status',
-    '/api/repos',
-    '/api/search',
-    '/api/source',
-    '/api/stream_search',
-    '/api/symbols',
-    '/api/tree',
-    '/api/version',
-    '/api/webhook',
-]);
+export const buildRouteTable = (manifest: RoutesManifest): RouteTable => {
+    return {
+        staticPages: new Set(manifest.staticRoutes.map(route => route.page)),
+        dynamicRoutes: manifest.dynamicRoutes.map(route => ({
+            page: route.page,
+            regex: new RegExp(route.regex),
+        })),
+    };
+};
 
-const OTHER_ROUTE = 'other';
+let routeTable: RouteTable | undefined;
 
 /**
- * Collapses a request path into a bounded label.
+ * Loads the route table from the build's routes-manifest. Next's standalone
+ * server chdirs to the app directory on boot, so the manifest sits at
+ * `.next/routes-manifest.json` relative to cwd.
  *
- * Depth is truncated first, because repository and file paths are unbounded and
- * `/browse/github.com/org/repo/-/blob/src/index.ts` must not mint a series per
- * file viewed. API paths keep two segments so `/api/health` stays distinct from
- * `/api/commits`; everything else keeps one. The result is then matched against
- * `KNOWN_ROUTES`, which is what actually bounds the label set.
+ * Deriving routes from the manifest (rather than a hardcoded list) keeps the
+ * label set in sync with the app automatically: new routes appear at build
+ * time, and the label is the route pattern itself (`/browse/[...path]`), so
+ * cardinality is bounded by the number of defined routes no matter what gets
+ * requested.
  */
-export const normalizeRoute = (pathname: string): string => {
+export const initRouteTable = (manifest?: RoutesManifest): boolean => {
+    try {
+        const resolved = manifest ?? (JSON.parse(
+            readFileSync(path.join(process.cwd(), '.next', 'routes-manifest.json'), 'utf-8'),
+        ) as RoutesManifest);
+        routeTable = buildRouteTable(resolved);
+        logger.info(`Route table loaded: ${routeTable.staticPages.size} static, ${routeTable.dynamicRoutes.length} dynamic routes.`);
+        return true;
+    } catch (error) {
+        // Fail closed: without a table every request is labelled `other`, which
+        // loses granularity but can never grow the label set.
+        logger.error(`Failed to load routes-manifest; all routes will be reported as '${OTHER_ROUTE}': ${error}`);
+        return false;
+    }
+};
+
+/**
+ * Maps a request path to its route pattern. The raw path can't be used as a
+ * label: repository and file paths are unbounded, so `/browse/...` would mint
+ * a new time series for every file anyone views, and unknown paths (scanners,
+ * bots) would grow the set without limit. Matching against the app's own
+ * routes bounds the label set to the number of defined routes plus `/_next`
+ * and `other`.
+ */
+export const normalizeRoute = (pathname: string, table: RouteTable | undefined = routeTable): string => {
     const segments = pathname.split('/').filter(segment => segment.length > 0);
     if (segments.length === 0) {
         return '/';
     }
 
-    const depth = segments[0] === 'api' ? 2 : 1;
-    const candidate = `/${segments.slice(0, depth).join('/')}`;
+    // Asset requests are real traffic but not manifest routes.
+    if (segments[0] === '_next') {
+        return '/_next';
+    }
 
-    return KNOWN_ROUTES.has(candidate) ? candidate : OTHER_ROUTE;
-};
+    if (!table) {
+        return OTHER_ROUTE;
+    }
 
-/** Upper bound on distinct `route` label values, for tests and review. */
-export const MAX_ROUTE_LABELS = KNOWN_ROUTES.size + 1;
+    const canonical = `/${segments.join('/')}`;
+
+    if (table.staticPages.has(canonical)) {
+        return canonical;
+    }
+
+    for (const route of table.dynamicRoutes) {
+        if (route.regex.test(canonical)) {
+            return route.page;
+        }
+    }
+
+    return OTHER_ROUTE;
+};
 
 interface RequestStartMessage {
     response?: ServerResponse;
@@ -117,6 +130,10 @@ export const startHttpMetrics = (): void => {
     }
     subscribed = true;
 
+    if (!routeTable) {
+        initRouteTable();
+    }
+
     const metricsPort = Number(env.WEB_METRICS_PORT);
 
     subscribe('http.server.request.start', (message) => {