feat(web): record HTTP request duration metrics

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Added a manually triggered cloud image release workflow for isolated internal deployments. [#1566](https://github.com/sourcebot-dev/sourcebot/pull/1566)
 - Added Prometheus metrics for the web process, served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570)
+- Added an `http_request_duration_seconds` metric recording web request latency by route, method, and status. [#1571](https://github.com/sourcebot-dev/sourcebot/pull/1571)
 
 ### Fixed
 - Fixed the web process being capped at a ~4GiB heap regardless of how much memory the container has, which caused multi-second garbage collection pauses on larger deployments. [#1569](https://github.com/sourcebot-dev/sourcebot/pull/1569)

packages/web/src/httpMetrics.integration.test.ts

@@ -0,0 +1,89 @@
+import { afterAll, describe, expect, it } from 'vitest';
+import { createServer, Server } from 'node:http';
+import { initRouteTable, startHttpMetrics } from './httpMetrics';
+import { registry } from './promClient';
+
+const app = createServer((_req, res) => {
+    res.writeHead(200);
+    res.end('ok');
+});
+
+// Stands in for the real metrics server: requests to it must not be recorded,
+// since the diagnostics channels are process-wide and would otherwise make
+// every scrape observe itself.
+const metricsServer = createServer((_req, res) => {
+    res.writeHead(200);
+    res.end('# metrics');
+});
+
+const listen = (server: Server): Promise<number> => {
+    return new Promise(resolve => {
+        server.listen(0, () => resolve((server.address() as { port: number }).port));
+    });
+};
+
+afterAll(() => {
+    app.close();
+    metricsServer.close();
+});
+
+const countLines = (output: string): string[] => {
+    return output.split('\n').filter(line => line.startsWith('http_request_duration_seconds_count'));
+};
+
+describe('httpMetrics', () => {
+    it('records durations per route pattern and ignores the metrics port', async () => {
+        // The table is injected rather than read from disk so the test doesn't
+        // depend on a prior `next build` having produced routes-manifest.json.
+        initRouteTable({
+            staticRoutes: [{ page: '/api/health' }],
+            dynamicRoutes: [{ page: '/browse/[...path]', regex: '^/browse/(.+?)(?:/)?$' }],
+            rewrites: {
+                afterFiles: [{ source: '/api/mcp', regex: '^/api/mcp(?:/)?$' }],
+            },
+        });
+
+        // Both servers take an ephemeral port, then the metrics port is published
+        // to env before subscribing, so the test never depends on a fixed port.
+        const metricsPort = await listen(metricsServer);
+        const appPort = await listen(app);
+        process.env.WEB_METRICS_PORT = String(metricsPort);
+
+        startHttpMetrics();
+
+        await fetch(`http://127.0.0.1:${appPort}/api/health`);
+        await fetch(`http://127.0.0.1:${appPort}/browse/github.com/a/b/-/blob/x.ts`);
+        await fetch(`http://127.0.0.1:${appPort}/browse/github.com/c/d/-/blob/y.ts`);
+        await fetch(`http://127.0.0.1:${appPort}/api/mcp`);
+        await fetch(`http://127.0.0.1:${metricsPort}/metrics`);
+
+        // The finish channel fires after the response is flushed to the client.
+        await new Promise(resolve => setTimeout(resolve, 100));
+
+        const metrics = await registry.metrics();
+        const counts = countLines(metrics);
+
+        expect(counts.some(line => line.includes('route="/api/health"'))).toBe(true);
+        expect(counts.some(line => line.includes('route="/api/mcp"'))).toBe(true);
+        expect(counts.some(line => line.includes('status="200"'))).toBe(true);
+
+        // Two distinct file paths must collapse to the single route-pattern series.
+        const browse = counts.filter(line => line.includes('route="/browse/[...path]"'));
+        expect(browse).toHaveLength(1);
+        expect(browse[0].trim().endsWith('2')).toBe(true);
+
+        // Keep enough resolution to distinguish the long-tail stalls this
+        // metric is intended to expose rather than collapsing them into +Inf.
+        for (const upperBound of [15, 20, 30, 60]) {
+            expect(metrics).toContain(`le="${upperBound}"`);
+        }
+
+        // The scrape of the metrics port must not be recorded. Asserted on the
+        // total observation count rather than on the absence of a `/metrics`
+        // label: `/metrics` is not a known route, so it would land in `other`
+        // and an absent-label check would pass even with the filter removed.
+        const total = counts.reduce((sum, line) => sum + Number(line.trim().split(' ').pop()), 0);
+        expect(total).toBe(4);
+        expect(counts.some(line => line.includes('route="other"'))).toBe(false);
+    });
+});

packages/web/src/httpMetrics.test.ts

@@ -0,0 +1,187 @@
+import { describe, expect, it } from 'vitest';
+import { buildRouteTable, initRouteTable, normalizeRoute } from './httpMetrics';
+
+// Mirrors the shape and ordering of the real .next/routes-manifest.json:
+// dynamic routes are listed in Next's resolution priority, with catch-alls
+// after specific routes and the root catch-all last.
+const table = buildRouteTable({
+    staticRoutes: [
+        { page: '/' },
+        { page: '/search' },
+        { page: '/repos' },
+        { page: '/api/health' },
+        { page: '/api/commits' },
+    ],
+    dynamicRoutes: [
+        { page: '/api/auth/[...nextauth]', regex: '^/api/auth/(.+?)(?:/)?$' },
+        { page: '/api/repos/[repoId]/image', regex: '^/api/repos/([^/]+?)/image(?:/)?$' },
+        { page: '/api/[...slug]', regex: '^/api/(.+?)(?:/)?$' },
+        { page: '/browse/[...path]', regex: '^/browse/(.+?)(?:/)?$' },
+        { page: '/settings/[...slug]', regex: '^/settings/(.+?)(?:/)?$' },
+        { page: '/[...slug]', regex: '^/(.+?)(?:/)?$' },
+    ],
+    rewrites: {
+        afterFiles: [
+            { source: '/ingest/:path*', regex: '^/ingest(?:/(.+?))?(?:/)?$' },
+            { source: '/.well-known/oauth-authorization-server', regex: '^/\\.well-known/oauth-authorization-server(?:/)?$' },
+            { source: '/.well-known/oauth-protected-resource/:path*', regex: '^/\\.well-known/oauth-protected-resource(?:/(.+?))?(?:/)?$' },
+            { source: '/register', regex: '^/register(?:/)?$' },
+            { source: '/api/mcp', regex: '^/api/mcp(?:/)?$' },
+            { source: '/scim/v2/:path*', regex: '^/scim/v2(?:/(.+?))?(?:/)?$' },
+        ],
+    },
+});
+
+const maxLabels = table.staticPages.size
+    + table.dynamicRoutes.length
+    + table.beforeFilesRewrites.length
+    + table.afterFilesRewrites.length
+    + table.fallbackRewrites.length
+    + 2; // + '/_next', 'other'
+
+describe('normalizeRoute', () => {
+    it('discards a previously loaded table when initialization fails', () => {
+        expect(initRouteTable({
+            staticRoutes: [{ page: '/api/health' }],
+            dynamicRoutes: [],
+        })).toBe(true);
+        expect(normalizeRoute('/api/health')).toBe('/api/health');
+
+        expect(initRouteTable({
+            staticRoutes: [],
+            dynamicRoutes: [{ page: '/broken', regex: '[' }],
+        })).toBe(false);
+        expect(normalizeRoute('/api/health')).toBe('other');
+    });
+
+    it('maps the root path', () => {
+        expect(normalizeRoute('/', table)).toBe('/');
+        expect(normalizeRoute('', table)).toBe('/');
+    });
+
+    it('matches static routes exactly', () => {
+        expect(normalizeRoute('/api/health', table)).toBe('/api/health');
+        expect(normalizeRoute('/search', table)).toBe('/search');
+    });
+
+    it('labels dynamic routes with their route pattern', () => {
+        expect(normalizeRoute('/api/auth/callback/github', table)).toBe('/api/auth/[...nextauth]');
+        expect(normalizeRoute('/api/repos/42/image', table)).toBe('/api/repos/[repoId]/image');
+        expect(normalizeRoute('/settings/connections/42', table)).toBe('/settings/[...slug]');
+    });
+
+    it('respects manifest ordering: specific routes win over catch-alls', () => {
+        // /api/auth/... must hit [...nextauth], not the /api/[...slug] catch-all.
+        expect(normalizeRoute('/api/auth/session', table)).toBe('/api/auth/[...nextauth]');
+        // Unknown API paths fall through to the catch-all that actually serves them.
+        expect(normalizeRoute('/api/not-a-real-route', table)).toBe('/api/[...slug]');
+    });
+
+    it('labels rewritten paths with their public source pattern', () => {
+        expect(normalizeRoute('/api/mcp', table)).toBe('/api/mcp');
+        expect(normalizeRoute('/scim/v2/Users/42', table)).toBe('/scim/v2/:path*');
+        expect(normalizeRoute('/.well-known/oauth-authorization-server', table))
+            .toBe('/.well-known/oauth-authorization-server');
+        expect(normalizeRoute('/.well-known/oauth-protected-resource/api/mcp', table))
+            .toBe('/.well-known/oauth-protected-resource/:path*');
+        expect(normalizeRoute('/register', table)).toBe('/register');
+        expect(normalizeRoute('/ingest/events', table)).toBe('/ingest/:path*');
+    });
+
+    it('matches rewrites in Next routing order', () => {
+        const precedenceTable = buildRouteTable({
+            staticRoutes: [
+                { page: '/docs' },
+                { page: '/api/health' },
+            ],
+            dynamicRoutes: [
+                { page: '/api/[...slug]', regex: '^/api/(.+?)(?:/)?$' },
+                { page: '/browse/[...path]', regex: '^/browse/(.+?)(?:/)?$' },
+            ],
+            rewrites: {
+                beforeFiles: [{ source: '/docs/:path*', regex: '^/docs(?:/(.+?))?(?:/)?$' }],
+                afterFiles: [{ source: '/api/:path*', regex: '^/api/(.+?)(?:/)?$' }],
+                fallback: [{ source: '/:path*', regex: '^/(.+?)(?:/)?$' }],
+            },
+        });
+
+        expect(normalizeRoute('/docs', precedenceTable)).toBe('/docs/:path*');
+        expect(normalizeRoute('/api/health', precedenceTable)).toBe('/api/health');
+        expect(normalizeRoute('/api/mcp', precedenceTable)).toBe('/api/:path*');
+        expect(normalizeRoute('/browse/org/repo', precedenceTable)).toBe('/browse/[...path]');
+        expect(normalizeRoute('/unmatched', precedenceTable)).toBe('/:path*');
+    });
+
+    it('collapses unbounded repository and file paths to one label', () => {
+        const a = normalizeRoute('/browse/github.com/org/repo/-/blob/src/index.ts', table);
+        const b = normalizeRoute('/browse/github.com/other/repo/-/blob/lib/other.ts', table);
+
+        expect(a).toBe('/browse/[...path]');
+        expect(b).toBe(a);
+    });
+
+    it('is unaffected by trailing or duplicate slashes', () => {
+        expect(normalizeRoute('/search/', table)).toBe('/search');
+        expect(normalizeRoute('//search//', table)).toBe('/search');
+        expect(normalizeRoute('/api/health/', table)).toBe('/api/health');
+    });
+
+    it('labels asset requests /_next without consulting the table', () => {
+        expect(normalizeRoute('/_next/static/chunks/main.js', table)).toBe('/_next');
+        expect(normalizeRoute('/_next/image', undefined)).toBe('/_next');
+    });
+
+    it('reports everything as other when no table is loaded', () => {
+        expect(normalizeRoute('/api/health', undefined)).toBe('other');
+        expect(normalizeRoute('/search', undefined)).toBe('other');
+    });
+
+    describe('cardinality bounding', () => {
+        it('routes scanner traffic to catch-alls, not new labels', () => {
+            expect(normalizeRoute('/wp-admin', table)).toBe('/[...slug]');
+            expect(normalizeRoute('/.env', table)).toBe('/[...slug]');
+            expect(normalizeRoute('/api/12345', table)).toBe('/api/[...slug]');
+        });
+
+        it('reports unmatched paths as other when there is no root catch-all', () => {
+            const noCatchAll = buildRouteTable({
+                staticRoutes: [{ page: '/search' }],
+                dynamicRoutes: [{ page: '/api/[...slug]', regex: '^/api/(.+?)(?:/)?$' }],
+            });
+
+            expect(normalizeRoute('/wp-admin', noCatchAll)).toBe('other');
+            expect(normalizeRoute('/api/anything', noCatchAll)).toBe('/api/[...slug]');
+        });
+
+        it('stays bounded under scanner traffic', () => {
+            const hostile: string[] = [];
+            for (let i = 0; i < 1000; i++) {
+                hostile.push(`/scan-${i}`);
+                hostile.push(`/api/scan-${i}`);
+                hostile.push(`/${i}/${i}/${i}`);
+            }
+
+            const labels = new Set(hostile.map(p => normalizeRoute(p, table)));
+
+            // 3,000 distinct hostile paths produce exactly the two catch-all labels.
+            expect(labels).toEqual(new Set(['/[...slug]', '/api/[...slug]']));
+        });
+
+        it('never exceeds the table-derived bound for any input', () => {
+            const paths = [
+                '/', '/search', '/repos', '/browse/a/b/c', '/api/health',
+                '/api/commits', '/api/auth/session', '/_next/static/x.js',
+                '/api/mcp', '/scim/v2/Users/42', '/ingest/events',
+                '/wp-admin', '/api/bogus', '/random', '/..%2f', '/a/b/c/d/e',
+            ];
+            for (let i = 0; i < 500; i++) {
+                paths.push(`/junk${i}`, `/api/junk${i}`);
+            }
+
+            const labels = new Set(paths.map(p => normalizeRoute(p, table)));
+
+            expect(labels.size).toBeLessThanOrEqual(maxLabels);
+            expect(labels).toContain('/api/health');
+        });
+    });
+});