feat(web): expose Prometheus metrics for the web process (#1570)

* feat(web): expose Prometheus metrics for the web process

The web process had no instrumentation of any kind, so heap usage, GC
duration, and event loop lag were invisible for the process that serves all
application traffic. Diagnosing a recent GC-thrash incident required reading
cgroup files inside the container and inferring the rest from trace spans.

Mirror the backend's prom-client setup and serve it on its own port
(WEB_METRICS_PORT, default 3070) rather than as a Next.js route, so scraping
bypasses app middleware and is not reachable through the ingress.

Also adds nodejs_heap_size_limit_bytes, which prom-client's default metrics
omit. Without the ceiling, heap usage alone cannot distinguish a busy process
from one pinned at its limit running back-to-back full mark-compacts.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* docs: add changelog entry for web process metrics

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix changelog

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: brendan-kellam

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Added a manually triggered cloud image release workflow for isolated internal deployments. [#1566](https://github.com/sourcebot-dev/sourcebot/pull/1566)
+- Added Prometheus metrics for the web process, served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570)
 
 ### Fixed
 - Fixed the web process being capped at a ~4GiB heap regardless of how much memory the container has, which caused multi-second garbage collection pauses on larger deployments. [#1569](https://github.com/sourcebot-dev/sourcebot/pull/1569)

packages/shared/src/env.server.ts

@@ -174,6 +174,9 @@ const options = {
 
         WORKER_API_URL: z.string().url().default("http://localhost:3060"),
 
+        // Port the web process serves its Prometheus metrics on.
+        WEB_METRICS_PORT: numberSchema.default(3070),
+
         // Auth
         AUTH_SECRET: z.string(),
         AUTH_URL: z.string().url(),

packages/web/package.json

@@ -173,6 +173,7 @@
     "posthog-js": "^1.369.0",
     "posthog-node": "^5.24.15",
     "pretty-bytes": "^6.1.1",
+    "prom-client": "^15.1.3",
     "psl": "^1.15.0",
     "react": "19.2.4",
     "react-day-picker": "^9.14.0",

packages/web/src/instrumentation.ts

@@ -9,6 +9,11 @@ export async function register() {
         await import('./sentry.edge.config');
     }
 
+    if (process.env.NEXT_RUNTIME === 'nodejs') {
+        const { startMetricsServer } = await import('./metricsServer');
+        startMetricsServer();
+    }
+
     if (process.env.NEXT_RUNTIME === 'nodejs') {
         const { initialize } = await import('./initialize');
         await initialize();

packages/web/src/metricsServer.ts

@@ -0,0 +1,50 @@
+import { createLogger, env } from '@sourcebot/shared';
+import { createServer, Server } from 'node:http';
+import { registry } from './promClient';
+
+const logger = createLogger('web-metrics-server');
+
+/**
+ * Serves the web process' Prometheus metrics on its own port, rather than as a
+ * Next.js route, so that scraping doesn't pass through the app's middleware or
+ * get exposed publicly through the ingress.
+ */
+export const startMetricsServer = (): Server | undefined => {
+    // Guard against a missing port: `listen(undefined)` binds a random one, which
+    // would leave the scrape target silently broken instead of loudly absent.
+    const port = Number(env.WEB_METRICS_PORT);
+    if (!Number.isInteger(port) || port <= 0) {
+        logger.error(`Invalid WEB_METRICS_PORT '${env.WEB_METRICS_PORT}'; metrics server not started.`);
+        return undefined;
+    }
+
+    const server = createServer(async (req, res) => {
+        if (req.url !== '/metrics') {
+            res.writeHead(404);
+            res.end();
+            return;
+        }
+
+        try {
+            const metrics = await registry.metrics();
+            res.writeHead(200, { 'Content-Type': registry.contentType });
+            res.end(metrics);
+        } catch (error) {
+            logger.error(`Failed to collect metrics: ${error}`);
+            res.writeHead(500);
+            res.end();
+        }
+    });
+
+    // Metrics must never take down the web server, so swallow listen failures
+    // (a port collision, most likely) instead of letting the 'error' event throw.
+    server.on('error', (error) => {
+        logger.error(`Metrics server error: ${error}`);
+    });
+
+    server.listen(port, () => {
+        logger.info(`Web metrics server listening on port ${port}`);
+    });
+
+    return server;
+};

packages/web/src/promClient.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest';
+import { registry } from './promClient';
+
+const metricNames = (output: string): Set<string> => {
+    return new Set(
+        output
+            .split('\n')
+            .filter(line => line.length > 0 && !line.startsWith('#'))
+            .map(line => line.split(/[ {]/)[0])
+    );
+};
+
+describe('web promClient', () => {
+    it('exposes the metrics needed to diagnose heap pressure', async () => {
+        const names = metricNames(await registry.metrics());
+
+        expect(names).toContain('nodejs_heap_size_limit_bytes');
+        expect(names).toContain('nodejs_heap_size_used_bytes');
+        expect(names).toContain('nodejs_eventloop_lag_p99_seconds');
+    });
+
+    it('registers the gc duration histogram', () => {
+        // Asserted via the registry rather than the rendered output: the histogram
+        // emits no series until a garbage collection has actually been observed.
+        expect(registry.getSingleMetric('nodejs_gc_duration_seconds')).toBeDefined();
+    });
+
+    it('reports a plausible heap size limit', async () => {
+        const output = await registry.metrics();
+        const line = output.split('\n').find(l => l.startsWith('nodejs_heap_size_limit_bytes '));
+
+        expect(line).toBeDefined();
+
+        const limit = Number(line!.split(' ')[1]);
+        expect(Number.isFinite(limit)).toBe(true);
+        // Any real V8 heap limit is well above 100MB and well below 100GB.
+        expect(limit).toBeGreaterThan(100 * 1024 * 1024);
+        expect(limit).toBeLessThan(100 * 1024 * 1024 * 1024);
+    });
+
+    it('can be collected repeatedly', async () => {
+        const first = await registry.metrics();
+        const second = await registry.metrics();
+
+        expect(metricNames(first)).toEqual(metricNames(second));
+    });
+});

packages/web/src/promClient.ts

@@ -0,0 +1,21 @@
+import client, { Gauge, Registry } from 'prom-client';
+import { getHeapStatistics } from 'node:v8';
+
+export const registry = new Registry();
+
+// `collectDefaultMetrics` reports heap usage but not the ceiling it's measured
+// against, and usage alone can't distinguish "busy" from "out of room". Without
+// the limit there's no way to tell whether V8 is doing cheap incremental
+// collections or is pinned at its ceiling running full mark-compacts.
+const heapSizeLimit = new Gauge({
+    name: 'nodejs_heap_size_limit_bytes',
+    help: 'V8 heap size limit in bytes',
+    collect() {
+        this.set(getHeapStatistics().heap_size_limit);
+    },
+});
+registry.registerMetric(heapSizeLimit);
+
+client.collectDefaultMetrics({
+    register: registry,
+});

yarn.lock

@@ -9291,6 +9291,7 @@ __metadata:
     posthog-js: "npm:^1.369.0"
     posthog-node: "npm:^5.24.15"
     pretty-bytes: "npm:^6.1.1"
+    prom-client: "npm:^15.1.3"
     psl: "npm:^1.15.0"
     raw-loader: "npm:^4.0.2"
     react: "npm:19.2.4"