diff --git a/CHANGELOG.md b/CHANGELOG.md index ff5090ed0..b7b8a4186 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added a manually triggered cloud image release workflow for isolated internal deployments. [#1566](https://github.com/sourcebot-dev/sourcebot/pull/1566) +- Added Prometheus metrics for the web process, served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570) ### Fixed - Fixed the web process being capped at a ~4GiB heap regardless of how much memory the container has, which caused multi-second garbage collection pauses on larger deployments. [#1569](https://github.com/sourcebot-dev/sourcebot/pull/1569) diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts index 2edd9050c..52e4f1d25 100644 --- a/packages/shared/src/env.server.ts +++ b/packages/shared/src/env.server.ts @@ -174,6 +174,9 @@ const options = { WORKER_API_URL: z.string().url().default("http://localhost:3060"), + // Port the web process serves its Prometheus metrics on. + WEB_METRICS_PORT: numberSchema.default(3070), + // Auth AUTH_SECRET: z.string(), AUTH_URL: z.string().url(), diff --git a/packages/web/package.json b/packages/web/package.json index f3e8d2715..91b7a0ad6 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -173,6 +173,7 @@ "posthog-js": "^1.369.0", "posthog-node": "^5.24.15", "pretty-bytes": "^6.1.1", + "prom-client": "^15.1.3", "psl": "^1.15.0", "react": "19.2.4", "react-day-picker": "^9.14.0", diff --git a/packages/web/src/instrumentation.ts b/packages/web/src/instrumentation.ts index 7bea4bbf0..e64926611 100644 --- a/packages/web/src/instrumentation.ts +++ b/packages/web/src/instrumentation.ts @@ -9,6 +9,11 @@ export async function register() { await import('./sentry.edge.config'); } + if (process.env.NEXT_RUNTIME === 'nodejs') { + const { startMetricsServer } = await import('./metricsServer'); + startMetricsServer(); + } + if (process.env.NEXT_RUNTIME === 'nodejs') { const { initialize } = await import('./initialize'); await initialize(); diff --git a/packages/web/src/metricsServer.ts b/packages/web/src/metricsServer.ts new file mode 100644 index 000000000..4755bbed6 --- /dev/null +++ b/packages/web/src/metricsServer.ts @@ -0,0 +1,50 @@ +import { createLogger, env } from '@sourcebot/shared'; +import { createServer, Server } from 'node:http'; +import { registry } from './promClient'; + +const logger = createLogger('web-metrics-server'); + +/** + * Serves the web process' Prometheus metrics on its own port, rather than as a + * Next.js route, so that scraping doesn't pass through the app's middleware or + * get exposed publicly through the ingress. + */ +export const startMetricsServer = (): Server | undefined => { + // Guard against a missing port: `listen(undefined)` binds a random one, which + // would leave the scrape target silently broken instead of loudly absent. + const port = Number(env.WEB_METRICS_PORT); + if (!Number.isInteger(port) || port <= 0) { + logger.error(`Invalid WEB_METRICS_PORT '${env.WEB_METRICS_PORT}'; metrics server not started.`); + return undefined; + } + + const server = createServer(async (req, res) => { + if (req.url !== '/metrics') { + res.writeHead(404); + res.end(); + return; + } + + try { + const metrics = await registry.metrics(); + res.writeHead(200, { 'Content-Type': registry.contentType }); + res.end(metrics); + } catch (error) { + logger.error(`Failed to collect metrics: ${error}`); + res.writeHead(500); + res.end(); + } + }); + + // Metrics must never take down the web server, so swallow listen failures + // (a port collision, most likely) instead of letting the 'error' event throw. + server.on('error', (error) => { + logger.error(`Metrics server error: ${error}`); + }); + + server.listen(port, () => { + logger.info(`Web metrics server listening on port ${port}`); + }); + + return server; +}; diff --git a/packages/web/src/promClient.test.ts b/packages/web/src/promClient.test.ts new file mode 100644 index 000000000..eb571555a --- /dev/null +++ b/packages/web/src/promClient.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest'; +import { registry } from './promClient'; + +const metricNames = (output: string): Set => { + return new Set( + output + .split('\n') + .filter(line => line.length > 0 && !line.startsWith('#')) + .map(line => line.split(/[ {]/)[0]) + ); +}; + +describe('web promClient', () => { + it('exposes the metrics needed to diagnose heap pressure', async () => { + const names = metricNames(await registry.metrics()); + + expect(names).toContain('nodejs_heap_size_limit_bytes'); + expect(names).toContain('nodejs_heap_size_used_bytes'); + expect(names).toContain('nodejs_eventloop_lag_p99_seconds'); + }); + + it('registers the gc duration histogram', () => { + // Asserted via the registry rather than the rendered output: the histogram + // emits no series until a garbage collection has actually been observed. + expect(registry.getSingleMetric('nodejs_gc_duration_seconds')).toBeDefined(); + }); + + it('reports a plausible heap size limit', async () => { + const output = await registry.metrics(); + const line = output.split('\n').find(l => l.startsWith('nodejs_heap_size_limit_bytes ')); + + expect(line).toBeDefined(); + + const limit = Number(line!.split(' ')[1]); + expect(Number.isFinite(limit)).toBe(true); + // Any real V8 heap limit is well above 100MB and well below 100GB. + expect(limit).toBeGreaterThan(100 * 1024 * 1024); + expect(limit).toBeLessThan(100 * 1024 * 1024 * 1024); + }); + + it('can be collected repeatedly', async () => { + const first = await registry.metrics(); + const second = await registry.metrics(); + + expect(metricNames(first)).toEqual(metricNames(second)); + }); +}); diff --git a/packages/web/src/promClient.ts b/packages/web/src/promClient.ts new file mode 100644 index 000000000..294159d31 --- /dev/null +++ b/packages/web/src/promClient.ts @@ -0,0 +1,21 @@ +import client, { Gauge, Registry } from 'prom-client'; +import { getHeapStatistics } from 'node:v8'; + +export const registry = new Registry(); + +// `collectDefaultMetrics` reports heap usage but not the ceiling it's measured +// against, and usage alone can't distinguish "busy" from "out of room". Without +// the limit there's no way to tell whether V8 is doing cheap incremental +// collections or is pinned at its ceiling running full mark-compacts. +const heapSizeLimit = new Gauge({ + name: 'nodejs_heap_size_limit_bytes', + help: 'V8 heap size limit in bytes', + collect() { + this.set(getHeapStatistics().heap_size_limit); + }, +}); +registry.registerMetric(heapSizeLimit); + +client.collectDefaultMetrics({ + register: registry, +}); diff --git a/yarn.lock b/yarn.lock index 9f0096f20..084538d10 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9291,6 +9291,7 @@ __metadata: posthog-js: "npm:^1.369.0" posthog-node: "npm:^5.24.15" pretty-bytes: "npm:^6.1.1" + prom-client: "npm:^15.1.3" psl: "npm:^1.15.0" raw-loader: "npm:^4.0.2" react: "npm:19.2.4"