fix(backend): cap repo-index metric cardinality and add OOM diagnostics

Add Node's --heapsnapshot-near-heap-limit to the backend program so a
heap snapshot is captured to a persisted directory if the process
approaches its heap limit again, giving a definitive object graph
instead of only a generic "JavaScript heap out of memory" crash log.

Also fix an identified metric-cardinality leak: repo-scoped
Prometheus metrics keyed by repo name are never cleared, so every
distinct repo name observed over the process's lifetime stays
resident in the registry forever, even after the repo is deleted.
Remove those time series once a repo is actually deleted.

Author: better-stack-hq[bot]

entrypoint.sh

@@ -104,6 +104,12 @@ if [ ! -d "$DATA_CACHE_DIR" ]; then
     mkdir -p "$DATA_CACHE_DIR"
 fi
 
+# Create a directory for heap snapshots. If the backend process approaches its heap
+# limit (e.g., due to a memory leak), Node will dump a snapshot here (see
+# --heapsnapshot-near-heap-limit in supervisord.conf) so the cause can be diagnosed
+# post-mortem instead of only seeing a generic "JavaScript heap out of memory" crash.
+mkdir -p "$DATA_CACHE_DIR/heap-snapshots"
+
 # As of v5, SOURCEBOT_ENCRYPTION_KEY must be provided explicitly via an environment variable.
 # @see: https://docs.sourcebot.dev/docs/upgrade/v4-to-v5-guide
 if [ -z "$SOURCEBOT_ENCRYPTION_KEY" ]; then

packages/backend/src/promClient.ts

@@ -91,4 +91,24 @@ export class PromClient {
             register: this.registry,
         });
     }
+
+    /**
+     * Removes all time series associated with a given repo across every repo-scoped
+     * metric. The `repo` label is a dynamic, user-controlled value (repo name), and
+     * prom-client never forgets a label combination on its own once it's been
+     * observed. Without this cleanup, every distinct repo name ever seen over the
+     * process's lifetime stays resident in memory forever, even after the repo is
+     * deleted. Call this once a repo is permanently removed (e.g., after a CLEANUP
+     * job deletes it) to keep the registry's memory footprint bounded by the current
+     * set of repos rather than the historical set.
+     */
+    public removeRepoMetrics(repoName: string) {
+        for (const type of ['index', 'cleanup']) {
+            this.activeRepoIndexJobs.remove({ repo: repoName, type });
+            this.pendingRepoIndexJobs.remove({ repo: repoName, type });
+            this.repoIndexJobReattemptsTotal.remove({ repo: repoName, type });
+            this.repoIndexJobFailTotal.remove({ repo: repoName, type });
+            this.repoIndexJobSuccessTotal.remove({ repo: repoName, type });
+        }
+    }
 }

packages/backend/src/repoIndexManager.ts

@@ -614,6 +614,10 @@ export class RepoIndexManager {
                     where: { id: jobData.repoId },
                 });
 
+                // The repo no longer exists, so drop its time series from the metrics
+                // registry rather than retaining them for the lifetime of the process.
+                this.promClient.removeRepoMetrics(repo.name);
+
                 logger.debug(`Completed cleanup job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`);
             }
 

supervisord.conf

@@ -31,4 +31,8 @@ autorestart=true
 startretries=3
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
-redirect_stderr=true
+redirect_stderr=true
+; Dump up to 3 heap snapshots to a persisted directory if the process approaches
+; its heap limit, so a real memory leak (as opposed to a transient spike) can be
+; diagnosed from the actual retained object graph instead of guessing from logs.
+environment=NODE_OPTIONS="--heapsnapshot-near-heap-limit=3 --diagnostic-dir=%(ENV_DATA_CACHE_DIR)s/heap-snapshots"