From a42d131b07f2a2917dd96466b02f6e40f6d4c13c Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 7 Jul 2026 20:16:00 -0700 Subject: [PATCH 01/15] feat(backend): add BullMQ JobManager framework Introduces a JobManager over BullMQ: work-queue Workloads with a ProcessContext, CronWorkloads multiplexed onto a shared cron worker with a reconcile() sweep helper, a JobProducer that owns the queues and the deduplicated enqueue path, and a Redis-backed read model (status/jobDetail). Wired into the backend entrypoint with a demo workload. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/backend/src/index.ts | 48 ++++ packages/backend/src/jobManager.test.ts | 114 +++++++++ packages/backend/src/jobManager.ts | 309 ++++++++++++++++++++++++ packages/backend/src/jobProducer.ts | 32 +++ packages/backend/src/types.ts | 86 ++++++- 5 files changed, 588 insertions(+), 1 deletion(-) create mode 100644 packages/backend/src/jobManager.test.ts create mode 100644 packages/backend/src/jobManager.ts create mode 100644 packages/backend/src/jobProducer.ts diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index b97fe248f..379ca388c 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -20,6 +20,8 @@ import { shutdownPosthog } from "./posthog.js"; import { PromClient } from './promClient.js'; import { RepoIndexManager } from "./repoIndexManager.js"; import { redis } from "./redis.js"; +import { BullMQJobManager, reconcile } from "./jobManager.js"; +import { QueueSpec, Workload } from "./types.js"; const logger = createLogger('backend-entrypoint'); @@ -83,6 +85,51 @@ const api = new Api( logger.info('Worker started.'); +// Background jobs run through the JobManager (BullMQ/Redis as the source of truth). Phase 0 +// wires the framework here in place of the old per-manager pollers; the real workloads +// (repo-index, connection-sync, permission syncers) are ported onto it in subsequent phases. +const jobManager = new BullMQJobManager(redis); + + +const demoSpec: QueueSpec<{ id: string }> = { + name: 'demo', + dedupKey: ({ id }) => `demo:${id}`, + jobOptions: { + attempts: 2, + backoff: { type: 'fixed', delayMs: 1000 }, + keep: { completed: 50, failed: 50 }, + }, +}; + +const demoWorkload: Workload<{ id: string }, { id: string; ranAt: number }> = { + spec: demoSpec, + concurrency: 2, + process: async ({ data: { id }, jobId, attemptsMade }) => { + logger.info(`demo: processing "${id}" (job ${jobId}, attempt ${attemptsMade})`); + await new Promise((resolve) => setTimeout(resolve, 1500)); + if (id === 'gamma') { + throw new Error(`demo: simulated failure processing "${id}"`); + } + return { id, ranAt: Date.now() }; + }, + onTerminalFailure: async ({ id }, err) => { + logger.warn(`demo: "${id}" failed terminally: ${err.message}`); + }, +}; +jobManager.register(demoWorkload); + +// The reconcile sweep for the demo queue, expressed as a cron workload: every 15s it triggers +// a fixed set into `demo`. Dedup keeps an in-flight item from re-queuing, but a finished one +// re-enqueues on the next tick, so the loop visibly cycles. +jobManager.registerCron(reconcile({ + name: 'demo-sweep', + schedule: { every: '15s' }, + target: 'demo', + scan: async () => ['alpha', 'beta', 'gamma'].map((id) => ({ id })), +})); + +await jobManager.start(); + const listenToShutdownSignals = () => { const signals = SHUTDOWN_SIGNALS; @@ -104,6 +151,7 @@ const listenToShutdownSignals = () => { await auditLogPruner.dispose() await attachmentPruner.dispose() await configManager.dispose() + await jobManager.stop(); await prisma.$disconnect(); await redis.quit(); diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts new file mode 100644 index 000000000..c65b1c06d --- /dev/null +++ b/packages/backend/src/jobManager.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, test, vi } from 'vitest'; + +// The module under test creates a logger at import time; stub it so importing pure helpers +// has no side effects (mirrors repoIndexManager.test.ts). +vi.mock('@sourcebot/shared', () => ({ + createLogger: vi.fn(() => ({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + })), +})); + +// Mock the constants module directly so its env-derived cache-dir paths don't load. +vi.mock('./constants.js', () => ({ + WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, +})); + +import { normalizeJobState, parseDuration, reconcile } from './jobManager.js'; + +describe('parseDuration', () => { + test.each([ + ['500ms', 500], + ['30s', 30_000], + ['5m', 300_000], + ['6h', 21_600_000], + ['1d', 86_400_000], + ])('parses %s', (input, expected) => { + expect(parseDuration(input)).toBe(expected); + }); + + test('trims surrounding whitespace', () => { + expect(parseDuration(' 10m ')).toBe(600_000); + }); + + test.each(['', '5', 'm', '5x', '1.5h', '-5m', '5 m'])('throws on malformed "%s"', (input) => { + expect(() => parseDuration(input)).toThrow(); + }); +}); + +describe('normalizeJobState', () => { + test.each([ + 'waiting', + 'active', + 'delayed', + 'completed', + 'failed', + 'paused', + ])('passes through "%s"', (state) => { + expect(normalizeJobState(state)).toBe(state); + }); + + test('collapses prioritized and waiting-children to waiting', () => { + expect(normalizeJobState('prioritized')).toBe('waiting'); + expect(normalizeJobState('waiting-children')).toBe('waiting'); + }); + + test('maps anything unrecognized to unknown', () => { + expect(normalizeJobState('something-else')).toBe('unknown'); + expect(normalizeJobState('unknown')).toBe('unknown'); + }); +}); + +describe('reconcile', () => { + test('produces a cron workload carrying the given name and schedule', () => { + const cron = reconcile({ + name: 'x-sweep', + schedule: { every: '5m' }, + target: 'x', + scan: async () => [], + }); + expect(cron.name).toBe('x-sweep'); + expect(cron.schedule).toEqual({ every: '5m' }); + }); + + test('handler triggers each scanned item into the target, in order', async () => { + const triggered: Array<{ workload: string; data: unknown }> = []; + const cron = reconcile({ + name: 'x-sweep', + schedule: { pattern: '*/5 * * * *' }, + target: 'x', + scan: async () => [{ id: 'a' }, { id: 'b' }], + }); + + await cron.handler({ + trigger: async (workload, data) => { + triggered.push({ workload, data }); + }, + }); + + expect(triggered).toEqual([ + { workload: 'x', data: { id: 'a' } }, + { workload: 'x', data: { id: 'b' } }, + ]); + }); + + test('handler triggers nothing when scan returns empty', async () => { + let calls = 0; + const cron = reconcile({ + name: 'empty-sweep', + schedule: { every: '1m' }, + target: 'x', + scan: async () => [], + }); + + await cron.handler({ + trigger: async () => { + calls += 1; + }, + }); + + expect(calls).toBe(0); + }); +}); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts new file mode 100644 index 000000000..cd7689ef5 --- /dev/null +++ b/packages/backend/src/jobManager.ts @@ -0,0 +1,309 @@ +import * as Sentry from "@sentry/node"; +import { createLogger } from "@sourcebot/shared"; +import { Job, Queue, Worker } from "bullmq"; +import { Redis } from "ioredis"; +import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; +import { JobProducer } from "./jobProducer.js"; +import { CronWorkload, JobDetail, JobManager, QueueCounts, Schedule, Workload } from "./types.js"; + +const LOG_TAG = 'job-manager'; +const logger = createLogger(LOG_TAG); + +const CRON_QUEUE_NAME = 'cron'; +const CRON_KEEP_COMPLETED = 50; +const CRON_KEEP_FAILED = 200; + +const DURATION_UNITS_MS: Record = { + ms: 1, + s: 1000, + m: 1000 * 60, + h: 1000 * 60 * 60, + d: 1000 * 60 * 60 * 24, +}; + +export const parseDuration = (value: string): number => { + const match = /^(\d+)(ms|s|m|h|d)$/.exec(value.trim()); + if (!match) { + throw new Error(`Invalid duration "${value}". Expected e.g. "500ms", "30s", "5m", "6h", "1d".`); + } + return Number(match[1]) * DURATION_UNITS_MS[match[2]]; +}; + +export const normalizeJobState = (state: string): JobDetail['state'] => { + switch (state) { + case 'waiting': + case 'active': + case 'delayed': + case 'completed': + case 'failed': + case 'paused': + return state; + case 'prioritized': + case 'waiting-children': + return 'waiting'; + default: + return 'unknown'; + } +}; + +const scheduleToRepeat = (schedule: Schedule) => + 'pattern' in schedule ? { pattern: schedule.pattern } : { every: parseDuration(schedule.every) }; + +export function reconcile(opts: { + name: string; + schedule: Schedule; + target: string; + scan: () => Promise; +}): CronWorkload { + return { + name: opts.name, + schedule: opts.schedule, + handler: async ({ trigger }) => { + const items = await opts.scan(); + for (const item of items) { + await trigger(opts.target, item); + } + }, + }; +} + +export class BullMQJobManager implements JobManager { + private readonly workloads = new Map>(); + private readonly cronWorkloads = new Map(); + private readonly workers = new Map(); + private readonly producer: JobProducer; + private cronQueue?: Queue; + private cronWorker?: Worker; + private readonly abortController = new AbortController(); + + constructor(private readonly connection: Redis) { + this.producer = new JobProducer(connection); + } + + register(workload: Workload): void { + const name = workload.spec.name; + if (this.workloads.has(name)) { + throw new Error(`Workload "${name}" is already registered`); + } + this.workloads.set(name, workload as unknown as Workload); + } + + registerCron(cron: CronWorkload): void { + if (this.cronWorkloads.has(cron.name)) { + throw new Error(`Cron workload "${cron.name}" is already registered`); + } + this.cronWorkloads.set(cron.name, cron); + } + + async start(): Promise { + if (this.workloads.size === 0 && this.cronWorkloads.size === 0) { + logger.debug('start() called with nothing registered; nothing to do'); + return; + } + + for (const workload of this.workloads.values()) { + this.startWorkload(workload); + } + + if (this.cronWorkloads.size > 0) { + this.cronQueue = new Queue(CRON_QUEUE_NAME, { connection: this.connection }); + this.cronWorker = new Worker( + CRON_QUEUE_NAME, + (job) => this.runCron(job.name), + { connection: this.connection, concurrency: 1 }, + ); + this.cronWorker.on('failed', (job, error) => { + logger.error(`Cron "${job?.name}" run failed: ${error.message}`); + Sentry.captureException(error); + }); + this.cronWorker.on('error', (error) => { + logger.error('Cron worker error:', error); + }); + + for (const cron of this.cronWorkloads.values()) { + await this.cronQueue.upsertJobScheduler( + `cron:${cron.name}`, + scheduleToRepeat(cron.schedule), + { + name: cron.name, + opts: { + removeOnComplete: { count: CRON_KEEP_COMPLETED }, + removeOnFail: { count: CRON_KEEP_FAILED }, + }, + }, + ); + } + } + + logger.info( + `Started ${this.workloads.size} workload(s) [${[...this.workloads.keys()].join(', ') || '—'}] ` + + `and ${this.cronWorkloads.size} cron workload(s) [${[...this.cronWorkloads.keys()].join(', ') || '—'}]`, + ); + } + + async trigger(workloadName: string, data: T): Promise { + const workload = this.workloads.get(workloadName); + if (!workload) { + throw new Error(`Cannot trigger unknown workload "${workloadName}"`); + } + await this.producer.enqueue(workload.spec, data); + } + + async status(workloadName: string): Promise { + this.requireRegistered(workloadName); + const counts = await this.producer.queue(workloadName).getJobCounts( + 'waiting', + 'active', + 'delayed', + 'completed', + 'failed', + 'paused', + 'prioritized', + 'waiting-children', + ); + return { + waiting: counts.waiting ?? 0, + active: counts.active ?? 0, + delayed: counts.delayed ?? 0, + completed: counts.completed ?? 0, + failed: counts.failed ?? 0, + paused: counts.paused ?? 0, + prioritized: counts.prioritized ?? 0, + 'waiting-children': counts['waiting-children'] ?? 0, + }; + } + + async jobDetail(workloadName: string, jobId: string): Promise { + this.requireRegistered(workloadName); + const queue = this.producer.queue(workloadName); + const job = await queue.getJob(jobId); + if (!job) { + return null; + } + + const [state, jobLogs] = await Promise.all([ + job.getState(), + queue.getJobLogs(jobId), + ]); + + const enqueuedAt = job.timestamp; + const startedAt = job.processedOn ?? null; + const finishedAt = job.finishedOn ?? null; + + return { + id: job.id ?? jobId, + name: job.name, + state: normalizeJobState(state), + data: job.data, + attemptsMade: job.attemptsMade, + maxAttempts: job.opts.attempts ?? 1, + result: job.returnvalue ?? null, + failedReason: job.failedReason ?? null, + stacktrace: job.stacktrace ?? [], + logs: jobLogs.logs, + enqueuedAt, + startedAt, + finishedAt, + waitMs: startedAt !== null ? startedAt - enqueuedAt : null, + runMs: startedAt !== null && finishedAt !== null ? finishedAt - startedAt : null, + }; + } + + async stop(): Promise { + this.abortController.abort(); + + const workers = [...this.workers.values()]; + if (this.cronWorker) { + workers.push(this.cronWorker); + } + await Promise.all(workers.map((worker) => + Promise.race([ + worker.close(), + new Promise((resolve) => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)), + ]), + )); + + if (this.cronQueue) { + await this.cronQueue.close(); + } + await this.producer.close(); + + logger.info('Job manager stopped'); + } + + private startWorkload(workload: Workload): void { + const { spec, concurrency, rateLimit } = workload; + + this.producer.queue(spec.name); + + const worker = new Worker( + spec.name, + (job) => workload.process({ + data: job.data, + jobId: job.id ?? '', + attemptsMade: job.attemptsMade, + maxAttempts: job.opts.attempts ?? 1, + signal: this.abortController.signal, + log: async (message) => { await job.log(message); }, + updateProgress: (progress) => job.updateProgress(progress), + }), + { + connection: this.connection, + concurrency, + maxStalledCount: 1, + ...(rateLimit + ? { limiter: { max: rateLimit.max, duration: parseDuration(rateLimit.per) } } + : {}), + }, + ); + + worker.on('failed', (job, error) => { + void this.onWorkloadJobFailed(workload, job, error); + }); + worker.on('error', (error) => { + logger.error(`Worker "${spec.name}" error:`, error); + }); + + this.workers.set(spec.name, worker); + } + + private async runCron(cronName: string): Promise { + const cron = this.cronWorkloads.get(cronName); + if (!cron) { + logger.warn(`Cron fired for unknown workload "${cronName}"; skipping`); + return; + } + await cron.handler({ + trigger: (workload, data) => this.trigger(workload, data), + }); + } + + private async onWorkloadJobFailed( + workload: Workload, + job: Job | undefined, + error: Error, + ): Promise { + if (!job) { + return; + } + const maxAttempts = job.opts.attempts ?? 1; + const isTerminal = job.attemptsMade >= maxAttempts; + if (!isTerminal) { + logger.warn(`Workload "${workload.spec.name}" job ${job.id} failed attempt ${job.attemptsMade}/${maxAttempts}; will retry: ${error.message}`); + return; + } + logger.error(`Workload "${workload.spec.name}" job ${job.id} failed terminally after ${job.attemptsMade} attempt(s): ${error.message}`); + try { + await workload.onTerminalFailure?.(job.data, error); + } catch (hookError) { + Sentry.captureException(hookError); + logger.error(`onTerminalFailure for workload "${workload.spec.name}" threw:`, hookError); + } + } + + private requireRegistered(workloadName: string): void { + if (!this.workloads.has(workloadName)) { + throw new Error(`Workload "${workloadName}" is not registered`); + } + } +} diff --git a/packages/backend/src/jobProducer.ts b/packages/backend/src/jobProducer.ts new file mode 100644 index 000000000..f856d0beb --- /dev/null +++ b/packages/backend/src/jobProducer.ts @@ -0,0 +1,32 @@ +import { Queue } from "bullmq"; +import { Redis } from "ioredis"; +import { QueueSpec } from "./types.js"; + +export class JobProducer { + private readonly queues = new Map(); + + constructor(private readonly connection: Redis) {} + + queue(name: string): Queue { + let queue = this.queues.get(name); + if (!queue) { + queue = new Queue(name, { connection: this.connection }); + this.queues.set(name, queue); + } + return queue; + } + + async enqueue(spec: QueueSpec, data: T): Promise { + await this.queue(spec.name).add(spec.name, data, { + deduplication: { id: spec.dedupKey(data) }, + attempts: spec.jobOptions.attempts, + backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, + removeOnComplete: { count: spec.jobOptions.keep.completed }, + removeOnFail: { count: spec.jobOptions.keep.failed }, + }); + } + + async close(): Promise { + await Promise.all([...this.queues.values()].map((queue) => queue.close())); + } +} diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 8803b48b9..b14d5840b 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -24,4 +24,88 @@ export type RepoAuthCredentials = { * credentials for this repo. */ connectionConfig?: ConnectionConfig; -} \ No newline at end of file +} + +export interface ProcessContext { + data: TData; + jobId: string; + attemptsMade: number; + maxAttempts: number; + signal: AbortSignal; + log(message: string): Promise; + updateProgress(progress: number | object): Promise; +} + +export interface QueueSpec { + name: string; + dedupKey(data: TData): string; + jobOptions: { + attempts: number; + backoff: { type: 'fixed' | 'exponential'; delayMs: number }; + keep: { completed: number; failed: number }; + }; +} + +export interface Workload { + spec: QueueSpec; + concurrency: number; + rateLimit?: { max: number; per: string }; + process(ctx: ProcessContext): Promise; + onTerminalFailure?(data: TData, err: Error): Promise; +} + + +export type Schedule = { every: string } | { pattern: string }; + +export interface JobManager { + register(w: Workload): void; + registerCron(cron: CronWorkload): void; + + start(): Promise; + stop(): Promise; + + trigger(workload: string, data: T): Promise; + + status(workload: string): Promise; + jobDetail(workload: string, jobId: string): Promise; +} + +export interface CronWorkload { + name: string; + schedule: Schedule; + handler(ctx: CronContext): Promise; +} + +export interface CronContext { + trigger(workload: string, data: T): Promise; +} + + +export interface QueueCounts { + waiting: number; + active: number; + delayed: number; + completed: number; + failed: number; + paused: number; + prioritized?: number; + 'waiting-children'?: number; +} + +export interface JobDetail { + id: string; + name: string; + state: 'waiting' | 'active' | 'delayed' | 'completed' | 'failed' | 'paused' | 'unknown'; + data: TData; + attemptsMade: number; + maxAttempts: number; + result?: TResult | null; + failedReason?: string | null; + stacktrace?: string[]; + logs: string[]; + enqueuedAt: number; + startedAt: number | null; + finishedAt: number | null; + waitMs?: number | null; + runMs?: number | null; +} From bb84b8e44b3b4d2238333ddc8c1e51618814c410 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 14 Jul 2026 09:55:17 -0700 Subject: [PATCH 02/15] wip --- docs/snippets/schemas/v3/index.schema.mdx | 12 +- packages/backend/src/configManager.ts | 24 +- packages/backend/src/connectionManager.ts | 410 ------------------ packages/backend/src/connectionWorkload.ts | 141 ++++++ .../backend/src/ee/syncSearchContexts.test.ts | 62 ++- packages/backend/src/ee/syncSearchContexts.ts | 19 +- packages/backend/src/index.ts | 163 ++++--- packages/backend/src/jobManager.test.ts | 53 +-- packages/backend/src/jobManager.ts | 131 ++---- packages/backend/src/jobProducer.ts | 10 +- packages/backend/src/types.ts | 147 ++++--- packages/schemas/src/v3/index.schema.ts | 12 +- packages/schemas/src/v3/index.type.ts | 2 + schemas/v3/index.json | 6 +- 14 files changed, 425 insertions(+), 767 deletions(-) delete mode 100644 packages/backend/src/connectionManager.ts create mode 100644 packages/backend/src/connectionWorkload.ts diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index 864359251..0a25f6917 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -32,7 +32,8 @@ "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -42,7 +43,8 @@ "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", @@ -216,7 +218,8 @@ "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -226,7 +229,8 @@ "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", diff --git a/packages/backend/src/configManager.ts b/packages/backend/src/configManager.ts index 4d4f61ff6..74f050fb4 100644 --- a/packages/backend/src/configManager.ts +++ b/packages/backend/src/configManager.ts @@ -1,12 +1,13 @@ -import { Prisma, PrismaClient } from "@sourcebot/db"; +import { Prisma } from "@sourcebot/db"; import { createLogger, env } from "@sourcebot/shared"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { loadConfig } from "@sourcebot/shared"; import chokidar, { FSWatcher } from 'chokidar'; -import { ConnectionManager } from "./connectionManager.js"; import { SINGLE_TENANT_ORG_ID } from "./constants.js"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import isEqual from 'fast-deep-equal'; +import { JobManager } from "./types.js"; +import { prisma } from "./prisma.js"; const logger = createLogger('config-manager'); @@ -14,8 +15,7 @@ export class ConfigManager { private watcher: FSWatcher; constructor( - private db: PrismaClient, - private connectionManager: ConnectionManager, + private jobManager: JobManager, configPath: string, ) { this.watcher = chokidar.watch(configPath, { @@ -46,14 +46,13 @@ export class ConfigManager { await syncSearchContexts({ contexts: config.contexts, orgId: SINGLE_TENANT_ORG_ID, - db: this.db, }); } private syncConnections = async (connections?: { [key: string]: ConnectionConfig }) => { if (connections) { for (const [key, newConnectionConfig] of Object.entries(connections)) { - const existingConnection = await this.db.connection.findUnique({ + const existingConnection = await prisma.connection.findUnique({ where: { name_orgId: { name: key, @@ -73,7 +72,7 @@ export class ConfigManager { // Either update the existing connection or create a new one. const connection = existingConnection ? - await this.db.connection.update({ + await prisma.connection.update({ where: { id: existingConnection.id, }, @@ -84,7 +83,7 @@ export class ConfigManager { enforcePermissionsForPublicRepos, } }) : - await this.db.connection.create({ + await prisma.connection.create({ data: { name: key, config: newConnectionConfig as unknown as Prisma.InputJsonValue, @@ -102,13 +101,16 @@ export class ConfigManager { if (connectionNeedsSyncing) { logger.debug(`Change detected for connection '${key}' (id: ${connection.id}). Creating sync job.`); - await this.connectionManager.createJobs([connection]); + await this.jobManager.trigger('connection', { + connectionId: connection.id, + orgId: SINGLE_TENANT_ORG_ID, + }) } } } // Delete any connections that are no longer in the config. - const deletedConnections = await this.db.connection.findMany({ + const deletedConnections = await prisma.connection.findMany({ where: { isDeclarative: true, name: { @@ -120,7 +122,7 @@ export class ConfigManager { for (const connection of deletedConnections) { logger.debug(`Deleting connection with name '${connection.name}'. Connection ID: ${connection.id}`); - await this.db.connection.delete({ + await prisma.connection.delete({ where: { id: connection.id, } diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts deleted file mode 100644 index c9db057e2..000000000 --- a/packages/backend/src/connectionManager.ts +++ /dev/null @@ -1,410 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { Connection, ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db"; -import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; -import { createLogger, env, loadConfig } from "@sourcebot/shared"; -import { Job, Queue, Worker } from "bullmq"; -import { Redis } from 'ioredis'; -import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; -import { syncSearchContexts } from "./ee/syncSearchContexts.js"; -import { captureEvent } from "./posthog.js"; -import { PromClient } from "./promClient.js"; -import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; -import { Settings } from "./types.js"; -import { setIntervalAsync } from "./utils.js"; - -const LOG_TAG = 'connection-manager'; -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); -const QUEUE_NAME = 'connection-sync-queue'; - -const CONNECTION_SYNC_TIMEOUT_MS = 1000 * 60 * 60 * 2; // 2 hours - -type JobPayload = { - jobId: string, - connectionId: number, - connectionName: string, - orgId: number, -}; - -type JobResult = { - repoCount: number, -} - -export class ConnectionManager { - private worker: Worker; - private queue: Queue; - private abortController: AbortController; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - private promClient: PromClient, - ) { - this.abortController = new AbortController(); - - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - defaultJobOptions: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - attempts: 2, - }, - }); - - this.worker = new Worker( - QUEUE_NAME, - this.runJob.bind(this), - { - connection: redis, - concurrency: this.settings.maxConnectionSyncJobConcurrency, - maxStalledCount: 1, - } - ); - - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobMaybeFailed.bind(this)); - this.worker.on('stalled', (jobId) => { - // Just log - BullMQ will automatically retry the job (up to maxStalledCount times). - // If all retries fail, onJobMaybeFailed will handle marking it as failed. - logger.warn(`Job ${jobId} stalled - BullMQ will retry`); - }); - this.worker.on('error', (error) => { - logger.error(`Connection syncer worker error:`, error); - }); - } - - public startScheduler() { - logger.debug('Starting scheduler'); - this.interval = setIntervalAsync(async () => { - const thresholdDate = new Date(Date.now() - this.settings.resyncConnectionIntervalMs); - const timeoutDate = new Date(Date.now() - CONNECTION_SYNC_TIMEOUT_MS); - - const connections = await this.db.connection.findMany({ - where: { - AND: [ - { - OR: [ - { syncedAt: null }, - { syncedAt: { lt: thresholdDate } }, - ] - }, - { - NOT: { - syncJobs: { - some: { - OR: [ - // Don't schedule if there are active jobs that were created within the threshold date. - // This handles the case where a job is stuck in a pending state and will never be scheduled. - { - AND: [ - { status: { in: [ConnectionSyncJobStatus.PENDING, ConnectionSyncJobStatus.IN_PROGRESS] } }, - { createdAt: { gt: timeoutDate } }, - ] - }, - // Don't schedule if there are recent failed jobs (within the threshold date). - { - AND: [ - { status: ConnectionSyncJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - } - } - } - ] - } - }); - - if (connections.length > 0) { - await this.createJobs(connections); - } - }, this.settings.resyncConnectionPollingIntervalMs); - } - - - public async createJobs(connections: Connection[]) { - const jobs = await this.db.connectionSyncJob.createManyAndReturn({ - data: connections.map(connection => ({ - connectionId: connection.id, - })), - include: { - connection: true, - } - }); - - for (const job of jobs) { - logger.debug(`Scheduling job ${job.id} for connection ${job.connection.name} (id: ${job.connectionId})`); - await this.queue.add( - 'connection-sync-job', - { - jobId: job.id, - connectionId: job.connectionId, - connectionName: job.connection.name, - orgId: job.connection.orgId, - }, - { jobId: job.id } - ); - - this.promClient.pendingConnectionSyncJobs.inc({ connection: job.connection.name }); - } - - return jobs.map(job => job.id); - } - - private async runJob(job: Job): Promise { - const { jobId, connectionName } = job.data; - const logger = createJobLogger(jobId); - logger.debug(`Running connection sync job ${jobId} for connection ${connectionName} (id: ${job.data.connectionId})`); - - const currentStatus = await this.db.connectionSyncJob.findUniqueOrThrow({ - where: { - id: jobId, - }, - select: { - status: true, - } - }); - - // Fail safe: if the job is not PENDING (first run) or IN_PROGRESS (retry), it indicates the job - // is in an invalid state and should be skipped. - if (currentStatus.status !== ConnectionSyncJobStatus.PENDING && currentStatus.status !== ConnectionSyncJobStatus.IN_PROGRESS) { - throw new Error(`Job ${jobId} is not in a valid state. Expected: ${ConnectionSyncJobStatus.PENDING} or ${ConnectionSyncJobStatus.IN_PROGRESS}. Actual: ${currentStatus.status}. Skipping.`); - } - - this.promClient.pendingConnectionSyncJobs.dec({ connection: connectionName }); - this.promClient.activeConnectionSyncJobs.inc({ connection: connectionName }); - - const { connection: { config: rawConnectionConfig, orgId } } = await this.db.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: ConnectionSyncJobStatus.IN_PROGRESS, - }, - select: { - connection: { - select: { - config: true, - orgId: true, - } - } - }, - }); - - const config = rawConnectionConfig as unknown as ConnectionConfig; - - const result = await (async () => { - switch (config.type) { - case 'github': { - return await compileGithubConfig(config, job.data.connectionId, this.abortController.signal); - } - case 'gitlab': { - return await compileGitlabConfig(config, job.data.connectionId); - } - case 'gitea': { - return await compileGiteaConfig(config, job.data.connectionId); - } - case 'gerrit': { - return await compileGerritConfig(config, job.data.connectionId); - } - case 'bitbucket': { - return await compileBitbucketConfig(config, job.data.connectionId); - } - case 'azuredevops': { - return await compileAzureDevOpsConfig(config, job.data.connectionId); - } - case 'git': { - return await compileGenericGitHostConfig(config, job.data.connectionId); - } - } - })(); - - let { repoData, warnings } = result; - - await this.db.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - warningMessages: warnings, - }, - }); - - - // Filter out any duplicates by external_id and external_codeHostUrl. - repoData = repoData.filter((repo, index, self) => { - return index === self.findIndex(r => - r.external_id === repo.external_id && - r.external_codeHostUrl === repo.external_codeHostUrl - ); - }) - - // @note: to handle orphaned Repos we delete all RepoToConnection records for this connection, - // and then recreate them when we upsert the repos. For example, if a repo is no-longer - // captured by the connection's config (e.g., it was deleted, marked archived, etc.), it won't - // appear in the repoData array above, and so the RepoToConnection record won't be re-created. - // Repos that have no RepoToConnection records are considered orphaned and can be deleted. - await this.db.$transaction(async (tx) => { - const deleteStart = performance.now(); - await tx.connection.update({ - where: { - id: job.data.connectionId, - }, - data: { - repos: { - deleteMany: {} - } - } - }); - const deleteDuration = performance.now() - deleteStart; - logger.debug(`Deleted all RepoToConnection records for connection ${connectionName} (id: ${job.data.connectionId}) in ${deleteDuration}ms`); - - const totalUpsertStart = performance.now(); - for (const repo of repoData) { - const upsertStart = performance.now(); - await tx.repo.upsert({ - where: { - external_id_external_codeHostUrl_orgId: { - external_id: repo.external_id, - external_codeHostUrl: repo.external_codeHostUrl, - orgId: orgId, - } - }, - update: repo, - create: repo, - }) - const upsertDuration = performance.now() - upsertStart; - logger.debug(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`); - } - const totalUpsertDuration = performance.now() - totalUpsertStart; - logger.debug(`Upserted ${repoData.length} repos for connection ${connectionName} (id: ${job.data.connectionId}) in ${totalUpsertDuration}ms`); - }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); - - return { - repoCount: repoData.length, - }; - } - - - private async onJobCompleted(job: Job, result: JobResult) { - try { - const logger = createJobLogger(job.id!); - const { connectionId, connectionName, orgId } = job.data; - - const { connection } = await this.db.connectionSyncJob.update({ - where: { - id: job.id!, - }, - data: { - status: ConnectionSyncJobStatus.COMPLETED, - completedAt: new Date(), - connection: { - update: { - syncedAt: new Date(), - } - } - }, - select: { - connection: true, - } - }); - - // After a connection has synced, we need to re-sync the org's search contexts as - // there may be new repos that match the search context's include/exclude patterns. - if (env.CONFIG_PATH) { - try { - const config = await loadConfig(env.CONFIG_PATH); - - await syncSearchContexts({ - db: this.db, - orgId, - contexts: config.contexts, - }); - } catch (err) { - logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`); - Sentry.captureException(err); - } - } - - logger.debug(`Connection sync job ${job.id} for connection ${job.data.connectionName} (id: ${job.data.connectionId}) completed`); - - this.promClient.activeConnectionSyncJobs.dec({ connection: connectionName }); - this.promClient.connectionSyncJobSuccessTotal.inc({ connection: connectionName }); - - const config = connection.config as unknown as ConnectionConfig; - captureEvent('backend_connection_sync_job_completed', { - connectionId: connectionId, - repoCount: result.repoCount, - type: config.type, - }); - } catch (error) { - Sentry.captureException(error); - logger.error(`Exception thrown while executing lifecycle function \`onJobCompleted\`.`, error); - } - } - - private async onJobMaybeFailed(job: Job | undefined, error: Error) { - try { - if (!job) { - logger.error(`Job failed but job object is undefined. Error: ${error.message}`); - return; - } - const jobLogger = createJobLogger(job.id!); - - // @note: we need to check the job state to determine if the job failed, - // or if it is being retried. - const jobState = await job.getState(); - if (jobState !== 'failed') { - jobLogger.warn(`Job ${job.id} for connection ${job.data.connectionName} (id: ${job.data.connectionId}) failed. Retrying...`); - return; - } - - const { connection } = await this.db.connectionSyncJob.update({ - where: { id: job.id }, - data: { - status: ConnectionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: job.failedReason, - }, - select: { - connection: true, - } - }); - - this.promClient.activeConnectionSyncJobs.dec({ connection: connection.name }); - this.promClient.connectionSyncJobFailTotal.inc({ connection: connection.name }); - - jobLogger.error(`Failed job ${job.id} for connection ${connection.name} (id: ${connection.id}). Reason: ${job.failedReason}`); - - const config = connection.config as unknown as ConnectionConfig; - captureEvent('backend_connection_sync_job_failed', { - connectionId: job.data.connectionId, - type: config.type, - }); - } catch (err) { - Sentry.captureException(err); - logger.error(`Exception thrown while executing lifecycle function \`onJobMaybeFailed\`.`, err); - } - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - - // Signal all active jobs to abort - this.abortController.abort(); - - // Wait for worker to finish with timeout - await Promise.race([ - this.worker.close(), - new Promise(resolve => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)) - ]); - - await this.queue.close(); - } -} diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts new file mode 100644 index 000000000..a3b965fb1 --- /dev/null +++ b/packages/backend/src/connectionWorkload.ts @@ -0,0 +1,141 @@ +import { QueueSpec, Workload } from "./types.js"; +import { prisma } from "./prisma.js"; +import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; +import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; +import { createLogger, env, loadConfig } from "@sourcebot/shared"; +import { syncSearchContexts } from "./ee/syncSearchContexts.js"; +import * as Sentry from "@sentry/node"; + + +const connectionQueueSpec: QueueSpec<'connection'> = { + name: 'connection', + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 } + }, + dedupKey: (data) => `connection:${data.connectionId}` +} + +// @todo +const logger = createLogger('connection-workflow'); + +export const connectionWorkload: Workload<'connection'> = { + spec: connectionQueueSpec, + concurrency: 2, + process: async ({ + data: { + connectionId, + orgId + }, + signal, + }) => { + const connection = await prisma.connection.findUniqueOrThrow({ + where: { + id: connectionId + } + }); + + const config = connection.config as unknown as ConnectionConfig; + + const result = await (async () => { + switch (config.type) { + case 'github': { + return await compileGithubConfig(config, connectionId, signal); + } + case 'gitlab': { + return await compileGitlabConfig(config, connectionId); + } + case 'gitea': { + return await compileGiteaConfig(config, connectionId); + } + case 'gerrit': { + return await compileGerritConfig(config, connectionId); + } + case 'bitbucket': { + return await compileBitbucketConfig(config, connectionId); + } + case 'azuredevops': { + return await compileAzureDevOpsConfig(config, connectionId); + } + case 'git': { + return await compileGenericGitHostConfig(config, connectionId); + } + } + })(); + + let { repoData } = result; + + // Filter out any duplicates by external_id and external_codeHostUrl. + repoData = repoData.filter((repo, index, self) => { + return index === self.findIndex(r => + r.external_id === repo.external_id && + r.external_codeHostUrl === repo.external_codeHostUrl + ); + }) + + // @note: to handle orphaned Repos we delete all RepoToConnection records for this connection, + // and then recreate them when we upsert the repos. For example, if a repo is no-longer + // captured by the connection's config (e.g., it was deleted, marked archived, etc.), it won't + // appear in the repoData array above, and so the RepoToConnection record won't be re-created. + // Repos that have no RepoToConnection records are considered orphaned and can be deleted. + await prisma.$transaction(async (tx) => { + const deleteStart = performance.now(); + await tx.connection.update({ + where: { + id: connectionId, + }, + data: { + repos: { + deleteMany: {} + } + } + }); + const deleteDuration = performance.now() - deleteStart; + logger.debug(`Deleted all RepoToConnection records for connection ${connection.name} (id: ${connectionId}) in ${deleteDuration}ms`); + + const totalUpsertStart = performance.now(); + for (const repo of repoData) { + const upsertStart = performance.now(); + await tx.repo.upsert({ + where: { + external_id_external_codeHostUrl_orgId: { + external_id: repo.external_id, + external_codeHostUrl: repo.external_codeHostUrl, + orgId: orgId, + } + }, + update: repo, + create: repo, + }) + const upsertDuration = performance.now() - upsertStart; + logger.debug(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`); + } + const totalUpsertDuration = performance.now() - totalUpsertStart; + logger.debug(`Upserted ${repoData.length} repos for connection ${connection.name} (id: ${connectionId}) in ${totalUpsertDuration}ms`); + }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); + + await prisma.connection.update({ + where: { + id: connectionId, + }, + data: { + syncedAt: new Date(), + } + }); + + // After a connection has synced, we need to re-sync the org's search contexts as + // there may be new repos that match the search context's include/exclude patterns. + try { + const config = await loadConfig(env.CONFIG_PATH); + + await syncSearchContexts({ + orgId, + contexts: config.contexts, + }); + } catch (err) { + logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`); + Sentry.captureException(err); + } + } +} diff --git a/packages/backend/src/ee/syncSearchContexts.test.ts b/packages/backend/src/ee/syncSearchContexts.test.ts index 9aa1decfd..89ffec5d6 100644 --- a/packages/backend/src/ee/syncSearchContexts.test.ts +++ b/packages/backend/src/ee/syncSearchContexts.test.ts @@ -21,6 +21,17 @@ vi.mock('../entitlements.js', () => ({ getPlan: vi.fn(() => Promise.resolve('enterprise')), })); +// `syncSearchContexts` imports the prisma singleton, which builds a real client (and needs +// DATABASE_URL) at import time. Stand in for it with an object that `buildDb` re-populates +// with fresh mocks for each test, so tests stay isolated from one another. +const { prismaMock } = vi.hoisted(() => ({ + prismaMock: {} as Record, +})); + +vi.mock('../prisma.js', () => ({ + prisma: prismaMock, +})); + import { syncSearchContexts } from './syncSearchContexts.js'; // Helper to build a repo record with GitLab topics stored in metadata. @@ -64,20 +75,25 @@ const buildDb = (overrides: Partial<{ connectionFindMany: unknown[]; searchContextFindUnique: unknown; searchContextFindMany: unknown[]; -}> = {}): PrismaClient => ({ - repo: { - findMany: vi.fn().mockResolvedValue(overrides.repoFindMany ?? []), - }, - connection: { - findMany: vi.fn().mockResolvedValue(overrides.connectionFindMany ?? []), - }, - searchContext: { - findUnique: vi.fn().mockResolvedValue(overrides.searchContextFindUnique ?? null), - findMany: vi.fn().mockResolvedValue(overrides.searchContextFindMany ?? []), - upsert: vi.fn().mockResolvedValue({}), - delete: vi.fn().mockResolvedValue({}), - }, -} as unknown as PrismaClient); +}> = {}): PrismaClient => { + // Overwrite every delegate so no mock survives from a previous test. + Object.assign(prismaMock, { + repo: { + findMany: vi.fn().mockResolvedValue(overrides.repoFindMany ?? []), + }, + connection: { + findMany: vi.fn().mockResolvedValue(overrides.connectionFindMany ?? []), + }, + searchContext: { + findUnique: vi.fn().mockResolvedValue(overrides.searchContextFindUnique ?? null), + findMany: vi.fn().mockResolvedValue(overrides.searchContextFindMany ?? []), + upsert: vi.fn().mockResolvedValue({}), + delete: vi.fn().mockResolvedValue({}), + }, + }); + + return prismaMock as unknown as PrismaClient; +}; describe('syncSearchContexts - includeTopics', () => { test('includes repos whose topics match an includeTopics entry', async () => { @@ -90,7 +106,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -109,7 +124,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -127,7 +141,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend', 'core'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -146,7 +159,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['core-*'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -165,7 +177,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -188,7 +199,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -209,7 +219,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -230,7 +239,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -251,7 +259,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -274,7 +281,6 @@ describe('syncSearchContexts - includeTopics + excludeTopics combined', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -298,7 +304,6 @@ describe('syncSearchContexts - includeTopics combined with include globs', () => }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -319,7 +324,6 @@ describe('syncSearchContexts - includeTopics combined with include globs', () => }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -340,7 +344,6 @@ describe('syncSearchContexts - GitHub includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -359,7 +362,6 @@ describe('syncSearchContexts - GitHub includeTopics', () => { myContext: { includeTopics: ['core-*'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -377,7 +379,6 @@ describe('syncSearchContexts - GitHub includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -400,7 +401,6 @@ describe('syncSearchContexts - GitHub excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -422,7 +422,6 @@ describe('syncSearchContexts - mixed GitHub and GitLab repos', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -446,7 +445,6 @@ describe('syncSearchContexts - mixed GitHub and GitLab repos', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; diff --git a/packages/backend/src/ee/syncSearchContexts.ts b/packages/backend/src/ee/syncSearchContexts.ts index 6a02c9062..a3592a30f 100644 --- a/packages/backend/src/ee/syncSearchContexts.ts +++ b/packages/backend/src/ee/syncSearchContexts.ts @@ -1,20 +1,19 @@ import micromatch from "micromatch"; import { createLogger } from "@sourcebot/shared"; -import { PrismaClient } from "@sourcebot/db"; import { repoMetadataSchema, SOURCEBOT_SUPPORT_EMAIL } from "@sourcebot/shared"; import { hasEntitlement } from "../entitlements.js"; import { SearchContext } from "@sourcebot/schemas/v3/index.type"; +import { prisma } from "../prisma.js"; const logger = createLogger('sync-search-contexts'); interface SyncSearchContextsParams { contexts?: { [key: string]: SearchContext } | undefined; orgId: number; - db: PrismaClient; } export const syncSearchContexts = async (params: SyncSearchContextsParams) => { - const { contexts, orgId, db } = params; + const { contexts, orgId } = params; if (!await hasEntitlement("search-contexts")) { if (contexts) { @@ -25,7 +24,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { if (contexts) { for (const [key, newContextConfig] of Object.entries(contexts)) { - const allRepos = await db.repo.findMany({ + const allRepos = await prisma.repo.findMany({ where: { orgId, }, @@ -44,7 +43,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } if(newContextConfig.includeConnections) { - const connections = await db.connection.findMany({ + const connections = await prisma.connection.findMany({ where: { orgId, name: { @@ -101,7 +100,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } if (newContextConfig.excludeConnections) { - const connections = await db.connection.findMany({ + const connections = await prisma.connection.findMany({ where: { orgId, name: { @@ -145,7 +144,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { }); } - const currentReposInContext = (await db.searchContext.findUnique({ + const currentReposInContext = (await prisma.searchContext.findUnique({ where: { name_orgId: { name: key, @@ -157,7 +156,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } }))?.repos ?? []; - await db.searchContext.upsert({ + await prisma.searchContext.upsert({ where: { name_orgId: { name: key, @@ -195,7 +194,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } } - const deletedContexts = await db.searchContext.findMany({ + const deletedContexts = await prisma.searchContext.findMany({ where: { name: { notIn: Object.keys(contexts ?? {}), @@ -206,7 +205,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { for (const context of deletedContexts) { logger.debug(`Deleting search context with name '${context.name}'. ID: ${context.id}`); - await db.searchContext.delete({ + await prisma.searchContext.delete({ where: { id: context.id, } diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 379ca388c..fee139a87 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -2,26 +2,20 @@ import "./instrument.js"; import * as Sentry from "@sentry/node"; import { createLogger, env, getConfigSettings } from "@sourcebot/shared"; -import { hasEntitlement } from "./entitlements.js"; -import { prisma } from "./prisma.js"; import 'express-async-errors'; import { existsSync } from 'fs'; import { mkdir } from 'fs/promises'; -import { Api } from "./api.js"; -import { AttachmentPruner } from "./attachmentPruner.js"; import { ConfigManager } from "./configManager.js"; -import { ConnectionManager } from './connectionManager.js'; -import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, SHUTDOWN_SIGNALS } from './constants.js'; -import { AccountPermissionSyncer } from "./ee/accountPermissionSyncer.js"; -import { AuditLogPruner } from "./ee/auditLogPruner.js"; +import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, SHUTDOWN_SIGNALS, SINGLE_TENANT_ORG_ID } from './constants.js'; import { GithubAppManager } from "./ee/githubAppManager.js"; -import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js'; +import { hasEntitlement } from "./entitlements.js"; +import { BullMQJobManager } from "./jobManager.js"; import { shutdownPosthog } from "./posthog.js"; +import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; -import { RepoIndexManager } from "./repoIndexManager.js"; import { redis } from "./redis.js"; -import { BullMQJobManager, reconcile } from "./jobManager.js"; import { QueueSpec, Workload } from "./types.js"; +import { connectionWorkload } from "./connectionWorkload.js"; const logger = createLogger('backend-entrypoint'); @@ -52,36 +46,35 @@ if (await hasEntitlement('github-app')) { await GithubAppManager.getInstance().init(prisma); } -const connectionManager = new ConnectionManager(prisma, settings, redis, promClient); -const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); -const accountPermissionSyncer = new AccountPermissionSyncer(prisma, settings, redis); -const repoIndexManager = new RepoIndexManager(prisma, settings, redis, promClient); -const configManager = new ConfigManager(prisma, connectionManager, env.CONFIG_PATH); -const auditLogPruner = new AuditLogPruner(prisma); -const attachmentPruner = new AttachmentPruner(prisma); - -connectionManager.startScheduler(); -await repoIndexManager.startScheduler(); -auditLogPruner.startScheduler(); -attachmentPruner.startScheduler(); - -if (env.PERMISSION_SYNC_ENABLED === 'true' && !await hasEntitlement('permission-syncing')) { - logger.warn('Permission syncing is not supported in current plan. Please contact team@sourcebot.dev for assistance.'); -} -else if (env.PERMISSION_SYNC_ENABLED === 'true' && await hasEntitlement('permission-syncing')) { - if (env.PERMISSION_SYNC_REPO_DRIVEN_ENABLED === 'true') { - await repoPermissionSyncer.startScheduler(); - } - await accountPermissionSyncer.startScheduler(); -} - -const api = new Api( - promClient, - prisma, - connectionManager, - repoIndexManager, - accountPermissionSyncer, -); +// const connectionManager = new ConnectionManager(prisma, settings, redis, promClient); +// const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); +// const accountPermissionSyncer = new AccountPermissionSyncer(prisma, settings, redis); +// const repoIndexManager = new RepoIndexManager(prisma, settings, redis, promClient); +// const auditLogPruner = new AuditLogPruner(prisma); +// const attachmentPruner = new AttachmentPruner(prisma); + +// connectionManager.startScheduler(); +// await repoIndexManager.startScheduler(); +// auditLogPruner.startScheduler(); +// attachmentPruner.startScheduler(); + +// if (env.PERMISSION_SYNC_ENABLED === 'true' && !await hasEntitlement('permission-syncing')) { +// logger.warn('Permission syncing is not supported in current plan. Please contact team@sourcebot.dev for assistance.'); +// } +// else if (env.PERMISSION_SYNC_ENABLED === 'true' && await hasEntitlement('permission-syncing')) { +// if (env.PERMISSION_SYNC_REPO_DRIVEN_ENABLED === 'true') { +// await repoPermissionSyncer.startScheduler(); +// } +// await accountPermissionSyncer.startScheduler(); +// } + +// const api = new Api( +// promClient, +// prisma, +// connectionManager, +// repoIndexManager, +// accountPermissionSyncer, +// ); logger.info('Worker started.'); @@ -90,46 +83,50 @@ logger.info('Worker started.'); // (repo-index, connection-sync, permission syncers) are ported onto it in subsequent phases. const jobManager = new BullMQJobManager(redis); - -const demoSpec: QueueSpec<{ id: string }> = { - name: 'demo', - dedupKey: ({ id }) => `demo:${id}`, +const cronQueueSpec: QueueSpec<'cron'> = { + name: 'cron', jobOptions: { attempts: 2, - backoff: { type: 'fixed', delayMs: 1000 }, - keep: { completed: 50, failed: 50 }, - }, -}; - -const demoWorkload: Workload<{ id: string }, { id: string; ranAt: number }> = { - spec: demoSpec, - concurrency: 2, - process: async ({ data: { id }, jobId, attemptsMade }) => { - logger.info(`demo: processing "${id}" (job ${jobId}, attempt ${attemptsMade})`); - await new Promise((resolve) => setTimeout(resolve, 1500)); - if (id === 'gamma') { - throw new Error(`demo: simulated failure processing "${id}"`); - } - return { id, ranAt: Date.now() }; - }, - onTerminalFailure: async ({ id }, err) => { - logger.warn(`demo: "${id}" failed terminally: ${err.message}`); - }, -}; -jobManager.register(demoWorkload); - -// The reconcile sweep for the demo queue, expressed as a cron workload: every 15s it triggers -// a fixed set into `demo`. Dedup keeps an in-flight item from re-queuing, but a finished one -// re-enqueues on the next tick, so the loop visibly cycles. -jobManager.registerCron(reconcile({ - name: 'demo-sweep', - schedule: { every: '15s' }, - target: 'demo', - scan: async () => ['alpha', 'beta', 'gamma'].map((id) => ({ id })), -})); + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 } + } +} + +const cronWorkload: Workload<'cron'> = { + concurrency: 1, + schedule: { every: '5s' }, + spec: cronQueueSpec, + process: async ({ jobId, trigger }) => { + console.log(`cron ${jobId}`); + + const thresholdDate = new Date(Date.now() - settings.resyncConnectionIntervalMs); + const connections = await prisma.connection.findMany({ + where: { + OR: [ + { syncedAt: null }, + { syncedAt: { lt: thresholdDate }} + ] + } + }); + + await Promise.all(connections.map(async (connection) => { + console.log(`Scheduling work for ${connection.id}`); + await trigger('connection', { + connectionId: connection.id, + orgId: SINGLE_TENANT_ORG_ID, + }) + })) + } +} + +jobManager.register(cronWorkload); +jobManager.register(connectionWorkload); await jobManager.start(); +const configManager = new ConfigManager(jobManager, env.CONFIG_PATH); + + const listenToShutdownSignals = () => { const signals = SHUTDOWN_SIGNALS; @@ -144,18 +141,18 @@ const listenToShutdownSignals = () => { logger.info(`Received ${signal}, cleaning up...`); - await repoIndexManager.dispose() - await connectionManager.dispose() - await repoPermissionSyncer.dispose() - await accountPermissionSyncer.dispose() - await auditLogPruner.dispose() - await attachmentPruner.dispose() + // await repoIndexManager.dispose() + // await connectionManager.dispose() + // await repoPermissionSyncer.dispose() + // await accountPermissionSyncer.dispose() + // await auditLogPruner.dispose() + // await attachmentPruner.dispose() await configManager.dispose() await jobManager.stop(); await prisma.$disconnect(); await redis.quit(); - await api.dispose(); + // await api.dispose(); await shutdownPosthog(); logger.info('All workers shut down gracefully'); diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts index c65b1c06d..5f01538a5 100644 --- a/packages/backend/src/jobManager.test.ts +++ b/packages/backend/src/jobManager.test.ts @@ -16,7 +16,7 @@ vi.mock('./constants.js', () => ({ WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, })); -import { normalizeJobState, parseDuration, reconcile } from './jobManager.js'; +import { normalizeJobState, parseDuration } from './jobManager.js'; describe('parseDuration', () => { test.each([ @@ -61,54 +61,3 @@ describe('normalizeJobState', () => { }); }); -describe('reconcile', () => { - test('produces a cron workload carrying the given name and schedule', () => { - const cron = reconcile({ - name: 'x-sweep', - schedule: { every: '5m' }, - target: 'x', - scan: async () => [], - }); - expect(cron.name).toBe('x-sweep'); - expect(cron.schedule).toEqual({ every: '5m' }); - }); - - test('handler triggers each scanned item into the target, in order', async () => { - const triggered: Array<{ workload: string; data: unknown }> = []; - const cron = reconcile({ - name: 'x-sweep', - schedule: { pattern: '*/5 * * * *' }, - target: 'x', - scan: async () => [{ id: 'a' }, { id: 'b' }], - }); - - await cron.handler({ - trigger: async (workload, data) => { - triggered.push({ workload, data }); - }, - }); - - expect(triggered).toEqual([ - { workload: 'x', data: { id: 'a' } }, - { workload: 'x', data: { id: 'b' } }, - ]); - }); - - test('handler triggers nothing when scan returns empty', async () => { - let calls = 0; - const cron = reconcile({ - name: 'empty-sweep', - schedule: { every: '1m' }, - target: 'x', - scan: async () => [], - }); - - await cron.handler({ - trigger: async () => { - calls += 1; - }, - }); - - expect(calls).toBe(0); - }); -}); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index cd7689ef5..35dfa0af0 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -1,18 +1,14 @@ import * as Sentry from "@sentry/node"; import { createLogger } from "@sourcebot/shared"; -import { Job, Queue, Worker } from "bullmq"; +import { Job, Worker } from "bullmq"; import { Redis } from "ioredis"; import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; import { JobProducer } from "./jobProducer.js"; -import { CronWorkload, JobDetail, JobManager, QueueCounts, Schedule, Workload } from "./types.js"; +import { DataOf, JobDetail, JobManager, QueueCounts, QueueName, Schedule, Workload } from "./types.js"; const LOG_TAG = 'job-manager'; const logger = createLogger(LOG_TAG); -const CRON_QUEUE_NAME = 'cron'; -const CRON_KEEP_COMPLETED = 50; -const CRON_KEEP_FAILED = 200; - const DURATION_UNITS_MS: Record = { ms: 1, s: 1000, @@ -49,99 +45,43 @@ export const normalizeJobState = (state: string): JobDetail['state'] => { const scheduleToRepeat = (schedule: Schedule) => 'pattern' in schedule ? { pattern: schedule.pattern } : { every: parseDuration(schedule.every) }; -export function reconcile(opts: { - name: string; - schedule: Schedule; - target: string; - scan: () => Promise; -}): CronWorkload { - return { - name: opts.name, - schedule: opts.schedule, - handler: async ({ trigger }) => { - const items = await opts.scan(); - for (const item of items) { - await trigger(opts.target, item); - } - }, - }; -} - export class BullMQJobManager implements JobManager { - private readonly workloads = new Map>(); - private readonly cronWorkloads = new Map(); + private readonly workloads = new Map>(); private readonly workers = new Map(); private readonly producer: JobProducer; - private cronQueue?: Queue; - private cronWorker?: Worker; private readonly abortController = new AbortController(); constructor(private readonly connection: Redis) { this.producer = new JobProducer(connection); } - register(workload: Workload): void { + register(workload: Workload): void { const name = workload.spec.name; if (this.workloads.has(name)) { throw new Error(`Workload "${name}" is already registered`); } - this.workloads.set(name, workload as unknown as Workload); - } - - registerCron(cron: CronWorkload): void { - if (this.cronWorkloads.has(cron.name)) { - throw new Error(`Cron workload "${cron.name}" is already registered`); - } - this.cronWorkloads.set(cron.name, cron); + this.workloads.set(name, workload); } async start(): Promise { - if (this.workloads.size === 0 && this.cronWorkloads.size === 0) { + if (this.workloads.size === 0) { logger.debug('start() called with nothing registered; nothing to do'); return; } for (const workload of this.workloads.values()) { - this.startWorkload(workload); - } - - if (this.cronWorkloads.size > 0) { - this.cronQueue = new Queue(CRON_QUEUE_NAME, { connection: this.connection }); - this.cronWorker = new Worker( - CRON_QUEUE_NAME, - (job) => this.runCron(job.name), - { connection: this.connection, concurrency: 1 }, - ); - this.cronWorker.on('failed', (job, error) => { - logger.error(`Cron "${job?.name}" run failed: ${error.message}`); - Sentry.captureException(error); - }); - this.cronWorker.on('error', (error) => { - logger.error('Cron worker error:', error); - }); - - for (const cron of this.cronWorkloads.values()) { - await this.cronQueue.upsertJobScheduler( - `cron:${cron.name}`, - scheduleToRepeat(cron.schedule), - { - name: cron.name, - opts: { - removeOnComplete: { count: CRON_KEEP_COMPLETED }, - removeOnFail: { count: CRON_KEEP_FAILED }, - }, - }, - ); - } + await this.startWorkload(workload); } logger.info( - `Started ${this.workloads.size} workload(s) [${[...this.workloads.keys()].join(', ') || '—'}] ` + - `and ${this.cronWorkloads.size} cron workload(s) [${[...this.cronWorkloads.keys()].join(', ') || '—'}]`, + `Started ${this.workloads.size} workload(s) [${[...this.workloads.keys()].join(', ')}]`, ); } - async trigger(workloadName: string, data: T): Promise { + async trigger( + workloadName: TName, + data: DataOf + ): Promise { const workload = this.workloads.get(workloadName); if (!workload) { throw new Error(`Cannot trigger unknown workload "${workloadName}"`); @@ -212,29 +152,22 @@ export class BullMQJobManager implements JobManager { async stop(): Promise { this.abortController.abort(); - const workers = [...this.workers.values()]; - if (this.cronWorker) { - workers.push(this.cronWorker); - } - await Promise.all(workers.map((worker) => + await Promise.all([...this.workers.values()].map((worker) => Promise.race([ worker.close(), new Promise((resolve) => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)), ]), )); - if (this.cronQueue) { - await this.cronQueue.close(); - } await this.producer.close(); logger.info('Job manager stopped'); } - private startWorkload(workload: Workload): void { - const { spec, concurrency, rateLimit } = workload; + private async startWorkload(workload: Workload): Promise { + const { spec, concurrency, rateLimit, schedule } = workload; - this.producer.queue(spec.name); + const queue = this.producer.queue(spec.name); const worker = new Worker( spec.name, @@ -246,6 +179,7 @@ export class BullMQJobManager implements JobManager { signal: this.abortController.signal, log: async (message) => { await job.log(message); }, updateProgress: (progress) => job.updateProgress(progress), + trigger: (target, data) => this.trigger(target, data), }), { connection: this.connection, @@ -265,21 +199,30 @@ export class BullMQJobManager implements JobManager { }); this.workers.set(spec.name, worker); - } - private async runCron(cronName: string): Promise { - const cron = this.cronWorkloads.get(cronName); - if (!cron) { - logger.warn(`Cron fired for unknown workload "${cronName}"; skipping`); - return; + if (schedule) { + // @note: jobs produced by BullMQ's scheduler bypass the deduplication check that + // `Queue.add` goes through, so a dedup key would be silently ignored here. A + // scheduled workload gets its overlap protection from `concurrency` instead: the + // next tick's job is only created once the current one goes active, so at most one + // run is ever queued behind the one in flight. + await queue.upsertJobScheduler( + `schedule:${spec.name}`, + scheduleToRepeat(schedule), + { + name: spec.name, + opts: { + attempts: spec.jobOptions.attempts, + removeOnComplete: { count: spec.jobOptions.keep.completed }, + removeOnFail: { count: spec.jobOptions.keep.failed }, + }, + }, + ); } - await cron.handler({ - trigger: (workload, data) => this.trigger(workload, data), - }); } - private async onWorkloadJobFailed( - workload: Workload, + private async onWorkloadJobFailed( + workload: Workload, job: Job | undefined, error: Error, ): Promise { diff --git a/packages/backend/src/jobProducer.ts b/packages/backend/src/jobProducer.ts index f856d0beb..8cd28a2e8 100644 --- a/packages/backend/src/jobProducer.ts +++ b/packages/backend/src/jobProducer.ts @@ -1,6 +1,6 @@ import { Queue } from "bullmq"; import { Redis } from "ioredis"; -import { QueueSpec } from "./types.js"; +import { DataOf, QueueName, QueueSpec } from "./types.js"; export class JobProducer { private readonly queues = new Map(); @@ -16,9 +16,13 @@ export class JobProducer { return queue; } - async enqueue(spec: QueueSpec, data: T): Promise { + async enqueue( + spec: QueueSpec, + data: DataOf + ): Promise { + const dedupKey = spec.dedupKey?.(data); await this.queue(spec.name).add(spec.name, data, { - deduplication: { id: spec.dedupKey(data) }, + ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), attempts: spec.jobOptions.attempts, backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, removeOnComplete: { count: spec.jobOptions.keep.completed }, diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index b14d5840b..66295017e 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -26,86 +26,109 @@ export type RepoAuthCredentials = { connectionConfig?: ConnectionConfig; } -export interface ProcessContext { - data: TData; - jobId: string; - attemptsMade: number; - maxAttempts: number; - signal: AbortSignal; - log(message: string): Promise; - updateProgress(progress: number | object): Promise; +export interface QueueRegistry { + 'connection': { + connectionId: number, + orgId: number + }, + 'cron': {} } -export interface QueueSpec { - name: string; - dedupKey(data: TData): string; - jobOptions: { - attempts: number; - backoff: { type: 'fixed' | 'exponential'; delayMs: number }; - keep: { completed: number; failed: number }; - }; +export type QueueName = keyof QueueRegistry; +export type DataOf = QueueRegistry[TName]; + +export interface ProcessContext { + data: DataOf; + jobId: string; + attemptsMade: number; + maxAttempts: number; + signal: AbortSignal; + log(message: string): Promise; + updateProgress(progress: number | object): Promise; + trigger(workload: T, data: DataOf): Promise; } -export interface Workload { - spec: QueueSpec; - concurrency: number; - rateLimit?: { max: number; per: string }; - process(ctx: ProcessContext): Promise; - onTerminalFailure?(data: TData, err: Error): Promise; +/** + * A QueueSpec defines the specification for a queue, including + * it's name, deduplication key, and settings. + */ +export interface QueueSpec { + name: TName; + dedupKey?(data: DataOf): string; + jobOptions: { + attempts: number; + backoff: { type: 'fixed' | 'exponential'; delayMs: number }; + keep: { completed: number; failed: number }; + }; } - export type Schedule = { every: string } | { pattern: string }; -export interface JobManager { - register(w: Workload): void; - registerCron(cron: CronWorkload): void; +/** + * A Workload is a single kind of background work, declared + * as the queue it runs on, the code that processes the job, + * and how much of it may run at once. + * + * Jobs reach a workload's queue in one of two ways: someone calls `trigger`, or - if the + * workload declares a `schedule` - the JobManager enqueues one on that cadence. A sweep is + * just a scheduled workload that carries no payload, and whose `process` scans for work and + * triggers it onto other workloads' queues. + */ +export interface Workload { + spec: QueueSpec; + concurrency: number; + /** + * If set, the JobManager enqueues a job on this cadence rather than waiting for someone to + * `trigger` one. Scheduled jobs carry no payload, so `TData` should be `void`. + */ + schedule?: Schedule; + rateLimit?: { max: number; per: string }; + process(ctx: ProcessContext): Promise; + onTerminalFailure?(data: DataOf, err: Error): Promise; +} - start(): Promise; - stop(): Promise; +export interface JobManager { + register(w: Workload): void; - trigger(workload: string, data: T): Promise; + start(): Promise; + stop(): Promise; - status(workload: string): Promise; - jobDetail(workload: string, jobId: string): Promise; -} + trigger( + workload: TName, + data: DataOf + ): Promise; -export interface CronWorkload { - name: string; - schedule: Schedule; - handler(ctx: CronContext): Promise; + status(workload: string): Promise; + jobDetail(workload: string, jobId: string): Promise; } -export interface CronContext { - trigger(workload: string, data: T): Promise; -} export interface QueueCounts { - waiting: number; - active: number; - delayed: number; - completed: number; - failed: number; - paused: number; - prioritized?: number; - 'waiting-children'?: number; + waiting: number; + active: number; + delayed: number; + completed: number; + failed: number; + paused: number; + prioritized?: number; + 'waiting-children'?: number; } export interface JobDetail { - id: string; - name: string; - state: 'waiting' | 'active' | 'delayed' | 'completed' | 'failed' | 'paused' | 'unknown'; - data: TData; - attemptsMade: number; - maxAttempts: number; - result?: TResult | null; - failedReason?: string | null; - stacktrace?: string[]; - logs: string[]; - enqueuedAt: number; - startedAt: number | null; - finishedAt: number | null; - waitMs?: number | null; - runMs?: number | null; + id: string; + name: string; + state: 'waiting' | 'active' | 'delayed' | 'completed' | 'failed' | 'paused' | 'unknown'; + data: TData; + attemptsMade: number; + maxAttempts: number; + result?: TResult | null; + failedReason?: string | null; + stacktrace?: string[]; + logs: string[]; + enqueuedAt: number; + startedAt: number | null; + finishedAt: number | null; + waitMs?: number | null; + runMs?: number | null; } diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index 8c1d64b52..c55c72a05 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -31,7 +31,8 @@ const schema = { "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -41,7 +42,8 @@ const schema = { "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", @@ -215,7 +217,8 @@ const schema = { "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -225,7 +228,8 @@ const schema = { "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index 7fa7f5a17..3bede3def 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -100,6 +100,7 @@ export interface Settings { */ resyncConnectionIntervalMs?: number; /** + * @deprecated * The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second. */ resyncConnectionPollingIntervalMs?: number; @@ -108,6 +109,7 @@ export interface Settings { */ reindexRepoPollingIntervalMs?: number; /** + * @deprecated * The number of connection sync jobs to run concurrently. Defaults to 8. */ maxConnectionSyncJobConcurrency?: number; diff --git a/schemas/v3/index.json b/schemas/v3/index.json index 874f9f8d5..6dc6255f1 100644 --- a/schemas/v3/index.json +++ b/schemas/v3/index.json @@ -30,7 +30,8 @@ "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -40,7 +41,8 @@ "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", From ea54e4cd464ebba9adc5bd886eee4c4463e56208 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 27 Jul 2026 19:10:47 -0700 Subject: [PATCH 03/15] further wip --- packages/backend/src/configManager.ts | 2 +- .../backend/src/connectionWorkload.test.ts | 203 ++++++++++++++++++ packages/backend/src/connectionWorkload.ts | 73 +++++-- packages/backend/src/index.ts | 45 +--- packages/backend/src/jobManager.ts | 135 +++++------- .../backend/src/jobManagerLifecycle.test.ts | 148 +++++++++++++ packages/backend/src/jobProducer.ts | 36 ---- .../src/reconciliationWorkload.test.ts | 97 +++++++++ .../backend/src/reconciliationWorkload.ts | 42 ++++ packages/backend/src/types.ts | 51 +---- packages/shared/package.json | 2 + packages/shared/src/index.server.ts | 13 ++ packages/shared/src/jobProducer.test.ts | 90 ++++++++ packages/shared/src/jobProducer.ts | 66 ++++++ packages/shared/src/queue.ts | 50 +++++ .../src/features/workerApi/actions.test.ts | 132 ++++++++++++ .../web/src/features/workerApi/actions.ts | 39 ++-- packages/web/src/lib/jobProducer.ts | 11 + yarn.lock | 2 + 19 files changed, 1001 insertions(+), 236 deletions(-) create mode 100644 packages/backend/src/connectionWorkload.test.ts create mode 100644 packages/backend/src/jobManagerLifecycle.test.ts delete mode 100644 packages/backend/src/jobProducer.ts create mode 100644 packages/backend/src/reconciliationWorkload.test.ts create mode 100644 packages/backend/src/reconciliationWorkload.ts create mode 100644 packages/shared/src/jobProducer.test.ts create mode 100644 packages/shared/src/jobProducer.ts create mode 100644 packages/shared/src/queue.ts create mode 100644 packages/web/src/features/workerApi/actions.test.ts create mode 100644 packages/web/src/lib/jobProducer.ts diff --git a/packages/backend/src/configManager.ts b/packages/backend/src/configManager.ts index 74f050fb4..669d3d43e 100644 --- a/packages/backend/src/configManager.ts +++ b/packages/backend/src/configManager.ts @@ -133,4 +133,4 @@ export class ConfigManager { public dispose = async () => { await this.watcher.close(); } -} \ No newline at end of file +} diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionWorkload.test.ts new file mode 100644 index 000000000..8490118fc --- /dev/null +++ b/packages/backend/src/connectionWorkload.test.ts @@ -0,0 +1,203 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + connectionSyncJobCreateMany: vi.fn(), + connectionSyncJobUpdate: vi.fn(), + connectionFindUniqueOrThrow: vi.fn(), + connectionUpdate: vi.fn(), + transactionConnectionUpdate: vi.fn(), + transactionRepoUpsert: vi.fn(), + compileGithubConfig: vi.fn(), + loadConfig: vi.fn(), + syncSearchContexts: vi.fn(), +})); + +vi.mock('@sentry/node', () => ({ + captureException: vi.fn(), +})); + +vi.mock('@sourcebot/shared', () => ({ + CONNECTION_QUEUE: { + name: 'connection', + dedupKey: ({ connectionId }: { connectionId: number }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + }, + }, + createLogger: vi.fn(() => ({ + debug: vi.fn(), + error: vi.fn(), + })), + env: { + CONFIG_PATH: '/config.json', + CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: 60_000, + }, + loadConfig: mocks.loadConfig, +})); + +vi.mock('./prisma.js', () => ({ + prisma: { + connectionSyncJob: { + createMany: mocks.connectionSyncJobCreateMany, + update: mocks.connectionSyncJobUpdate, + }, + connection: { + findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, + update: mocks.connectionUpdate, + }, + $transaction: vi.fn(async (callback) => callback({ + connection: { + update: mocks.transactionConnectionUpdate, + }, + repo: { + upsert: mocks.transactionRepoUpsert, + }, + })), + }, +})); + +vi.mock('./repoCompileUtils.js', () => ({ + compileAzureDevOpsConfig: vi.fn(), + compileBitbucketConfig: vi.fn(), + compileGenericGitHostConfig: vi.fn(), + compileGerritConfig: vi.fn(), + compileGiteaConfig: vi.fn(), + compileGithubConfig: mocks.compileGithubConfig, + compileGitlabConfig: vi.fn(), +})); + +vi.mock('./ee/syncSearchContexts.js', () => ({ + syncSearchContexts: mocks.syncSearchContexts, +})); + +import { connectionWorkload } from './connectionWorkload.js'; + +const data = { + connectionId: 42, + orgId: 7, +}; + +const lifecycleContext = { + data, + jobId: 'job-1', + attemptsMade: 0, + maxAttempts: 2, +}; + +describe('connectionWorkload lifecycle', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-07-27T12:00:00.000Z')); + mocks.connectionSyncJobCreateMany.mockResolvedValue({ count: 1 }); + mocks.connectionSyncJobUpdate.mockResolvedValue({}); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + test('creates a pending ConnectionSyncJob when enqueued', async () => { + await connectionWorkload.queueSpec.onEnqueued?.(lifecycleContext); + + expect(mocks.connectionSyncJobCreateMany).toHaveBeenCalledWith({ + data: [{ + id: 'job-1', + connectionId: 42, + status: 'PENDING', + warningMessages: [], + }], + skipDuplicates: true, + }); + }); + + test('marks the ConnectionSyncJob in progress when processing starts', async () => { + await connectionWorkload.onStarted?.(lifecycleContext); + + expect(mocks.connectionSyncJobCreateMany).toHaveBeenCalledWith({ + data: [{ + id: 'job-1', + connectionId: 42, + status: 'IN_PROGRESS', + warningMessages: [], + }], + skipDuplicates: true, + }); + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'IN_PROGRESS', + }, + }); + }); + + test('persists compile warnings while processing', async () => { + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: 'github', + config: { + type: 'github', + }, + }); + mocks.compileGithubConfig.mockResolvedValue({ + repoData: [], + warnings: ['Repository was archived'], + }); + mocks.connectionUpdate.mockResolvedValue({}); + mocks.loadConfig.mockResolvedValue({ contexts: undefined }); + mocks.syncSearchContexts.mockResolvedValue(undefined); + + await connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + log: vi.fn(), + updateProgress: vi.fn(), + trigger: vi.fn(), + }); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + warningMessages: ['Repository was archived'], + }, + }); + }); + + test('marks the ConnectionSyncJob completed', async () => { + await connectionWorkload.onCompleted?.(lifecycleContext, undefined); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'COMPLETED', + completedAt: new Date('2026-07-27T12:00:00.000Z'), + }, + }); + }); + + test('marks the ConnectionSyncJob failed with its error', async () => { + await connectionWorkload.onTerminalFailure?.( + { ...lifecycleContext, attemptsMade: 2 }, + new Error('Connection failed'), + ); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'FAILED', + completedAt: new Date('2026-07-27T12:00:00.000Z'), + errorMessage: 'Connection failed', + }, + }); + }); +}); diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts index a3b965fb1..e81b9c954 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionWorkload.ts @@ -1,33 +1,23 @@ -import { QueueSpec, Workload } from "./types.js"; +import { Workload } from "./types.js"; import { prisma } from "./prisma.js"; +import { ConnectionSyncJobStatus } from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; -import { createLogger, env, loadConfig } from "@sourcebot/shared"; +import { CONNECTION_QUEUE, createLogger, env, loadConfig } from "@sourcebot/shared"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import * as Sentry from "@sentry/node"; - -const connectionQueueSpec: QueueSpec<'connection'> = { - name: 'connection', - jobOptions: { - attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 } - }, - dedupKey: (data) => `connection:${data.connectionId}` -} - -// @todo const logger = createLogger('connection-workflow'); export const connectionWorkload: Workload<'connection'> = { - spec: connectionQueueSpec, + queueSpec: CONNECTION_QUEUE, concurrency: 2, process: async ({ data: { connectionId, orgId }, + jobId, signal, }) => { const connection = await prisma.connection.findUniqueOrThrow({ @@ -64,7 +54,16 @@ export const connectionWorkload: Workload<'connection'> = { } })(); - let { repoData } = result; + let { repoData, warnings } = result; + + await prisma.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + warningMessages: warnings, + }, + }); // Filter out any duplicates by external_id and external_codeHostUrl. repoData = repoData.filter((repo, index, self) => { @@ -137,5 +136,47 @@ export const connectionWorkload: Workload<'connection'> = { logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`); Sentry.captureException(err); } + }, + onStarted: async ({ data, jobId }) => { + await prisma.connectionSyncJob.createMany({ + data: [{ + id: jobId, + connectionId: data.connectionId, + status: ConnectionSyncJobStatus.IN_PROGRESS, + warningMessages: [], + }], + skipDuplicates: true, + }); + await prisma.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.IN_PROGRESS, + }, + }); + }, + onCompleted: async ({ jobId }) => { + await prisma.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.COMPLETED, + completedAt: new Date(), + }, + }); + }, + onTerminalFailure: async ({ jobId }, error) => { + await prisma.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + }); } } diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index fee139a87..9efa68768 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -6,15 +6,15 @@ import 'express-async-errors'; import { existsSync } from 'fs'; import { mkdir } from 'fs/promises'; import { ConfigManager } from "./configManager.js"; -import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, SHUTDOWN_SIGNALS, SINGLE_TENANT_ORG_ID } from './constants.js'; +import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, SHUTDOWN_SIGNALS } from './constants.js'; import { GithubAppManager } from "./ee/githubAppManager.js"; import { hasEntitlement } from "./entitlements.js"; import { BullMQJobManager } from "./jobManager.js"; import { shutdownPosthog } from "./posthog.js"; import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; +import { createReconciliationWorkload } from "./reconciliationWorkload.js"; import { redis } from "./redis.js"; -import { QueueSpec, Workload } from "./types.js"; import { connectionWorkload } from "./connectionWorkload.js"; const logger = createLogger('backend-entrypoint'); @@ -83,43 +83,12 @@ logger.info('Worker started.'); // (repo-index, connection-sync, permission syncers) are ported onto it in subsequent phases. const jobManager = new BullMQJobManager(redis); -const cronQueueSpec: QueueSpec<'cron'> = { - name: 'cron', - jobOptions: { - attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 } - } -} - -const cronWorkload: Workload<'cron'> = { - concurrency: 1, - schedule: { every: '5s' }, - spec: cronQueueSpec, - process: async ({ jobId, trigger }) => { - console.log(`cron ${jobId}`); - - const thresholdDate = new Date(Date.now() - settings.resyncConnectionIntervalMs); - const connections = await prisma.connection.findMany({ - where: { - OR: [ - { syncedAt: null }, - { syncedAt: { lt: thresholdDate }} - ] - } - }); - - await Promise.all(connections.map(async (connection) => { - console.log(`Scheduling work for ${connection.id}`); - await trigger('connection', { - connectionId: connection.id, - orgId: SINGLE_TENANT_ORG_ID, - }) - })) - } -} +const reconciliationWorkload = createReconciliationWorkload({ + db: prisma, + settings, +}); -jobManager.register(cronWorkload); +jobManager.register(reconciliationWorkload); jobManager.register(connectionWorkload); await jobManager.start(); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index 35dfa0af0..31e935b1f 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -1,10 +1,9 @@ import * as Sentry from "@sentry/node"; -import { createLogger } from "@sourcebot/shared"; +import { BullMQJobProducer, createLogger, DataOf, JobLifecycleContext, QueueName } from "@sourcebot/shared"; import { Job, Worker } from "bullmq"; import { Redis } from "ioredis"; import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; -import { JobProducer } from "./jobProducer.js"; -import { DataOf, JobDetail, JobManager, QueueCounts, QueueName, Schedule, Workload } from "./types.js"; +import { JobDetail, JobManager, Schedule, Workload } from "./types.js"; const LOG_TAG = 'job-manager'; const logger = createLogger(LOG_TAG); @@ -48,15 +47,15 @@ const scheduleToRepeat = (schedule: Schedule) => export class BullMQJobManager implements JobManager { private readonly workloads = new Map>(); private readonly workers = new Map(); - private readonly producer: JobProducer; + private readonly producer: BullMQJobProducer; private readonly abortController = new AbortController(); constructor(private readonly connection: Redis) { - this.producer = new JobProducer(connection); + this.producer = new BullMQJobProducer(connection); } register(workload: Workload): void { - const name = workload.spec.name; + const name = workload.queueSpec.name; if (this.workloads.has(name)) { throw new Error(`Workload "${name}" is already registered`); } @@ -81,72 +80,12 @@ export class BullMQJobManager implements JobManager { async trigger( workloadName: TName, data: DataOf - ): Promise { - const workload = this.workloads.get(workloadName); + ): Promise { + const workload = this.workloads.get(workloadName) as Workload | undefined; if (!workload) { throw new Error(`Cannot trigger unknown workload "${workloadName}"`); } - await this.producer.enqueue(workload.spec, data); - } - - async status(workloadName: string): Promise { - this.requireRegistered(workloadName); - const counts = await this.producer.queue(workloadName).getJobCounts( - 'waiting', - 'active', - 'delayed', - 'completed', - 'failed', - 'paused', - 'prioritized', - 'waiting-children', - ); - return { - waiting: counts.waiting ?? 0, - active: counts.active ?? 0, - delayed: counts.delayed ?? 0, - completed: counts.completed ?? 0, - failed: counts.failed ?? 0, - paused: counts.paused ?? 0, - prioritized: counts.prioritized ?? 0, - 'waiting-children': counts['waiting-children'] ?? 0, - }; - } - - async jobDetail(workloadName: string, jobId: string): Promise { - this.requireRegistered(workloadName); - const queue = this.producer.queue(workloadName); - const job = await queue.getJob(jobId); - if (!job) { - return null; - } - - const [state, jobLogs] = await Promise.all([ - job.getState(), - queue.getJobLogs(jobId), - ]); - - const enqueuedAt = job.timestamp; - const startedAt = job.processedOn ?? null; - const finishedAt = job.finishedOn ?? null; - - return { - id: job.id ?? jobId, - name: job.name, - state: normalizeJobState(state), - data: job.data, - attemptsMade: job.attemptsMade, - maxAttempts: job.opts.attempts ?? 1, - result: job.returnvalue ?? null, - failedReason: job.failedReason ?? null, - stacktrace: job.stacktrace ?? [], - logs: jobLogs.logs, - enqueuedAt, - startedAt, - finishedAt, - waitMs: startedAt !== null ? startedAt - enqueuedAt : null, - runMs: startedAt !== null && finishedAt !== null ? finishedAt - startedAt : null, - }; + return this.producer.enqueue(workload.queueSpec, data); } async stop(): Promise { @@ -165,22 +104,24 @@ export class BullMQJobManager implements JobManager { } private async startWorkload(workload: Workload): Promise { - const { spec, concurrency, rateLimit, schedule } = workload; + const { queueSpec: spec, concurrency, rateLimit, schedule } = workload; const queue = this.producer.queue(spec.name); const worker = new Worker( spec.name, - (job) => workload.process({ - data: job.data, - jobId: job.id ?? '', - attemptsMade: job.attemptsMade, - maxAttempts: job.opts.attempts ?? 1, - signal: this.abortController.signal, - log: async (message) => { await job.log(message); }, - updateProgress: (progress) => job.updateProgress(progress), - trigger: (target, data) => this.trigger(target, data), - }), + async (job) => { + const lifecycleContext = this.jobLifecycleContext(job); + await workload.onStarted?.(lifecycleContext); + + return workload.process({ + ...lifecycleContext, + signal: this.abortController.signal, + log: async (message) => { await job.log(message); }, + updateProgress: (progress) => job.updateProgress(progress), + trigger: (target, data) => this.trigger(target, data), + }); + }, { connection: this.connection, concurrency, @@ -194,6 +135,9 @@ export class BullMQJobManager implements JobManager { worker.on('failed', (job, error) => { void this.onWorkloadJobFailed(workload, job, error); }); + worker.on('completed', (job, result) => { + void this.onWorkloadJobCompleted(workload, job, result); + }); worker.on('error', (error) => { logger.error(`Worker "${spec.name}" error:`, error); }); @@ -232,21 +176,38 @@ export class BullMQJobManager implements JobManager { const maxAttempts = job.opts.attempts ?? 1; const isTerminal = job.attemptsMade >= maxAttempts; if (!isTerminal) { - logger.warn(`Workload "${workload.spec.name}" job ${job.id} failed attempt ${job.attemptsMade}/${maxAttempts}; will retry: ${error.message}`); + logger.warn(`Workload "${workload.queueSpec.name}" job ${job.id} failed attempt ${job.attemptsMade}/${maxAttempts}; will retry: ${error.message}`); return; } - logger.error(`Workload "${workload.spec.name}" job ${job.id} failed terminally after ${job.attemptsMade} attempt(s): ${error.message}`); + logger.error(`Workload "${workload.queueSpec.name}" job ${job.id} failed terminally after ${job.attemptsMade} attempt(s): ${error.message}`); + try { - await workload.onTerminalFailure?.(job.data, error); + await workload.onTerminalFailure?.(this.jobLifecycleContext(job), error); } catch (hookError) { Sentry.captureException(hookError); - logger.error(`onTerminalFailure for workload "${workload.spec.name}" threw:`, hookError); + logger.error(`onTerminalFailure for workload "${workload.queueSpec.name}" threw:`, hookError); } } - private requireRegistered(workloadName: string): void { - if (!this.workloads.has(workloadName)) { - throw new Error(`Workload "${workloadName}" is not registered`); + private async onWorkloadJobCompleted( + workload: Workload, + job: Job, + result: TResult, + ): Promise { + try { + await workload.onCompleted?.(this.jobLifecycleContext(job), result); + } catch (hookError) { + Sentry.captureException(hookError); + logger.error(`onCompleted for workload "${workload.queueSpec.name}" threw:`, hookError); } } + + private jobLifecycleContext(job: Job): JobLifecycleContext { + return { + data: job.data, + jobId: job.id ?? '', + attemptsMade: job.attemptsMade, + maxAttempts: job.opts.attempts ?? 1, + }; + } } diff --git a/packages/backend/src/jobManagerLifecycle.test.ts b/packages/backend/src/jobManagerLifecycle.test.ts new file mode 100644 index 000000000..5117233b3 --- /dev/null +++ b/packages/backend/src/jobManagerLifecycle.test.ts @@ -0,0 +1,148 @@ +import { Redis } from 'ioredis'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { Workload } from './types.js'; + +const mocks = vi.hoisted(() => ({ + enqueue: vi.fn(), + producerClose: vi.fn(), + workerClose: vi.fn(), + workers: [] as Array<{ + processor: (job: unknown) => Promise; + handlers: Map void>; + }>, +})); + +vi.mock('@sentry/node', () => ({ + captureException: vi.fn(), +})); + +vi.mock('@sourcebot/shared', () => ({ + createLogger: vi.fn(() => ({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + })), + BullMQJobProducer: class { + enqueue = mocks.enqueue; + close = mocks.producerClose; + queue = vi.fn(() => ({ + getJobCounts: vi.fn(), + upsertJobScheduler: vi.fn(), + })); + }, +})); + +vi.mock('./constants.js', () => ({ + WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, +})); + +vi.mock('bullmq', () => ({ + Worker: class { + private readonly record: (typeof mocks.workers)[number]; + + constructor(_name: string, processor: (job: unknown) => Promise) { + this.record = { processor, handlers: new Map() }; + mocks.workers.push(this.record); + } + + on(event: string, handler: (...args: unknown[]) => void) { + this.record.handlers.set(event, handler); + } + + close = mocks.workerClose; + }, +})); + +import { BullMQJobManager } from './jobManager.js'; + +const createWorkload = ( + overrides: Partial> = {}, +): Workload<'connection', { repoCount: number }> => ({ + queueSpec: { + name: 'connection', + dedupKey: ({ connectionId }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + }, + }, + concurrency: 2, + process: vi.fn(async () => ({ repoCount: 3 })), + ...overrides, +}); + +const data = { connectionId: 42, orgId: 1 }; +const job = { + id: 'job-1', + data, + attemptsMade: 2, + opts: { attempts: 2 }, + log: vi.fn(), + updateProgress: vi.fn(), +}; + +describe('BullMQJobManager lifecycle', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.workers.length = 0; + mocks.enqueue.mockResolvedValue('job-1'); + }); + + test('delegates enqueueing to BullMQJobProducer and returns its job id', async () => { + const manager = new BullMQJobManager({} as Redis); + const workload = createWorkload(); + manager.register(workload); + + const result = await manager.trigger('connection', data); + + expect(result).toBe('job-1'); + expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data); + }); + + test('calls onStarted before processing and onCompleted after completion', async () => { + const calls: string[] = []; + const workload = createWorkload({ + onStarted: vi.fn(async () => { calls.push('started'); }), + process: vi.fn(async () => { + calls.push('processed'); + return { repoCount: 3 }; + }), + onCompleted: vi.fn(async () => { calls.push('completed'); }), + }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + const result = await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); + expect(calls).toEqual(['started', 'processed']); + + mocks.workers[0].handlers.get('completed')?.(job, result); + await vi.waitFor(() => expect(calls).toEqual(['started', 'processed', 'completed'])); + expect(workload.onCompleted).toHaveBeenCalledWith( + expect.objectContaining({ data, jobId: 'job-1', maxAttempts: 2 }), + { repoCount: 3 }, + ); + }); + + test('reports lifecycle metadata after terminal failure', async () => { + const onTerminalFailure = vi.fn(); + const workload = createWorkload({ onTerminalFailure }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + const error = new Error('failed'); + mocks.workers[0].handlers.get('failed')?.(job, error); + + await vi.waitFor(() => { + expect(onTerminalFailure).toHaveBeenCalledWith({ + data, + jobId: 'job-1', + attemptsMade: 2, + maxAttempts: 2, + }, error); + }); + }); +}); diff --git a/packages/backend/src/jobProducer.ts b/packages/backend/src/jobProducer.ts deleted file mode 100644 index 8cd28a2e8..000000000 --- a/packages/backend/src/jobProducer.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Queue } from "bullmq"; -import { Redis } from "ioredis"; -import { DataOf, QueueName, QueueSpec } from "./types.js"; - -export class JobProducer { - private readonly queues = new Map(); - - constructor(private readonly connection: Redis) {} - - queue(name: string): Queue { - let queue = this.queues.get(name); - if (!queue) { - queue = new Queue(name, { connection: this.connection }); - this.queues.set(name, queue); - } - return queue; - } - - async enqueue( - spec: QueueSpec, - data: DataOf - ): Promise { - const dedupKey = spec.dedupKey?.(data); - await this.queue(spec.name).add(spec.name, data, { - ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), - attempts: spec.jobOptions.attempts, - backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, - removeOnComplete: { count: spec.jobOptions.keep.completed }, - removeOnFail: { count: spec.jobOptions.keep.failed }, - }); - } - - async close(): Promise { - await Promise.all([...this.queues.values()].map((queue) => queue.close())); - } -} diff --git a/packages/backend/src/reconciliationWorkload.test.ts b/packages/backend/src/reconciliationWorkload.test.ts new file mode 100644 index 000000000..61db94afb --- /dev/null +++ b/packages/backend/src/reconciliationWorkload.test.ts @@ -0,0 +1,97 @@ +import type { PrismaClient } from '@sourcebot/db'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; + +vi.mock('@sourcebot/shared', () => ({ + RECONCILIATION_QUEUE: { + name: 'reconciliation', + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + }, + }, + createLogger: vi.fn(() => ({ + debug: vi.fn(), + })), +})); + +import { createReconciliationWorkload } from './reconciliationWorkload.js'; + +describe('reconciliationWorkload', () => { + const findMany = vi.fn(); + const db = { + connection: { + findMany, + }, + } as unknown as PrismaClient; + + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-07-27T12:00:00.000Z')); + findMany.mockReset(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + test('runs every 15 minutes on the reconciliation queue', () => { + const workload = createReconciliationWorkload({ + db, + settings: { + resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, + }, + }); + + expect(workload.queueSpec.name).toBe('reconciliation'); + expect(workload.schedule).toEqual({ every: '15m' }); + expect(workload.concurrency).toBe(1); + }); + + test('triggers connection syncs for connections that are due', async () => { + findMany.mockResolvedValue([ + { id: 42, orgId: 1 }, + { id: 84, orgId: 2 }, + ]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings: { + resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, + }, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + signal: new AbortController().signal, + log: vi.fn(), + updateProgress: vi.fn(), + trigger, + }); + + expect(findMany).toHaveBeenCalledWith({ + where: { + OR: [ + { syncedAt: null }, + { syncedAt: { lt: new Date('2026-07-26T12:00:00.000Z') } }, + ], + }, + select: { + id: true, + orgId: true, + }, + }); + expect(trigger).toHaveBeenCalledTimes(2); + expect(trigger).toHaveBeenCalledWith('connection', { + connectionId: 42, + orgId: 1, + }); + expect(trigger).toHaveBeenCalledWith('connection', { + connectionId: 84, + orgId: 2, + }); + }); +}); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts new file mode 100644 index 000000000..500389b2e --- /dev/null +++ b/packages/backend/src/reconciliationWorkload.ts @@ -0,0 +1,42 @@ +import { PrismaClient } from "@sourcebot/db"; +import { createLogger, RECONCILIATION_QUEUE } from "@sourcebot/shared"; +import { Settings, Workload } from "./types.js"; + +const logger = createLogger('reconciliation-workload'); + +interface ReconciliationWorkloadDependencies { + db: PrismaClient; + settings: Pick; +} + +export const createReconciliationWorkload = ({ + db, + settings, +}: ReconciliationWorkloadDependencies): Workload<'reconciliation'> => ({ + concurrency: 1, + schedule: { every: '15m' }, + queueSpec: RECONCILIATION_QUEUE, + process: async ({ trigger }) => { + const thresholdDate = new Date(Date.now() - settings.resyncConnectionIntervalMs); + const connections = await db.connection.findMany({ + where: { + OR: [ + { syncedAt: null }, + { syncedAt: { lt: thresholdDate } }, + ], + }, + select: { + id: true, + orgId: true, + }, + }); + + await Promise.all(connections.map(async (connection) => { + logger.debug(`Scheduling connection sync for connection ${connection.id}`); + await trigger('connection', { + connectionId: connection.id, + orgId: connection.orgId, + }); + })); + }, +}); diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 66295017e..ae6b7fc7e 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -1,6 +1,7 @@ import { Connection, Repo, RepoToConnection } from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type"; +import { DataOf, JobLifecycleContext, QueueName, QueueSpec } from "@sourcebot/shared"; export type Settings = Required; @@ -14,52 +15,20 @@ export type WithRequired = T & { [P in K]-?: T[P] }; export type RepoWithConnections = Repo & { connections: (RepoToConnection & { connection: Connection })[] }; - export type RepoAuthCredentials = { hostUrl?: string; token: string; cloneUrlWithToken?: string; authHeader?: string; - /** The connection that configured the - * credentials for this repo. - */ connectionConfig?: ConnectionConfig; } -export interface QueueRegistry { - 'connection': { - connectionId: number, - orgId: number - }, - 'cron': {} -} - -export type QueueName = keyof QueueRegistry; -export type DataOf = QueueRegistry[TName]; -export interface ProcessContext { - data: DataOf; - jobId: string; - attemptsMade: number; - maxAttempts: number; +export interface ProcessContext extends JobLifecycleContext { signal: AbortSignal; log(message: string): Promise; updateProgress(progress: number | object): Promise; - trigger(workload: T, data: DataOf): Promise; -} - -/** - * A QueueSpec defines the specification for a queue, including - * it's name, deduplication key, and settings. - */ -export interface QueueSpec { - name: TName; - dedupKey?(data: DataOf): string; - jobOptions: { - attempts: number; - backoff: { type: 'fixed' | 'exponential'; delayMs: number }; - keep: { completed: number; failed: number }; - }; + trigger(workload: T, data: DataOf): Promise; } export type Schedule = { every: string } | { pattern: string }; @@ -75,7 +44,7 @@ export type Schedule = { every: string } | { pattern: string }; * triggers it onto other workloads' queues. */ export interface Workload { - spec: QueueSpec; + queueSpec: QueueSpec; concurrency: number; /** * If set, the JobManager enqueues a job on this cadence rather than waiting for someone to @@ -84,7 +53,12 @@ export interface Workload { schedule?: Schedule; rateLimit?: { max: number; per: string }; process(ctx: ProcessContext): Promise; - onTerminalFailure?(data: DataOf, err: Error): Promise; + /** Called before `process` on every attempt. */ + onStarted?(ctx: JobLifecycleContext): Promise; + /** Called after BullMQ marks the job as completed. */ + onCompleted?(ctx: JobLifecycleContext, result: TResult): Promise; + /** Called after BullMQ exhausts all attempts and marks the job as failed. */ + onTerminalFailure?(ctx: JobLifecycleContext, err: Error): Promise; } export interface JobManager { @@ -96,10 +70,7 @@ export interface JobManager { trigger( workload: TName, data: DataOf - ): Promise; - - status(workload: string): Promise; - jobDetail(workload: string, jobId: string): Promise; + ): Promise; } diff --git a/packages/shared/package.json b/packages/shared/package.json index 6ded4e24b..e865230c3 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -14,10 +14,12 @@ "@google-cloud/secret-manager": "^6.1.1", "@logtail/node": "^0.5.2", "@logtail/winston": "^0.5.2", + "@sentry/node": "^10.40.0", "@sourcebot/db": "workspace:*", "@sourcebot/schemas": "workspace:*", "@t3-oss/env-core": "^0.13.10", "ajv": "^8.17.1", + "bullmq": "^5.34.10", "ioredis": "^5.4.2", "micromatch": "^4.0.8", "strip-json-comments": "^5.0.1", diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 80fe65ce8..753240ab5 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -92,3 +92,16 @@ export { compareVersions, } from "./versionUtils.js"; export type { Version } from "./versionUtils.js"; +export type { + QueueName, + JobLifecycleContext, + DataOf, + QueueSpec, +} from "./queue.js" +export { + CONNECTION_QUEUE, + RECONCILIATION_QUEUE, +} from "./queue.js"; +export { + BullMQJobProducer, +} from "./jobProducer.js"; diff --git a/packages/shared/src/jobProducer.test.ts b/packages/shared/src/jobProducer.test.ts new file mode 100644 index 000000000..ffdad845b --- /dev/null +++ b/packages/shared/src/jobProducer.test.ts @@ -0,0 +1,90 @@ +import type { Redis } from 'ioredis'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import type { QueueSpec } from './queue.js'; + +const queueMocks = vi.hoisted(() => ({ + add: vi.fn(), + close: vi.fn(), +})); + +vi.mock('@sentry/node', () => ({ + captureException: vi.fn(), +})); + +vi.mock('./logger.js', () => ({ + createLogger: vi.fn(() => ({ + error: vi.fn(), + })), +})); + +vi.mock('bullmq', () => ({ + Queue: class { + constructor() { + return queueMocks; + } + }, +})); + +import { BullMQJobProducer } from './jobProducer.js'; + +const connectionSpec: QueueSpec<'connection'> = { + name: 'connection', + dedupKey: ({ connectionId }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + }, +}; + +const data = { connectionId: 42, orgId: 1 }; + +describe('BullMQJobProducer', () => { + const redis = {} as Redis; + + beforeEach(() => { + vi.clearAllMocks(); + queueMocks.add.mockImplementation(async (_name, _data, options) => ({ id: options.jobId })); + }); + + test('returns the job id when BullMQ accepts the proposed id', async () => { + const producer = new BullMQJobProducer(redis); + + const result = await producer.enqueue(connectionSpec, data); + + expect(result).toEqual(expect.any(String)); + expect(queueMocks.add).toHaveBeenCalledWith( + 'connection', + data, + expect.objectContaining({ + jobId: result, + deduplication: { id: 'connection:42' }, + }), + ); + }); + + test('calls onEnqueued when a new job is created', async () => { + const onEnqueued = vi.fn(); + const producer = new BullMQJobProducer(redis); + + const result = await producer.enqueue({ ...connectionSpec, onEnqueued }, data); + + expect(onEnqueued).toHaveBeenCalledWith({ + data, + jobId: result, + attemptsMade: 0, + maxAttempts: 2, + }); + }); + + test('returns the existing job id without calling onEnqueued when BullMQ deduplicates the enqueue', async () => { + const onEnqueued = vi.fn(); + queueMocks.add.mockResolvedValue({ id: 'existing-job' }); + const producer = new BullMQJobProducer(redis); + + const result = await producer.enqueue({ ...connectionSpec, onEnqueued }, data); + + expect(result).toBe('existing-job'); + expect(onEnqueued).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/shared/src/jobProducer.ts b/packages/shared/src/jobProducer.ts new file mode 100644 index 000000000..595a46787 --- /dev/null +++ b/packages/shared/src/jobProducer.ts @@ -0,0 +1,66 @@ +import * as Sentry from "@sentry/node"; +import { Queue } from "bullmq"; +import { randomUUID } from "crypto"; +import { Redis } from "ioredis"; +import { createLogger } from "./logger.js"; +import { DataOf, QueueName, QueueSpec } from "./queue.js"; + +const logger = createLogger('job-producer'); + +export class BullMQJobProducer { + private readonly queues = new Map(); + + constructor(private readonly connection: Redis) {} + + queue(name: string): Queue { + let queue = this.queues.get(name); + if (!queue) { + queue = new Queue(name, { connection: this.connection }); + this.queues.set(name, queue); + } + return queue; + } + + async enqueue( + spec: QueueSpec, + data: DataOf + ): Promise { + const dedupKey = spec.dedupKey?.(data); + const queue = this.queue(spec.name); + + const requestedJobId = randomUUID(); + const job = await queue.add(spec.name, data, { + jobId: requestedJobId, + ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), + attempts: spec.jobOptions.attempts, + backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, + removeOnComplete: { count: spec.jobOptions.keep.completed }, + removeOnFail: { count: spec.jobOptions.keep.failed }, + }); + + if (!job.id) { + throw new Error(`BullMQ did not return an id for workload "${spec.name}"`); + } + + const isEnqueued = job.id === requestedJobId; + if (isEnqueued && spec.onEnqueued) { + try { + await spec.onEnqueued({ + data, + jobId: job.id, + attemptsMade: 0, + maxAttempts: spec.jobOptions.attempts, + }); + } catch (error) { + Sentry.captureException(error); + logger.error(`onEnqueued for workload "${spec.name}" threw:`, error); + } + } + + return job.id; + } + + async close(): Promise { + await Promise.all([...this.queues.values()].map((queue) => queue.close())); + } +} diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts new file mode 100644 index 000000000..ef9dd67d5 --- /dev/null +++ b/packages/shared/src/queue.ts @@ -0,0 +1,50 @@ + +export type QueueName = keyof QueueRegistry; +export type DataOf = QueueRegistry[TName]; +type EmptyJobData = Record; + +interface QueueRegistry { + 'connection': { + connectionId: number, + orgId: number + }, + 'reconciliation': EmptyJobData, +} + +export const CONNECTION_QUEUE: QueueSpec<'connection'> = { + name: 'connection', + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 } + }, + dedupKey: (data) => `connection:${data.connectionId}`, +} + + +export const RECONCILIATION_QUEUE: QueueSpec<'reconciliation'> = { + name: 'reconciliation', + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + }, +}; + +export interface QueueSpec { + name: TName; + dedupKey?(data: DataOf): string; + jobOptions: { + attempts: number; + backoff: { type: 'fixed' | 'exponential'; delayMs: number }; + keep: { completed: number; failed: number }; + }; + onEnqueued?(ctx: JobLifecycleContext): Promise; +} + +export interface JobLifecycleContext { + data: DataOf; + jobId: string; + attemptsMade: number; + maxAttempts: number; +} diff --git a/packages/web/src/features/workerApi/actions.test.ts b/packages/web/src/features/workerApi/actions.test.ts new file mode 100644 index 000000000..0a949d983 --- /dev/null +++ b/packages/web/src/features/workerApi/actions.test.ts @@ -0,0 +1,132 @@ +import { ConnectionSyncJobStatus, OrgRole } from '@sourcebot/db'; +import type { DataOf, QueueSpec } from '@sourcebot/shared'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + connectionFindUnique: vi.fn(), + connectionSyncJobCreateMany: vi.fn(), + enqueue: vi.fn(), +})); + +vi.mock('@/middleware/sew', () => ({ + sew: (fn: () => Promise) => fn(), +})); + +vi.mock('@/middleware/withAuth', () => ({ + withAuth: (fn: (context: unknown) => Promise) => fn({ + org: { id: 7 }, + prisma: { + connection: { + findUnique: mocks.connectionFindUnique, + }, + connectionSyncJob: { + createMany: mocks.connectionSyncJobCreateMany, + }, + }, + role: 'OWNER', + }), + withOptionalAuth: vi.fn(), +})); + +vi.mock('@/middleware/withMinimumOrgRole', () => ({ + withMinimumOrgRole: ( + _role: OrgRole, + _minimumRole: OrgRole, + fn: () => Promise, + ) => fn(), +})); + +vi.mock('@/lib/jobProducer', () => ({ + getJobProducer: () => ({ + enqueue: mocks.enqueue, + }), +})); + +vi.mock('@sourcebot/shared', () => ({ + CONNECTION_QUEUE: { + name: 'connection', + dedupKey: ({ connectionId }: { connectionId: number }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + }, + }, + env: { + WORKER_API_URL: 'http://localhost:3060', + }, +})); + +import { syncConnection } from './actions'; + +describe('syncConnection', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.connectionSyncJobCreateMany.mockResolvedValue({ count: 1 }); + mocks.enqueue.mockImplementation(async ( + spec: QueueSpec<'connection'>, + data: DataOf<'connection'>, + ) => { + const jobId = 'job-1'; + await spec.onEnqueued?.({ + data, + jobId, + attemptsMade: 0, + maxAttempts: spec.jobOptions.attempts, + }); + return jobId; + }); + }); + + test('enqueues an org-scoped connection sync and creates its pending job record', async () => { + mocks.connectionFindUnique.mockResolvedValue({ + id: 42, + orgId: 7, + }); + + const result = await syncConnection(42); + + expect(mocks.connectionFindUnique).toHaveBeenCalledWith({ + where: { + id: 42, + orgId: 7, + }, + select: { + id: true, + orgId: true, + }, + }); + expect(mocks.enqueue).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'connection', + onEnqueued: expect.any(Function), + }), + { + connectionId: 42, + orgId: 7, + }, + ); + expect(mocks.connectionSyncJobCreateMany).toHaveBeenCalledWith({ + data: [{ + id: 'job-1', + connectionId: 42, + status: ConnectionSyncJobStatus.PENDING, + warningMessages: [], + }], + skipDuplicates: true, + }); + expect(result).toEqual({ jobId: 'job-1' }); + }); + + test('does not enqueue a missing connection', async () => { + mocks.connectionFindUnique.mockResolvedValue(null); + + const result = await syncConnection(42); + + expect(mocks.enqueue).not.toHaveBeenCalled(); + expect(result).toEqual(expect.objectContaining({ + statusCode: 404, + message: 'Connection not found', + })); + }); +}); diff --git a/packages/web/src/features/workerApi/actions.ts b/packages/web/src/features/workerApi/actions.ts index e040ea1e0..b0fa45ac5 100644 --- a/packages/web/src/features/workerApi/actions.ts +++ b/packages/web/src/features/workerApi/actions.ts @@ -1,37 +1,40 @@ 'use server'; import { sew } from "@/middleware/sew"; -import { repositoryNotFound, unexpectedError } from "@/lib/serviceError"; +import { notFound, repositoryNotFound, unexpectedError } from "@/lib/serviceError"; import { withAuth, withOptionalAuth } from "@/middleware/withAuth"; import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; -import { OrgRole } from "@sourcebot/db"; -import { env } from "@sourcebot/shared"; +import { ConnectionSyncJobStatus, OrgRole } from "@sourcebot/db"; +import { CONNECTION_QUEUE, env } from "@sourcebot/shared"; import z from "zod"; +import { getJobProducer } from "@/lib/jobProducer"; const WORKER_API_URL = env.WORKER_API_URL; export const syncConnection = async (connectionId: number) => sew(() => - withAuth(({ role }) => + withAuth(({ org, prisma, role }) => withMinimumOrgRole(role, OrgRole.OWNER, async () => { - const response = await fetch(`${WORKER_API_URL}/api/sync-connection`, { - method: 'POST', - body: JSON.stringify({ - connectionId - }), - headers: { - 'Content-Type': 'application/json', + const connection = await prisma.connection.findUnique({ + where: { + id: connectionId, + orgId: org.id, + }, + select: { + id: true, + orgId: true, }, }); - if (!response.ok) { - return unexpectedError('Failed to sync connection'); + if (!connection) { + return notFound('Connection not found'); } - const data = await response.json(); - const schema = z.object({ - jobId: z.string(), + const jobId = await getJobProducer().enqueue(CONNECTION_QUEUE, { + connectionId: connection.id, + orgId: connection.orgId, }); - return schema.parse(data); + + return { jobId }; }) ) ); @@ -108,4 +111,4 @@ export const addGithubRepo = async (owner: string, repo: string) => sew(() => }); return schema.parse(data); }) -); \ No newline at end of file +); diff --git a/packages/web/src/lib/jobProducer.ts b/packages/web/src/lib/jobProducer.ts new file mode 100644 index 000000000..1d50ebe6d --- /dev/null +++ b/packages/web/src/lib/jobProducer.ts @@ -0,0 +1,11 @@ +import 'server-only'; + +import { BullMQJobProducer } from '@sourcebot/shared'; +import { getRedisClient } from './redis'; + +let jobProducer: BullMQJobProducer | undefined; + +export function getJobProducer() { + jobProducer ??= new BullMQJobProducer(getRedisClient()); + return jobProducer; +} diff --git a/yarn.lock b/yarn.lock index 088a16ec4..ece7d1c39 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9266,12 +9266,14 @@ __metadata: "@google-cloud/secret-manager": "npm:^6.1.1" "@logtail/node": "npm:^0.5.2" "@logtail/winston": "npm:^0.5.2" + "@sentry/node": "npm:^10.40.0" "@sourcebot/db": "workspace:*" "@sourcebot/schemas": "workspace:*" "@t3-oss/env-core": "npm:^0.13.10" "@types/micromatch": "npm:^4.0.9" "@types/node": "npm:^22.7.5" ajv: "npm:^8.17.1" + bullmq: "npm:^5.34.10" cross-env: "npm:^7.0.3" ioredis: "npm:^5.4.2" micromatch: "npm:^4.0.8" From 809bad44d810c063f2747b9a42b460854736c704 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 28 Jul 2026 15:51:21 -0700 Subject: [PATCH 04/15] further wip --- .../backend/src/connectionWorkload.test.ts | 132 ++----- packages/backend/src/connectionWorkload.ts | 152 ++++---- .../backend/src/ee/accountPermissionSyncer.ts | 2 +- packages/backend/src/jobManager.ts | 47 ++- .../backend/src/jobManagerLifecycle.test.ts | 38 +- .../src/reconciliationWorkload.test.ts | 9 +- .../backend/src/reconciliationWorkload.ts | 6 +- packages/backend/src/types.ts | 4 +- packages/backend/src/utils.ts | 2 +- .../migration.sql | 20 + packages/db/prisma/schema.prisma | 32 +- packages/shared/src/bullmqClient.test.ts | 189 ++++++++++ packages/shared/src/bullmqClient.ts | 142 ++++++++ packages/shared/src/index.server.ts | 23 +- packages/shared/src/jobLogger.test.ts | 113 ++++++ packages/shared/src/jobLogger.ts | 207 +++++++++++ packages/shared/src/jobProducer.test.ts | 90 ----- packages/shared/src/jobProducer.ts | 66 ---- packages/shared/src/queue.ts | 23 +- packages/web/src/actions.ts | 38 +- .../components/defaultSidebar/index.tsx | 5 +- .../(app)/settings/connections/[id]/page.tsx | 211 ----------- .../components/connectionJobsTable.tsx | 344 ------------------ .../components/connectionsTable.tsx | 294 --------------- .../connections/connectionSyncLogsDialog.tsx | 203 +++++++++++ .../settings/connections/connectionsList.tsx | 282 ++++++++++++++ .../app/(app)/settings/connections/page.tsx | 97 +++-- .../web/src/app/(app)/settings/layout.tsx | 7 - .../src/features/workerApi/actions.test.ts | 132 ------- .../web/src/features/workerApi/actions.ts | 45 ++- packages/web/src/lib/bullmqClient.ts | 12 + packages/web/src/lib/jobProducer.ts | 11 - 32 files changed, 1485 insertions(+), 1493 deletions(-) create mode 100644 packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql create mode 100644 packages/shared/src/bullmqClient.test.ts create mode 100644 packages/shared/src/bullmqClient.ts create mode 100644 packages/shared/src/jobLogger.test.ts create mode 100644 packages/shared/src/jobLogger.ts delete mode 100644 packages/shared/src/jobProducer.test.ts delete mode 100644 packages/shared/src/jobProducer.ts delete mode 100644 packages/web/src/app/(app)/settings/connections/[id]/page.tsx delete mode 100644 packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx delete mode 100644 packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx create mode 100644 packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx create mode 100644 packages/web/src/app/(app)/settings/connections/connectionsList.tsx delete mode 100644 packages/web/src/features/workerApi/actions.test.ts create mode 100644 packages/web/src/lib/bullmqClient.ts delete mode 100644 packages/web/src/lib/jobProducer.ts diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionWorkload.test.ts index 8490118fc..d21d4a78a 100644 --- a/packages/backend/src/connectionWorkload.test.ts +++ b/packages/backend/src/connectionWorkload.test.ts @@ -1,8 +1,6 @@ -import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ - connectionSyncJobCreateMany: vi.fn(), - connectionSyncJobUpdate: vi.fn(), connectionFindUniqueOrThrow: vi.fn(), connectionUpdate: vi.fn(), transactionConnectionUpdate: vi.fn(), @@ -24,6 +22,7 @@ vi.mock('@sourcebot/shared', () => ({ attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, keep: { completed: 50, failed: 50 }, + keepLogs: 500, }, }, createLogger: vi.fn(() => ({ @@ -39,10 +38,6 @@ vi.mock('@sourcebot/shared', () => ({ vi.mock('./prisma.js', () => ({ prisma: { - connectionSyncJob: { - createMany: mocks.connectionSyncJobCreateMany, - update: mocks.connectionSyncJobUpdate, - }, connection: { findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, update: mocks.connectionUpdate, @@ -86,62 +81,26 @@ const lifecycleContext = { maxAttempts: 2, }; -describe('connectionWorkload lifecycle', () => { +describe('connectionWorkload', () => { beforeEach(() => { vi.clearAllMocks(); - vi.useFakeTimers(); - vi.setSystemTime(new Date('2026-07-27T12:00:00.000Z')); - mocks.connectionSyncJobCreateMany.mockResolvedValue({ count: 1 }); - mocks.connectionSyncJobUpdate.mockResolvedValue({}); - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - test('creates a pending ConnectionSyncJob when enqueued', async () => { - await connectionWorkload.queueSpec.onEnqueued?.(lifecycleContext); - - expect(mocks.connectionSyncJobCreateMany).toHaveBeenCalledWith({ - data: [{ - id: 'job-1', - connectionId: 42, - status: 'PENDING', - warningMessages: [], - }], - skipDuplicates: true, - }); }); - test('marks the ConnectionSyncJob in progress when processing starts', async () => { - await connectionWorkload.onStarted?.(lifecycleContext); - - expect(mocks.connectionSyncJobCreateMany).toHaveBeenCalledWith({ - data: [{ - id: 'job-1', - connectionId: 42, - status: 'IN_PROGRESS', - warningMessages: [], - }], - skipDuplicates: true, - }); - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ - where: { - id: 'job-1', - }, - data: { - status: 'IN_PROGRESS', - }, - }); + test('does not declare database-backed lifecycle hooks', () => { + expect(connectionWorkload.queueSpec.onEnqueued).toBeUndefined(); + expect(connectionWorkload.onStarted).toBeUndefined(); + expect(connectionWorkload.onCompleted).toBeUndefined(); + expect(connectionWorkload.onTerminalFailure).toBeUndefined(); }); - test('persists compile warnings while processing', async () => { + test('discovers repositories using the connection provider', async () => { + const config = { + type: 'github' as const, + }; mocks.connectionFindUniqueOrThrow.mockResolvedValue({ id: 42, name: 'github', - config: { - type: 'github', - }, + config, }); mocks.compileGithubConfig.mockResolvedValue({ repoData: [], @@ -150,54 +109,33 @@ describe('connectionWorkload lifecycle', () => { mocks.connectionUpdate.mockResolvedValue({}); mocks.loadConfig.mockResolvedValue({ contexts: undefined }); mocks.syncSearchContexts.mockResolvedValue(undefined); - - await connectionWorkload.process({ + const updateProgress = vi.fn(); + const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), + }; + const signal = new AbortController().signal; + + const result = await connectionWorkload.process({ ...lifecycleContext, - signal: new AbortController().signal, - log: vi.fn(), - updateProgress: vi.fn(), + signal, + logger, + updateProgress, trigger: vi.fn(), }); - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ - where: { - id: 'job-1', - }, - data: { - warningMessages: ['Repository was archived'], - }, - }); - }); - - test('marks the ConnectionSyncJob completed', async () => { - await connectionWorkload.onCompleted?.(lifecycleContext, undefined); - - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ - where: { - id: 'job-1', - }, - data: { - status: 'COMPLETED', - completedAt: new Date('2026-07-27T12:00:00.000Z'), + expect(mocks.compileGithubConfig).toHaveBeenCalledWith(config, 42, signal); + expect(logger.info).toHaveBeenCalledWith( + 'Discovered 0 repositories', + { + connectionId: 42, + repositoryCount: 0, }, - }); - }); - - test('marks the ConnectionSyncJob failed with its error', async () => { - await connectionWorkload.onTerminalFailure?.( - { ...lifecycleContext, attemptsMade: 2 }, - new Error('Connection failed'), ); - - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ - where: { - id: 'job-1', - }, - data: { - status: 'FAILED', - completedAt: new Date('2026-07-27T12:00:00.000Z'), - errorMessage: 'Connection failed', - }, - }); + expect(updateProgress).not.toHaveBeenCalled(); + expect(result).toBeUndefined(); }); }); diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts index e81b9c954..b05b9537e 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionWorkload.ts @@ -1,14 +1,11 @@ import { Workload } from "./types.js"; import { prisma } from "./prisma.js"; -import { ConnectionSyncJobStatus } from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; -import { CONNECTION_QUEUE, createLogger, env, loadConfig } from "@sourcebot/shared"; +import { CONNECTION_QUEUE, env, loadConfig } from "@sourcebot/shared"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import * as Sentry from "@sentry/node"; -const logger = createLogger('connection-workflow'); - export const connectionWorkload: Workload<'connection'> = { queueSpec: CONNECTION_QUEUE, concurrency: 2, @@ -17,9 +14,13 @@ export const connectionWorkload: Workload<'connection'> = { connectionId, orgId }, - jobId, + logger, signal, }) => { + logger.info(`Syncing connection ${connectionId}`, { + connectionId, + orgId, + }); const connection = await prisma.connection.findUniqueOrThrow({ where: { id: connectionId @@ -28,41 +29,17 @@ export const connectionWorkload: Workload<'connection'> = { const config = connection.config as unknown as ConnectionConfig; - const result = await (async () => { - switch (config.type) { - case 'github': { - return await compileGithubConfig(config, connectionId, signal); - } - case 'gitlab': { - return await compileGitlabConfig(config, connectionId); - } - case 'gitea': { - return await compileGiteaConfig(config, connectionId); - } - case 'gerrit': { - return await compileGerritConfig(config, connectionId); - } - case 'bitbucket': { - return await compileBitbucketConfig(config, connectionId); - } - case 'azuredevops': { - return await compileAzureDevOpsConfig(config, connectionId); - } - case 'git': { - return await compileGenericGitHostConfig(config, connectionId); - } - } - })(); + const result = await discoverConnectionRepositories({ + config, + connectionId, + signal, + }); - let { repoData, warnings } = result; + let { repoData } = result; - await prisma.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - warningMessages: warnings, - }, + logger.info(`Discovered ${repoData.length} repositories`, { + connectionId, + repositoryCount: repoData.length, }); // Filter out any duplicates by external_id and external_codeHostUrl. @@ -91,7 +68,11 @@ export const connectionWorkload: Workload<'connection'> = { } }); const deleteDuration = performance.now() - deleteStart; - logger.debug(`Deleted all RepoToConnection records for connection ${connection.name} (id: ${connectionId}) in ${deleteDuration}ms`); + logger.debug(`Deleted existing repository associations`, { + connectionId, + connectionName: connection.name, + durationMs: deleteDuration, + }); const totalUpsertStart = performance.now(); for (const repo of repoData) { @@ -108,10 +89,19 @@ export const connectionWorkload: Workload<'connection'> = { create: repo, }) const upsertDuration = performance.now() - upsertStart; - logger.debug(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`); + logger.debug(`Upserted repository ${repo.displayName}`, { + connectionId, + externalId: repo.external_id, + durationMs: upsertDuration, + }); } const totalUpsertDuration = performance.now() - totalUpsertStart; - logger.debug(`Upserted ${repoData.length} repos for connection ${connection.name} (id: ${connectionId}) in ${totalUpsertDuration}ms`); + logger.info(`Stored ${repoData.length} repositories`, { + connectionId, + connectionName: connection.name, + repositoryCount: repoData.length, + durationMs: totalUpsertDuration, + }); }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); await prisma.connection.update({ @@ -133,50 +123,46 @@ export const connectionWorkload: Workload<'connection'> = { contexts: config.contexts, }); } catch (err) { - logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`); + logger.error(`Failed to sync search contexts for connection ${connectionId}`, err); Sentry.captureException(err); } - }, - onStarted: async ({ data, jobId }) => { - await prisma.connectionSyncJob.createMany({ - data: [{ - id: jobId, - connectionId: data.connectionId, - status: ConnectionSyncJobStatus.IN_PROGRESS, - warningMessages: [], - }], - skipDuplicates: true, - }); - await prisma.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: ConnectionSyncJobStatus.IN_PROGRESS, - }, - }); - }, - onCompleted: async ({ jobId }) => { - await prisma.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: ConnectionSyncJobStatus.COMPLETED, - completedAt: new Date(), - }, - }); - }, - onTerminalFailure: async ({ jobId }, error) => { - await prisma.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: ConnectionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, - }, + + logger.info(`Connection ${connectionId} sync finished`, { + connectionId, }); } } + +const discoverConnectionRepositories = async ({ + config, + connectionId, + signal, +}: { + config: ConnectionConfig; + connectionId: number; + signal: AbortSignal; +}) => { + switch (config.type) { + case 'github': { + return compileGithubConfig(config, connectionId, signal); + } + case 'gitlab': { + return compileGitlabConfig(config, connectionId); + } + case 'gitea': { + return compileGiteaConfig(config, connectionId); + } + case 'gerrit': { + return compileGerritConfig(config, connectionId); + } + case 'bitbucket': { + return compileBitbucketConfig(config, connectionId); + } + case 'azuredevops': { + return compileAzureDevOpsConfig(config, connectionId); + } + case 'git': { + return compileGenericGitHostConfig(config, connectionId); + } + } +}; \ No newline at end of file diff --git a/packages/backend/src/ee/accountPermissionSyncer.ts b/packages/backend/src/ee/accountPermissionSyncer.ts index 88f3e57b5..14facb324 100644 --- a/packages/backend/src/ee/accountPermissionSyncer.ts +++ b/packages/backend/src/ee/accountPermissionSyncer.ts @@ -443,4 +443,4 @@ export class AccountPermissionSyncer { logger.error(errorMessage('unknown account (id not found)', 'unknown user (id not found)')); } } -} \ No newline at end of file +} diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index 31e935b1f..cd49b5c76 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -1,9 +1,10 @@ import * as Sentry from "@sentry/node"; -import { BullMQJobProducer, createLogger, DataOf, JobLifecycleContext, QueueName } from "@sourcebot/shared"; +import { BullMQClient, createBullMQJobLogger, createLogger, DataOf, JobLifecycleContext, QueueName } from "@sourcebot/shared"; import { Job, Worker } from "bullmq"; import { Redis } from "ioredis"; import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; import { JobDetail, JobManager, Schedule, Workload } from "./types.js"; +import { prisma } from "./prisma.js"; const LOG_TAG = 'job-manager'; const logger = createLogger(LOG_TAG); @@ -47,11 +48,11 @@ const scheduleToRepeat = (schedule: Schedule) => export class BullMQJobManager implements JobManager { private readonly workloads = new Map>(); private readonly workers = new Map(); - private readonly producer: BullMQJobProducer; + private readonly bullmqClient: BullMQClient; private readonly abortController = new AbortController(); constructor(private readonly connection: Redis) { - this.producer = new BullMQJobProducer(connection); + this.bullmqClient = new BullMQClient(connection, prisma); } register(workload: Workload): void { @@ -85,7 +86,7 @@ export class BullMQJobManager implements JobManager { if (!workload) { throw new Error(`Cannot trigger unknown workload "${workloadName}"`); } - return this.producer.enqueue(workload.queueSpec, data); + return this.bullmqClient.enqueue(workload.queueSpec, data); } async stop(): Promise { @@ -98,7 +99,7 @@ export class BullMQJobManager implements JobManager { ]), )); - await this.producer.close(); + await this.bullmqClient.close(); logger.info('Job manager stopped'); } @@ -106,21 +107,35 @@ export class BullMQJobManager implements JobManager { private async startWorkload(workload: Workload): Promise { const { queueSpec: spec, concurrency, rateLimit, schedule } = workload; - const queue = this.producer.queue(spec.name); + const queue = this.bullmqClient.getQueue(spec); const worker = new Worker( spec.name, async (job) => { + const jobLogger = createBullMQJobLogger( + job, + `${LOG_TAG}:${spec.name}:job:${job.id ?? 'unknown'}`, + ); const lifecycleContext = this.jobLifecycleContext(job); - await workload.onStarted?.(lifecycleContext); - - return workload.process({ - ...lifecycleContext, - signal: this.abortController.signal, - log: async (message) => { await job.log(message); }, - updateProgress: (progress) => job.updateProgress(progress), - trigger: (target, data) => this.trigger(target, data), - }); + jobLogger.info(`Started workload "${spec.name}"`); + + try { + await workload.onStarted?.(lifecycleContext); + const result = await workload.process({ + ...lifecycleContext, + signal: this.abortController.signal, + logger: jobLogger, + updateProgress: (progress) => job.updateProgress(progress), + trigger: (target, data) => this.trigger(target, data), + }); + jobLogger.info(`Completed workload "${spec.name}"`); + return result; + } catch (error) { + jobLogger.error(`Workload "${spec.name}" attempt failed`, error); + throw error; + } finally { + await jobLogger.flush(); + } }, { connection: this.connection, @@ -159,6 +174,7 @@ export class BullMQJobManager implements JobManager { attempts: spec.jobOptions.attempts, removeOnComplete: { count: spec.jobOptions.keep.completed }, removeOnFail: { count: spec.jobOptions.keep.failed }, + keepLogs: spec.jobOptions.keepLogs, }, }, ); @@ -208,6 +224,7 @@ export class BullMQJobManager implements JobManager { jobId: job.id ?? '', attemptsMade: job.attemptsMade, maxAttempts: job.opts.attempts ?? 1, + prisma, }; } } diff --git a/packages/backend/src/jobManagerLifecycle.test.ts b/packages/backend/src/jobManagerLifecycle.test.ts index 5117233b3..86a1120f4 100644 --- a/packages/backend/src/jobManagerLifecycle.test.ts +++ b/packages/backend/src/jobManagerLifecycle.test.ts @@ -1,11 +1,18 @@ import { Redis } from 'ioredis'; import { beforeEach, describe, expect, test, vi } from 'vitest'; -import { Workload } from './types.js'; +import { ProcessContext, Workload } from './types.js'; const mocks = vi.hoisted(() => ({ enqueue: vi.fn(), producerClose: vi.fn(), workerClose: vi.fn(), + jobLogger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + flush: vi.fn(), + }, workers: [] as Array<{ processor: (job: unknown) => Promise; handlers: Map void>; @@ -23,7 +30,8 @@ vi.mock('@sourcebot/shared', () => ({ error: vi.fn(), debug: vi.fn(), })), - BullMQJobProducer: class { + createBullMQJobLogger: vi.fn(() => mocks.jobLogger), + BullMQClient: class { enqueue = mocks.enqueue; close = mocks.producerClose; queue = vi.fn(() => ({ @@ -66,6 +74,7 @@ const createWorkload = ( attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, keep: { completed: 50, failed: 50 }, + keepLogs: 500, }, }, concurrency: 2, @@ -90,7 +99,7 @@ describe('BullMQJobManager lifecycle', () => { mocks.enqueue.mockResolvedValue('job-1'); }); - test('delegates enqueueing to BullMQJobProducer and returns its job id', async () => { + test('delegates enqueueing to BullMQClient and returns its job id', async () => { const manager = new BullMQJobManager({} as Redis); const workload = createWorkload(); manager.register(workload); @@ -124,6 +133,25 @@ describe('BullMQJobManager lifecycle', () => { expect.objectContaining({ data, jobId: 'job-1', maxAttempts: 2 }), { repoCount: 3 }, ); + expect(mocks.jobLogger.flush).toHaveBeenCalled(); + }); + + test('provides the structured job logger to the workload processor', async () => { + const process = vi.fn(async (context: ProcessContext<'connection'>) => { + context.logger.info('Processing connection'); + return { repoCount: 3 }; + }); + const manager = new BullMQJobManager({} as Redis); + manager.register(createWorkload({ process })); + await manager.start(); + + await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); + + expect(process).toHaveBeenCalledWith(expect.objectContaining({ + logger: mocks.jobLogger, + })); + expect(mocks.jobLogger.info).toHaveBeenCalledWith('Processing connection'); + expect(mocks.jobLogger.flush).toHaveBeenCalled(); }); test('reports lifecycle metadata after terminal failure', async () => { @@ -137,12 +165,12 @@ describe('BullMQJobManager lifecycle', () => { mocks.workers[0].handlers.get('failed')?.(job, error); await vi.waitFor(() => { - expect(onTerminalFailure).toHaveBeenCalledWith({ + expect(onTerminalFailure).toHaveBeenCalledWith(expect.objectContaining({ data, jobId: 'job-1', attemptsMade: 2, maxAttempts: 2, - }, error); + }), error); }); }); }); diff --git a/packages/backend/src/reconciliationWorkload.test.ts b/packages/backend/src/reconciliationWorkload.test.ts index 61db94afb..3aea21082 100644 --- a/packages/backend/src/reconciliationWorkload.test.ts +++ b/packages/backend/src/reconciliationWorkload.test.ts @@ -8,6 +8,7 @@ vi.mock('@sourcebot/shared', () => ({ attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, keep: { completed: 50, failed: 50 }, + keepLogs: 500, }, }, createLogger: vi.fn(() => ({ @@ -67,7 +68,13 @@ describe('reconciliationWorkload', () => { attemptsMade: 0, maxAttempts: 2, signal: new AbortController().signal, - log: vi.fn(), + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), + }, updateProgress: vi.fn(), trigger, }); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts index 500389b2e..73209b26c 100644 --- a/packages/backend/src/reconciliationWorkload.ts +++ b/packages/backend/src/reconciliationWorkload.ts @@ -1,9 +1,7 @@ import { PrismaClient } from "@sourcebot/db"; -import { createLogger, RECONCILIATION_QUEUE } from "@sourcebot/shared"; +import { RECONCILIATION_QUEUE } from "@sourcebot/shared"; import { Settings, Workload } from "./types.js"; -const logger = createLogger('reconciliation-workload'); - interface ReconciliationWorkloadDependencies { db: PrismaClient; settings: Pick; @@ -16,7 +14,7 @@ export const createReconciliationWorkload = ({ concurrency: 1, schedule: { every: '15m' }, queueSpec: RECONCILIATION_QUEUE, - process: async ({ trigger }) => { + process: async ({ logger, trigger }) => { const thresholdDate = new Date(Date.now() - settings.resyncConnectionIntervalMs); const connections = await db.connection.findMany({ where: { diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index ae6b7fc7e..848d9f48f 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -1,7 +1,7 @@ import { Connection, Repo, RepoToConnection } from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type"; -import { DataOf, JobLifecycleContext, QueueName, QueueSpec } from "@sourcebot/shared"; +import { DataOf, JobLifecycleContext, JobLogger, QueueName, QueueSpec } from "@sourcebot/shared"; export type Settings = Required; @@ -26,7 +26,7 @@ export type RepoAuthCredentials = { export interface ProcessContext extends JobLifecycleContext { signal: AbortSignal; - log(message: string): Promise; + logger: JobLogger; updateProgress(progress: number | object): Promise; trigger(workload: T, data: DataOf): Promise; } diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index d99727ea3..e50ee0ed5 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -284,4 +284,4 @@ export const setIntervalAsync = (target: () => Promise, pollingIntervalMs: setIntervalWithPromise(target), pollingIntervalMs ); -} \ No newline at end of file +} diff --git a/packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql b/packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql new file mode 100644 index 000000000..c4315cee2 --- /dev/null +++ b/packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql @@ -0,0 +1,20 @@ +/* + Warnings: + + - You are about to drop the `ConnectionSyncJob` table. If the table is not empty, all the data it contains will be lost. + +*/ +-- DropForeignKey +ALTER TABLE "ConnectionSyncJob" DROP CONSTRAINT "ConnectionSyncJob_connectionId_fkey"; + +-- AlterTable +ALTER TABLE "Connection" ADD COLUMN "latestSyncJobId" TEXT; + +-- DropTable +DROP TABLE "ConnectionSyncJob"; + +-- DropEnum +DROP TYPE "ConnectionSyncJobStatus"; + +-- DropEnum +DROP TYPE "ConnectionSyncStatus"; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 0f72c97d0..336a306e9 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -10,15 +10,6 @@ datasource db { url = env("DATABASE_URL") } -enum ConnectionSyncStatus { - SYNC_NEEDED - IN_SYNC_QUEUE - SYNCING - SYNCED - SYNCED_WITH_WARNINGS - FAILED -} - enum ChatVisibility { PRIVATE PUBLIC @@ -181,9 +172,9 @@ model Connection { // The type of connection (e.g., github, gitlab, etc.) connectionType ConnectionType - syncJobs ConnectionSyncJob[] /// When the connection was last synced successfully. syncedAt DateTime? + latestSyncJobId String? /// Controls whether repository permissions are enforced for this connection. /// When `PERMISSION_SYNC_ENABLED` is false, this setting has no effect. @@ -208,27 +199,6 @@ model Connection { @@unique([name, orgId]) } -enum ConnectionSyncJobStatus { - PENDING - IN_PROGRESS - COMPLETED - FAILED -} - -model ConnectionSyncJob { - id String @id @default(cuid()) - status ConnectionSyncJobStatus @default(PENDING) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - completedAt DateTime? - - warningMessages String[] - errorMessage String? - - connection Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade) - connectionId Int -} - model RepoToConnection { addedAt DateTime @default(now()) diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts new file mode 100644 index 000000000..ee165d5b8 --- /dev/null +++ b/packages/shared/src/bullmqClient.test.ts @@ -0,0 +1,189 @@ +import type { Redis } from 'ioredis'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import type { QueueSpec } from './queue.js'; +import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from './jobLogger.js'; + +const queueMocks = vi.hoisted(() => ({ + add: vi.fn(), + close: vi.fn(), + getJob: vi.fn(), + getJobLogs: vi.fn(), +})); + +vi.mock('@sentry/node', () => ({ + captureException: vi.fn(), +})); + +vi.mock('./logger.js', () => ({ + createLogger: vi.fn(() => ({ + error: vi.fn(), + })), +})); + +vi.mock('bullmq', () => ({ + Queue: class { + constructor() { + return queueMocks; + } + }, +})); + +import { BullMQClient } from './bullmqClient.js'; +import { PrismaClient } from '@sourcebot/db'; + +const connectionSpec: QueueSpec<'connection'> = { + name: 'connection', + dedupKey: ({ connectionId }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }, +}; + +const data = { connectionId: 42, orgId: 1 }; + +describe('BullMQClient', () => { + const redis = {} as Redis; + const prisma = {} as PrismaClient; + + beforeEach(() => { + vi.clearAllMocks(); + queueMocks.add.mockImplementation(async (_name, _data, options) => ({ id: options.jobId })); + }); + + test('returns the job id when BullMQ accepts the proposed id', async () => { + const client = new BullMQClient(redis, prisma); + + const result = await client.enqueue(connectionSpec, data); + + expect(result).toEqual(expect.any(String)); + expect(queueMocks.add).toHaveBeenCalledWith( + 'connection', + data, + expect.objectContaining({ + jobId: result, + deduplication: { id: 'connection:42' }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }), + ); + }); + + test('calls onEnqueued when a new job is created', async () => { + const onEnqueued = vi.fn(); + const client = new BullMQClient(redis, prisma); + + const result = await client.enqueue({ ...connectionSpec, onEnqueued }, data); + + expect(onEnqueued).toHaveBeenCalledWith({ + data, + jobId: result, + attemptsMade: 0, + maxAttempts: 2, + prisma, + }); + }); + + test('returns the existing job id without calling onEnqueued when BullMQ deduplicates the enqueue', async () => { + const onEnqueued = vi.fn(); + queueMocks.add.mockResolvedValue({ id: 'existing-job' }); + const client = new BullMQClient(redis, prisma); + + const result = await client.enqueue({ ...connectionSpec, onEnqueued }, data); + + expect(result).toBe('existing-job'); + expect(onEnqueued).not.toHaveBeenCalled(); + }); + + test.each([ + ['waiting', 'PENDING'], + ['waiting-children', 'PENDING'], + ['delayed', 'PENDING'], + ['prioritized', 'PENDING'], + ['paused', 'PENDING'], + ['active', 'IN_PROGRESS'], + ['completed', 'COMPLETED'], + ])('maps BullMQ state %s to %s', async (state, expectedStatus) => { + queueMocks.getJob.mockResolvedValue({ + id: 'job-1', + data, + getState: vi.fn().mockResolvedValue(state), + }); + const client = new BullMQClient(redis, prisma); + + await expect(client.getJob(connectionSpec, 'job-1')).resolves.toEqual({ + id: 'job-1', + data, + status: expectedStatus, + errorMessage: null, + }); + }); + + test('returns the failure reason for a failed job', async () => { + queueMocks.getJob.mockResolvedValue({ + id: 'job-1', + data, + failedReason: 'Connection credentials expired', + getState: vi.fn().mockResolvedValue('failed'), + }); + const client = new BullMQClient(redis, prisma); + + await expect(client.getJob(connectionSpec, 'job-1')).resolves.toEqual({ + id: 'job-1', + data, + status: 'FAILED', + errorMessage: 'Connection credentials expired', + }); + }); + + test.each([ + ['missing job', undefined], + ['unknown state', { + id: 'job-1', + data, + getState: vi.fn().mockResolvedValue('unknown'), + }], + ])('returns null for a %s', async (_label, job) => { + queueMocks.getJob.mockResolvedValue(job); + const client = new BullMQClient(redis, prisma); + + await expect(client.getJob(connectionSpec, 'job-1')).resolves.toBeNull(); + }); + + test('reads and parses incremental job logs', async () => { + queueMocks.getJobLogs.mockResolvedValue({ + logs: [ + JSON.stringify({ + version: 1, + timestamp: '2026-07-28T12:00:00.000Z', + level: 'warn', + message: 'Repository skipped', + attempt: 1, + }), + ], + count: 4, + }); + const client = new BullMQClient(redis, prisma); + + await expect(client.getJobLogs(connectionSpec, 'job-1', { + start: 3, + ascending: true, + })).resolves.toEqual({ + logs: [{ + version: 1, + timestamp: '2026-07-28T12:00:00.000Z', + level: 'warn', + message: 'Repository skipped', + attempt: 1, + }], + count: 4, + }); + expect(queueMocks.getJobLogs).toHaveBeenCalledWith( + 'job-1', + 3, + undefined, + true, + ); + }); +}); diff --git a/packages/shared/src/bullmqClient.ts b/packages/shared/src/bullmqClient.ts new file mode 100644 index 000000000..52a6bff54 --- /dev/null +++ b/packages/shared/src/bullmqClient.ts @@ -0,0 +1,142 @@ +import * as Sentry from "@sentry/node"; +import { Queue } from "bullmq"; +import { randomUUID } from "crypto"; +import { Redis } from "ioredis"; +import { createLogger } from "./logger.js"; +import { DataOf, QueueName, QueueSpec } from "./queue.js"; +import { PrismaClient } from "@sourcebot/db"; +import { readBullMQJobLogs } from "./jobLogger.js"; +import type { GetJobLogsOptions, JobLogs } from "./jobLogger.js"; + +const logger = createLogger('job-producer'); + +export type WorkloadJobStatus = "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED"; + +export interface WorkloadJob { + id: string; + data: DataOf; + status: WorkloadJobStatus; + errorMessage: string | null; +} + +type WorkloadQueue = Queue< + DataOf, + unknown, + string, + DataOf, + unknown, + string +>; + +const normalizeJobState = (state: string): WorkloadJobStatus | null => { + switch (state) { + case "waiting": + case "waiting-children": + case "delayed": + case "prioritized": + case "paused": + return "PENDING"; + case "active": + return "IN_PROGRESS"; + case "completed": + return "COMPLETED"; + case "failed": + return "FAILED"; + default: + return null; + } +}; + +export class BullMQClient { + private readonly queues = new Map(); + + constructor( + private readonly connection: Redis, + private readonly prisma: PrismaClient, + ) {} + + getQueue(spec: QueueSpec): WorkloadQueue { + const queueName = spec.name; + let queue = this.queues.get(queueName); + if (!queue) { + queue = new Queue(queueName, { connection: this.connection }); + this.queues.set(queueName, queue); + } + return queue as WorkloadQueue; + } + + async getJob( + spec: QueueSpec, + jobId: string, + ): Promise | null> { + const job = await this.getQueue(spec).getJob(jobId); + if (!job) { + return null; + } + + const status = normalizeJobState(await job.getState()); + if (!status) { + return null; + } + + return { + id: job.id ?? jobId, + data: job.data as DataOf, + status, + errorMessage: status === "FAILED" ? job.failedReason || null : null, + }; + } + + async getJobLogs( + spec: QueueSpec, + jobId: string, + options: GetJobLogsOptions = {}, + ): Promise { + return readBullMQJobLogs(this.getQueue(spec), jobId, options); + } + + async enqueue( + spec: QueueSpec, + data: DataOf + ): Promise { + const dedupKey = spec.dedupKey?.(data); + const queue = this.getQueue(spec); + + const requestedJobId = randomUUID(); + const job = await queue.add(spec.name, data, { + jobId: requestedJobId, + ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), + attempts: spec.jobOptions.attempts, + backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, + removeOnComplete: { count: spec.jobOptions.keep.completed }, + removeOnFail: { count: spec.jobOptions.keep.failed }, + keepLogs: spec.jobOptions.keepLogs, + }); + + if (!job.id) { + throw new Error(`BullMQ did not return an id for workload "${spec.name}"`); + } + + const isEnqueued = job.id === requestedJobId; + if (isEnqueued && spec.onEnqueued) { + try { + await spec.onEnqueued({ + data, + jobId: job.id, + attemptsMade: 0, + maxAttempts: spec.jobOptions.attempts, + prisma: this.prisma, + }); + } catch (error) { + Sentry.captureException(error); + logger.error(`onEnqueued for workload "${spec.name}" threw:`, error); + } + } + + return job.id; + } + + async close(): Promise { + await Promise.all([...this.queues.values()].map((queue) => queue.close())); + } +} diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 753240ab5..51d4c9077 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -103,5 +103,24 @@ export { RECONCILIATION_QUEUE, } from "./queue.js"; export { - BullMQJobProducer, -} from "./jobProducer.js"; + BullMQClient, +} from "./bullmqClient.js"; +export type { + WorkloadJob, + WorkloadJobStatus, +} from "./bullmqClient.js"; +export { + createBullMQJobLogger, + DEFAULT_JOB_LOGS_MAX_ENTRIES, + parseJobLogEntry, + readBullMQJobLogs, +} from "./jobLogger.js"; +export type { + GetJobLogsOptions, + JobLogEntry, + JobLogFields, + JobLogLevel, + JobLogger, + JobLogs, + JobLogSink, +} from "./jobLogger.js"; diff --git a/packages/shared/src/jobLogger.test.ts b/packages/shared/src/jobLogger.test.ts new file mode 100644 index 000000000..c6435ea00 --- /dev/null +++ b/packages/shared/src/jobLogger.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + applicationLog: vi.fn(), + applicationError: vi.fn(), +})); + +vi.mock("./logger.js", () => ({ + createLogger: vi.fn(() => ({ + log: mocks.applicationLog, + error: mocks.applicationError, + })), +})); + +import { + createBullMQJobLogger, + parseJobLogEntry, + readBullMQJobLogs, +} from "./jobLogger.js"; + +describe("createBullMQJobLogger", () => { + test("writes structured, redacted entries to BullMQ and the application logger", async () => { + const log = vi.fn().mockResolvedValue(1); + const logger = createBullMQJobLogger({ + id: "job-1", + name: "connection", + queueName: "connection", + attemptsMade: 1, + log, + }); + + logger.warn("Some repositories were skipped", { + skipped: 2, + accessToken: "do-not-store", + }); + await logger.flush(); + + expect(mocks.applicationLog).toHaveBeenCalledWith( + "warn", + "Some repositories were skipped", + { + skipped: 2, + accessToken: "[REDACTED]", + }, + ); + + const storedEntry = JSON.parse(log.mock.calls[0][0]); + expect(storedEntry).toMatchObject({ + version: 1, + level: "warn", + message: "Some repositories were skipped", + attempt: 2, + fields: { + skipped: 2, + accessToken: "[REDACTED]", + }, + }); + expect(storedEntry.timestamp).toEqual(expect.any(String)); + }); + + test("does not fail the workload when persisting a log entry fails", async () => { + const logger = createBullMQJobLogger({ + id: "job-1", + name: "connection", + queueName: "connection", + attemptsMade: 0, + log: vi.fn().mockRejectedValue(new Error("Redis unavailable")), + }); + + logger.info("Starting"); + await expect(logger.flush()).resolves.toBeUndefined(); + expect(mocks.applicationError).toHaveBeenCalled(); + }); +}); + +describe("readBullMQJobLogs", () => { + test("parses structured entries and preserves legacy string logs", async () => { + const structuredEntry = JSON.stringify({ + version: 1, + timestamp: "2026-07-28T03:00:00.000Z", + level: "info", + message: "Started", + attempt: 1, + }); + const queue = { + getJobLogs: vi.fn().mockResolvedValue({ + logs: [structuredEntry, "legacy log"], + count: 2, + }), + }; + + const result = await readBullMQJobLogs(queue, "job-1", { + start: 10, + end: 20, + ascending: true, + }); + + expect(queue.getJobLogs).toHaveBeenCalledWith("job-1", 10, 20, true); + expect(result).toEqual({ + logs: [ + parseJobLogEntry(structuredEntry), + { + version: 0, + timestamp: null, + level: "info", + message: "legacy log", + attempt: null, + }, + ], + count: 2, + }); + }); +}); diff --git a/packages/shared/src/jobLogger.ts b/packages/shared/src/jobLogger.ts new file mode 100644 index 000000000..57df9311b --- /dev/null +++ b/packages/shared/src/jobLogger.ts @@ -0,0 +1,207 @@ +import type { Job, Queue } from "bullmq"; +import { createLogger } from "./logger.js"; + +export const DEFAULT_JOB_LOGS_MAX_ENTRIES = 500; + +export type JobLogLevel = "debug" | "info" | "warn" | "error"; +export type JobLogFields = Record; + +export interface JobLogEntry { + version: 1 | 0; + timestamp: string | null; + level: JobLogLevel; + message: string; + attempt: number | null; + fields?: JobLogFields; +} + +export interface JobLogSink { + debug(message: string, fields?: unknown): void; + info(message: string, fields?: unknown): void; + warn(message: string, fields?: unknown): void; + error(message: string, fields?: unknown): void; +} + +export interface JobLogger extends JobLogSink { + flush(): Promise; +} + +export interface GetJobLogsOptions { + start?: number; + end?: number; + ascending?: boolean; +} + +export interface JobLogs { + logs: JobLogEntry[]; + count: number; +} + +type BullMQLogJob = Pick; +type JobLogQueue = Pick; + +const JOB_LOG_LEVELS = new Set(["debug", "info", "warn", "error"]); +const SENSITIVE_FIELD_NAME = /authorization|cookie|credential|password|private.?key|secret|token/i; +const MAX_FIELD_DEPTH = 6; + +const sanitizeValue = ( + value: unknown, + seen: WeakSet, + depth: number, +): unknown => { + if (depth > MAX_FIELD_DEPTH) { + return "[Max depth reached]"; + } + if ( + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + return value; + } + if (typeof value === "bigint") { + return value.toString(); + } + if (typeof value === "undefined") { + return "[undefined]"; + } + if (typeof value === "symbol" || typeof value === "function") { + return String(value); + } + if (value instanceof Date) { + return value.toISOString(); + } + if (value instanceof Error) { + return { + name: value.name, + message: value.message, + stack: value.stack, + }; + } + if (seen.has(value)) { + return "[Circular]"; + } + + seen.add(value); + if (Array.isArray(value)) { + return value.map((item) => sanitizeValue(item, seen, depth + 1)); + } + + return Object.fromEntries( + Object.entries(value).map(([key, nestedValue]) => [ + key, + SENSITIVE_FIELD_NAME.test(key) + ? "[REDACTED]" + : sanitizeValue(nestedValue, seen, depth + 1), + ]), + ); +}; + +const sanitizeFields = (fields: unknown): JobLogFields | undefined => { + if (fields === undefined) { + return undefined; + } + + const sanitized = sanitizeValue(fields, new WeakSet(), 0); + if (sanitized !== null && typeof sanitized === "object" && !Array.isArray(sanitized)) { + return sanitized as JobLogFields; + } + return { value: sanitized }; +}; + +export const parseJobLogEntry = (rawLog: string): JobLogEntry => { + try { + const parsed = JSON.parse(rawLog) as Partial; + if ( + parsed.version === 1 && + typeof parsed.timestamp === "string" && + typeof parsed.level === "string" && + JOB_LOG_LEVELS.has(parsed.level as JobLogLevel) && + typeof parsed.message === "string" && + typeof parsed.attempt === "number" + ) { + return { + version: 1, + timestamp: parsed.timestamp, + level: parsed.level as JobLogLevel, + message: parsed.message, + attempt: parsed.attempt, + ...(parsed.fields ? { fields: parsed.fields } : {}), + }; + } + } catch { + // Older BullMQ logs were stored as plain strings. + } + + return { + version: 0, + timestamp: null, + level: "info", + message: rawLog, + attempt: null, + }; +}; + +export const readBullMQJobLogs = async ( + queue: JobLogQueue, + jobId: string, + options: GetJobLogsOptions = {}, +): Promise => { + const result = await queue.getJobLogs( + jobId, + options.start, + options.end, + options.ascending, + ); + + return { + logs: result.logs.map(parseJobLogEntry), + count: result.count, + }; +}; + +export const createBullMQJobLogger = ( + job: BullMQLogJob, + label = `${job.queueName}:job:${job.id ?? "unknown"}`, +): JobLogger => { + const applicationLogger = createLogger(label); + const pendingWrites = new Set>(); + + const write = (level: JobLogLevel, message: string, rawFields?: unknown): void => { + const fields = sanitizeFields(rawFields); + applicationLogger.log(level, message, fields); + + const entry: JobLogEntry = { + version: 1, + timestamp: new Date().toISOString(), + level, + message, + attempt: job.attemptsMade + 1, + ...(fields ? { fields } : {}), + }; + const pendingWrite = job.log(JSON.stringify(entry)) + .then(() => undefined) + .catch((error: unknown) => { + applicationLogger.error( + `Failed to persist a BullMQ log entry for job ${job.id ?? "unknown"}`, + error, + ); + }); + + pendingWrites.add(pendingWrite); + void pendingWrite.finally(() => { + pendingWrites.delete(pendingWrite); + }); + }; + + return { + debug: (message, fields) => write("debug", message, fields), + info: (message, fields) => write("info", message, fields), + warn: (message, fields) => write("warn", message, fields), + error: (message, fields) => write("error", message, fields), + flush: async () => { + await Promise.all([...pendingWrites]); + }, + }; +}; diff --git a/packages/shared/src/jobProducer.test.ts b/packages/shared/src/jobProducer.test.ts deleted file mode 100644 index ffdad845b..000000000 --- a/packages/shared/src/jobProducer.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import type { Redis } from 'ioredis'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; -import type { QueueSpec } from './queue.js'; - -const queueMocks = vi.hoisted(() => ({ - add: vi.fn(), - close: vi.fn(), -})); - -vi.mock('@sentry/node', () => ({ - captureException: vi.fn(), -})); - -vi.mock('./logger.js', () => ({ - createLogger: vi.fn(() => ({ - error: vi.fn(), - })), -})); - -vi.mock('bullmq', () => ({ - Queue: class { - constructor() { - return queueMocks; - } - }, -})); - -import { BullMQJobProducer } from './jobProducer.js'; - -const connectionSpec: QueueSpec<'connection'> = { - name: 'connection', - dedupKey: ({ connectionId }) => `connection:${connectionId}`, - jobOptions: { - attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - }, -}; - -const data = { connectionId: 42, orgId: 1 }; - -describe('BullMQJobProducer', () => { - const redis = {} as Redis; - - beforeEach(() => { - vi.clearAllMocks(); - queueMocks.add.mockImplementation(async (_name, _data, options) => ({ id: options.jobId })); - }); - - test('returns the job id when BullMQ accepts the proposed id', async () => { - const producer = new BullMQJobProducer(redis); - - const result = await producer.enqueue(connectionSpec, data); - - expect(result).toEqual(expect.any(String)); - expect(queueMocks.add).toHaveBeenCalledWith( - 'connection', - data, - expect.objectContaining({ - jobId: result, - deduplication: { id: 'connection:42' }, - }), - ); - }); - - test('calls onEnqueued when a new job is created', async () => { - const onEnqueued = vi.fn(); - const producer = new BullMQJobProducer(redis); - - const result = await producer.enqueue({ ...connectionSpec, onEnqueued }, data); - - expect(onEnqueued).toHaveBeenCalledWith({ - data, - jobId: result, - attemptsMade: 0, - maxAttempts: 2, - }); - }); - - test('returns the existing job id without calling onEnqueued when BullMQ deduplicates the enqueue', async () => { - const onEnqueued = vi.fn(); - queueMocks.add.mockResolvedValue({ id: 'existing-job' }); - const producer = new BullMQJobProducer(redis); - - const result = await producer.enqueue({ ...connectionSpec, onEnqueued }, data); - - expect(result).toBe('existing-job'); - expect(onEnqueued).not.toHaveBeenCalled(); - }); -}); diff --git a/packages/shared/src/jobProducer.ts b/packages/shared/src/jobProducer.ts deleted file mode 100644 index 595a46787..000000000 --- a/packages/shared/src/jobProducer.ts +++ /dev/null @@ -1,66 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { Queue } from "bullmq"; -import { randomUUID } from "crypto"; -import { Redis } from "ioredis"; -import { createLogger } from "./logger.js"; -import { DataOf, QueueName, QueueSpec } from "./queue.js"; - -const logger = createLogger('job-producer'); - -export class BullMQJobProducer { - private readonly queues = new Map(); - - constructor(private readonly connection: Redis) {} - - queue(name: string): Queue { - let queue = this.queues.get(name); - if (!queue) { - queue = new Queue(name, { connection: this.connection }); - this.queues.set(name, queue); - } - return queue; - } - - async enqueue( - spec: QueueSpec, - data: DataOf - ): Promise { - const dedupKey = spec.dedupKey?.(data); - const queue = this.queue(spec.name); - - const requestedJobId = randomUUID(); - const job = await queue.add(spec.name, data, { - jobId: requestedJobId, - ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), - attempts: spec.jobOptions.attempts, - backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, - removeOnComplete: { count: spec.jobOptions.keep.completed }, - removeOnFail: { count: spec.jobOptions.keep.failed }, - }); - - if (!job.id) { - throw new Error(`BullMQ did not return an id for workload "${spec.name}"`); - } - - const isEnqueued = job.id === requestedJobId; - if (isEnqueued && spec.onEnqueued) { - try { - await spec.onEnqueued({ - data, - jobId: job.id, - attemptsMade: 0, - maxAttempts: spec.jobOptions.attempts, - }); - } catch (error) { - Sentry.captureException(error); - logger.error(`onEnqueued for workload "${spec.name}" threw:`, error); - } - } - - return job.id; - } - - async close(): Promise { - await Promise.all([...this.queues.values()].map((queue) => queue.close())); - } -} diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index ef9dd67d5..fb375d711 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -1,4 +1,7 @@ +import { PrismaClient } from "@sourcebot/db"; +import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from "./jobLogger.js"; + export type QueueName = keyof QueueRegistry; export type DataOf = QueueRegistry[TName]; type EmptyJobData = Record; @@ -16,7 +19,22 @@ export const CONNECTION_QUEUE: QueueSpec<'connection'> = { jobOptions: { attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 } + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }, + onEnqueued: async ({ + prisma, + data: { connectionId }, + jobId + }) => { + await prisma.connection.update({ + where: { + id: connectionId + }, + data: { + latestSyncJobId: jobId + } + }); }, dedupKey: (data) => `connection:${data.connectionId}`, } @@ -28,6 +46,7 @@ export const RECONCILIATION_QUEUE: QueueSpec<'reconciliation'> = { attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, }; @@ -38,6 +57,7 @@ export interface QueueSpec { attempts: number; backoff: { type: 'fixed' | 'exponential'; delayMs: number }; keep: { completed: number; failed: number }; + keepLogs: number; }; onEnqueued?(ctx: JobLifecycleContext): Promise; } @@ -47,4 +67,5 @@ export interface JobLifecycleContext { jobId: string; attemptsMade: number; maxAttempts: number; + prisma: PrismaClient; } diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index c64979173..5db1dbdfc 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -4,7 +4,7 @@ import { createAudit } from "@/ee/features/audit/audit"; import { ErrorCode } from "@/lib/errorCodes"; import { notFound, ServiceError } from "@/lib/serviceError"; import { sew } from "@/middleware/sew"; -import { ConnectionSyncJobStatus, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; +import { OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; @@ -274,42 +274,6 @@ export const getReposStats = async () => sew(() => }) ) -export const getConnectionStats = async () => sew(() => - withAuth(async ({ org, prisma }) => { - const [ - numberOfConnections, - numberOfConnectionsWithFirstTimeSyncJobsInProgress, - ] = await Promise.all([ - prisma.connection.count({ - where: { - orgId: org.id, - } - }), - prisma.connection.count({ - where: { - orgId: org.id, - syncedAt: null, - syncJobs: { - some: { - status: { - in: [ - ConnectionSyncJobStatus.PENDING, - ConnectionSyncJobStatus.IN_PROGRESS, - ] - } - } - } - } - }) - ]); - - return { - numberOfConnections, - numberOfConnectionsWithFirstTimeSyncJobsInProgress, - }; - }) -); - export const getRepoInfoByName = async (repoName: string) => sew(() => withOptionalAuth(async ({ org, prisma }) => { // @note: repo names are represented by their remote url diff --git a/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx b/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx index 04ff3caf6..62a6c0256 100644 --- a/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx +++ b/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx @@ -2,7 +2,6 @@ import { cookies } from "next/headers"; import { auth } from "@/auth"; import { HOME_VIEW_COOKIE_NAME } from "@/lib/constants"; import { HomeView } from "@/hooks/useHomeView"; -import { getConnectionStats } from "@/actions"; import { getOrgAccountRequests } from "@/features/membership/actions"; import { isServiceError } from "@/lib/utils"; import { ServiceErrorException } from "@/lib/serviceError"; @@ -45,11 +44,9 @@ export async function DefaultSidebar() { if (!isOwner) { return false; } - const connectionStats = await getConnectionStats(); const joinRequests = await getOrgAccountRequests(); - const hasConnectionNotification = !isServiceError(connectionStats) && connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0; const hasJoinRequestNotification = !isServiceError(joinRequests) && joinRequests.length > 0; - return hasConnectionNotification || hasJoinRequestNotification; + return hasJoinRequestNotification; })(); return ( diff --git a/packages/web/src/app/(app)/settings/connections/[id]/page.tsx b/packages/web/src/app/(app)/settings/connections/[id]/page.tsx deleted file mode 100644 index edcc61069..000000000 --- a/packages/web/src/app/(app)/settings/connections/[id]/page.tsx +++ /dev/null @@ -1,211 +0,0 @@ -import { sew } from "@/middleware/sew"; -import { BackButton } from "@/app/(app)/components/backButton"; -import { DisplayDate } from "@/app/(app)/components/DisplayDate"; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; -import { Skeleton } from "@/components/ui/skeleton"; -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { notFound as notFoundServiceError, ServiceErrorException } from "@/lib/serviceError"; -import { notFound } from "next/navigation"; -import { isServiceError } from "@/lib/utils"; -import { withAuth } from "@/middleware/withAuth"; -import { AzureDevOpsConnectionConfig, BitbucketConnectionConfig, GenericGitHostConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GithubConnectionConfig, GitlabConnectionConfig } from "@sourcebot/schemas/v3/index.type"; -import { env, getConfigSettings } from "@sourcebot/shared"; -import { Info } from "lucide-react"; -import Link from "next/link"; -import { Suspense } from "react"; -import { ConnectionJobsTable } from "../components/connectionJobsTable"; - -interface ConnectionDetailPageProps { - params: Promise<{ - id: string - }> -} - -export default async function ConnectionDetailPage(props: ConnectionDetailPageProps) { - const params = await props.params; - const { id } = params; - - const connectionId = Number.parseInt(id); - if (isNaN(connectionId)) { - return notFound(); - } - - const connection = await getConnectionWithJobs(connectionId); - if (isServiceError(connection)) { - throw new ServiceErrorException(connection); - } - - const configSettings = await getConfigSettings(env.CONFIG_PATH); - - const nextSyncAttempt = (() => { - const latestJob = connection.syncJobs.length > 0 ? connection.syncJobs[0] : null; - if (!latestJob) { - return undefined; - } - - if (latestJob.completedAt) { - return new Date(latestJob.completedAt.getTime() + configSettings.resyncConnectionIntervalMs); - } - - return undefined; - })(); - - // Extracts the code host URL from the connection config. - const codeHostUrl: string = (() => { - const connectionType = connection.connectionType; - switch (connectionType) { - case 'github': { - const config = connection.config as unknown as GithubConnectionConfig; - return config.url ?? 'https://github.com'; - } - case 'gitlab': { - const config = connection.config as unknown as GitlabConnectionConfig; - return config.url ?? 'https://gitlab.com'; - } - case 'gitea': { - const config = connection.config as unknown as GiteaConnectionConfig; - return config.url ?? 'https://gitea.com'; - } - case 'gerrit': { - const config = connection.config as unknown as GerritConnectionConfig; - return config.url; - } - case 'bitbucket': { - const config = connection.config as unknown as BitbucketConnectionConfig; - if (config.deploymentType === 'cloud') { - return config.url ?? 'https://bitbucket.org'; - } else { - return config.url!; - } - } - case 'azuredevops': { - const config = connection.config as unknown as AzureDevOpsConnectionConfig; - return config.url ?? 'https://dev.azure.com'; - } - case 'git': { - const config = connection.config as unknown as GenericGitHostConnectionConfig; - return config.url; - } - } - })(); - - return ( -
- -
-

{connection.name}

- - - {codeHostUrl} - -
- -
- - - - Created - - - - - -

When this connection was first added to Sourcebot

-
-
-
-
- - - -
- - - - - Last synced - - - - - -

The last time this connection was successfully synced

-
-
-
-
- - {connection.syncedAt ? : "Never"} - -
- - - - - Scheduled - - - - - -

When the connection will be resynced next. Modifying the config will also trigger a resync.

-
-
-
-
- - {nextSyncAttempt ? : "-"} - -
-
- - - - Sync History - History of all sync jobs for this connection. - - - }> - - - - -
- ) -} - -const getConnectionWithJobs = async (id: number) => sew(() => - withAuth(async ({ prisma, org }) => { - const connection = await prisma.connection.findUnique({ - where: { - id, - orgId: org.id, - }, - include: { - syncJobs: { - orderBy: { - createdAt: 'desc', - }, - }, - }, - }); - - if (!connection) { - return notFoundServiceError(); - } - - return connection; - }) -) \ No newline at end of file diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx deleted file mode 100644 index 9277d81b7..000000000 --- a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx +++ /dev/null @@ -1,344 +0,0 @@ -"use client" - -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" -import { - type ColumnDef, - type ColumnFiltersState, - type SortingState, - type VisibilityState, - flexRender, - getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, - useReactTable, -} from "@tanstack/react-table" -import { cva } from "class-variance-authority" -import { AlertCircle, AlertTriangle, ArrowUpDown, PlusCircleIcon, RefreshCwIcon } from "lucide-react" -import * as React from "react" -import { CopyIconButton } from "@/app/(app)/components/copyIconButton" -import { useMemo } from "react" -import { LightweightCodeHighlighter } from "@/app/(app)/components/lightweightCodeHighlighter" -import { useRouter } from "next/navigation" -import { useToast } from "@/components/hooks/use-toast" -import { DisplayDate } from "@/app/(app)/components/DisplayDate" -import { LoadingButton } from "@/components/ui/loading-button" -import { syncConnection } from "@/features/workerApi/actions" -import { isServiceError } from "@/lib/utils" - - -export type ConnectionSyncJob = { - id: string - status: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" - createdAt: Date - updatedAt: Date - completedAt: Date | null - errorMessage: string | null - warningMessages: string[] -} - -const statusBadgeVariants = cva("", { - variants: { - status: { - PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", - IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", - COMPLETED: "bg-green-600 text-white hover:bg-green-700", - FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", - }, - }, -}) - -const getStatusBadge = (status: ConnectionSyncJob["status"]) => { - const labels = { - PENDING: "Pending", - IN_PROGRESS: "In Progress", - COMPLETED: "Completed", - FAILED: "Failed", - } - - return {labels[status]} -} - -const getDuration = (start: Date, end: Date | null) => { - if (!end) return "-" - const diff = end.getTime() - start.getTime() - const minutes = Math.floor(diff / 60000) - const seconds = Math.floor((diff % 60000) / 1000) - return `${minutes}m ${seconds}s` -} - -export const columns: ColumnDef[] = [ - { - accessorKey: "status", - header: "Status", - cell: ({ row }) => { - const job = row.original - return ( -
- {getStatusBadge(row.getValue("status"))} - {job.errorMessage ? ( - - - - - - - - {job.errorMessage} - - - - - ) : job.warningMessages.length > 0 ? ( - - - - - - -

{job.warningMessages.length} warning(s) while syncing:

-
- {job.warningMessages.map((warning, index) => ( -
- {index + 1}. - {warning} -
- ))} -
-
-
-
- ) : null} -
- ) - }, - filterFn: (row, id, value) => { - return value.includes(row.getValue(id)) - }, - }, - { - accessorKey: "createdAt", - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => , - }, - { - accessorKey: "completedAt", - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => { - const completedAt = row.getValue("completedAt") as Date | null; - if (!completedAt) { - return "-"; - } - - return - }, - }, - { - id: "duration", - header: "Duration", - cell: ({ row }) => { - const job = row.original - return getDuration(job.createdAt, job.completedAt) - }, - }, - { - accessorKey: "id", - header: "Job ID", - cell: ({ row }) => { - const id = row.getValue("id") as string - return ( -
- {id} - { - navigator.clipboard.writeText(id); - return true; - }} /> -
- ) - }, - }, -] - -export const ConnectionJobsTable = ({ data, connectionId }: { data: ConnectionSyncJob[], connectionId: number }) => { - const [sorting, setSorting] = React.useState([{ id: "createdAt", desc: true }]) - const [columnFilters, setColumnFilters] = React.useState([]) - const [columnVisibility, setColumnVisibility] = React.useState({}) - const router = useRouter(); - const { toast } = useToast(); - - const [isSyncSubmitting, setIsSyncSubmitting] = React.useState(false); - const onSyncButtonClick = React.useCallback(async () => { - setIsSyncSubmitting(true); - const response = await syncConnection(connectionId); - - if (!isServiceError(response)) { - const { jobId } = response; - toast({ - description: `✅ Connection synced successfully. Job ID: ${jobId}`, - }) - router.refresh(); - } else { - toast({ - description: `❌ Failed to sync connection. ${response.message}`, - }); - } - - setIsSyncSubmitting(false); - }, [connectionId, router, toast]); - - const table = useReactTable({ - data, - columns, - onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, - getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onColumnVisibilityChange: setColumnVisibility, - state: { - sorting, - columnFilters, - columnVisibility, - }, - }) - - const { - numCompleted, - numInProgress, - numPending, - numFailed, - } = useMemo(() => { - return { - numCompleted: data.filter((job) => job.status === "COMPLETED").length, - numInProgress: data.filter((job) => job.status === "IN_PROGRESS").length, - numPending: data.filter((job) => job.status === "PENDING").length, - numFailed: data.filter((job) => job.status === "FAILED").length, - }; - }, [data]); - - return ( -
-
- - -
- - - - - Trigger sync - -
-
- -
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ) - })} - - ))} - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} - ))} - - )) - ) : ( - - - No sync jobs found. - - - )} - -
-
- -
-
- {table.getFilteredRowModel().rows.length} job(s) total -
-
- - -
-
-
- ) -} diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx deleted file mode 100644 index e52e7ef1e..000000000 --- a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx +++ /dev/null @@ -1,294 +0,0 @@ -"use client" - -import { DisplayDate } from "@/app/(app)/components/DisplayDate" -import { NotificationDot } from "@/app/(app)/components/notificationDot" -import { useToast } from "@/components/hooks/use-toast" -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { Input } from "@/components/ui/input" -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" -import { getCodeHostIcon } from "@/lib/utils" -import { ConnectionType } from "@sourcebot/db" -import { - type ColumnDef, - type ColumnFiltersState, - type SortingState, - type VisibilityState, - flexRender, - getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, - useReactTable, -} from "@tanstack/react-table" -import { cva } from "class-variance-authority" -import { ArrowUpDown, RefreshCwIcon } from "lucide-react" -import Image from "next/image" -import Link from "next/link" -import { useRouter } from "next/navigation" -import { useMemo, useState } from "react" - - -export type Connection = { - id: number - name: string - syncedAt: Date | null - connectionType: ConnectionType - latestJobStatus: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" | null - isFirstTimeSync: boolean -} - -const statusBadgeVariants = cva("", { - variants: { - status: { - PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", - IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", - COMPLETED: "bg-green-600 text-white hover:bg-green-700", - FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", - }, - }, -}) - -const getStatusBadge = (status: Connection["latestJobStatus"]) => { - if (!status) { - return "-"; - } - - const labels = { - PENDING: "Pending", - IN_PROGRESS: "In Progress", - COMPLETED: "Completed", - FAILED: "Failed", - } - - return {labels[status]} -} - -export const columns: ColumnDef[] = [ - { - accessorKey: "name", - size: 400, - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => { - const connection = row.original; - const codeHostIcon = getCodeHostIcon(connection.connectionType); - - return ( -
- {`${connection.connectionType} - - {connection.name} - - {connection.isFirstTimeSync && ( - - - - - - - - This is the first time Sourcebot is syncing this connection. It may take a few minutes to complete. - - - )} -
- ) - }, - }, - { - accessorKey: "latestJobStatus", - size: 150, - header: "Lastest status", - cell: ({ row }) => getStatusBadge(row.getValue("latestJobStatus")), - }, - { - accessorKey: "syncedAt", - size: 200, - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => { - const syncedAt = row.getValue("syncedAt") as Date | null; - if (!syncedAt) { - return "-"; - } - - return ( - - ) - } - }, -] - -export const ConnectionsTable = ({ data }: { data: Connection[] }) => { - const [sorting, setSorting] = useState([]) - const [columnFilters, setColumnFilters] = useState([]) - const [columnVisibility, setColumnVisibility] = useState({}) - const [rowSelection, setRowSelection] = useState({}) - const router = useRouter(); - const { toast } = useToast(); - - const { - numCompleted, - numInProgress, - numPending, - numFailed, - numNoJobs, - } = useMemo(() => { - return { - numCompleted: data.filter((connection) => connection.latestJobStatus === "COMPLETED").length, - numInProgress: data.filter((connection) => connection.latestJobStatus === "IN_PROGRESS").length, - numPending: data.filter((connection) => connection.latestJobStatus === "PENDING").length, - numFailed: data.filter((connection) => connection.latestJobStatus === "FAILED").length, - numNoJobs: data.filter((connection) => connection.latestJobStatus === null).length, - } - }, [data]); - - const table = useReactTable({ - data, - columns, - onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, - getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onColumnVisibilityChange: setColumnVisibility, - onRowSelectionChange: setRowSelection, - columnResizeMode: 'onChange', - enableColumnResizing: false, - state: { - sorting, - columnFilters, - columnVisibility, - rowSelection, - }, - }) - - return ( -
-
- table.getColumn("name")?.setFilterValue(event.target.value)} - className="max-w-sm" - /> - - -
-
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ) - })} - - ))} - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - - {flexRender(cell.column.columnDef.cell, cell.getContext())} - - ))} - - )) - ) : ( - - - No results. - - - )} - -
-
-
-
- {table.getFilteredRowModel().rows.length} {data.length > 1 ? 'connections' : 'connection'} total -
-
- - -
-
-
- ) -} diff --git a/packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx b/packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx new file mode 100644 index 000000000..0c41c6b38 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx @@ -0,0 +1,203 @@ +"use client"; + +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { getConnectionSyncJobLogs } from "@/features/workerApi/actions"; +import { cn, isServiceError } from "@/lib/utils"; +import type { JobLogEntry, JobLogLevel } from "@sourcebot/shared"; +import { Loader2, RefreshCw } from "lucide-react"; +import { useEffect, useRef, useState } from "react"; + +const LOG_POLL_INTERVAL_MS = 1000; + +const JOB_LOG_LEVEL_CLASS_NAMES: Record = { + debug: "text-muted-foreground", + info: "text-blue-600 dark:text-blue-400", + warn: "text-amber-600 dark:text-amber-400", + error: "text-destructive", +}; + +const formatLogTimestamp = (timestamp: string | null) => { + if (!timestamp) { + return "—"; + } + return new Date(timestamp).toLocaleTimeString(); +}; + +type ConnectionSyncLogsDialogProps = { + connectionId: number; + connectionName: string; + jobId: string | null; + open: boolean; + onOpenChange: (open: boolean) => void; +}; + +export const ConnectionSyncLogsDialog = ({ + connectionId, + connectionName, + jobId, + open, + onOpenChange, +}: ConnectionSyncLogsDialogProps) => { + const [logs, setLogs] = useState([]); + const [hasLoaded, setHasLoaded] = useState(false); + const [isRefreshing, setIsRefreshing] = useState(false); + const [logsError, setLogsError] = useState(null); + const pollNowRef = useRef<() => void>(() => undefined); + + useEffect(() => { + if (!open || !jobId) { + return; + } + + let cursor = 0; + let isCancelled = false; + let isRequestInFlight = false; + + setLogs([]); + setHasLoaded(false); + setLogsError(null); + + const poll = async () => { + if (isRequestInFlight) { + return; + } + + isRequestInFlight = true; + setIsRefreshing(true); + + try { + const result = await getConnectionSyncJobLogs( + connectionId, + jobId, + cursor, + ); + if (isCancelled) { + return; + } + if (isServiceError(result)) { + setLogsError(result.message); + return; + } + + setLogsError(null); + setLogs((currentLogs) => ( + result.count < cursor + ? result.logs + : [...currentLogs, ...result.logs] + )); + cursor = result.count; + } catch { + if (!isCancelled) { + setLogsError("Failed to load logs for this sync."); + } + } finally { + if (!isCancelled) { + setHasLoaded(true); + setIsRefreshing(false); + } + isRequestInFlight = false; + } + }; + + const pollNow = () => { + void poll(); + }; + pollNowRef.current = pollNow; + + pollNow(); + const interval = window.setInterval(pollNow, LOG_POLL_INTERVAL_MS); + + return () => { + isCancelled = true; + window.clearInterval(interval); + if (pollNowRef.current === pollNow) { + pollNowRef.current = () => undefined; + } + }; + }, [connectionId, jobId, open]); + + const isInitialLoading = !hasLoaded && isRefreshing; + + return ( + + +
+ + {connectionName} sync logs + + Latest sync job {jobId}. Logs update automatically while this dialog is open. + + + +
+ +
+ {isInitialLoading ? ( +
+ + Loading logs… +
+ ) : logs.length > 0 ? ( + <> + {logsError && ( +
+ {logsError} +
+ )} +
+ {logs.map((entry, index) => ( +
+ + {formatLogTimestamp(entry.timestamp)} + + + {entry.level} + +

{entry.message}

+
+ ))} +
+ + ) : logsError ? ( +
+ {logsError} +
+ ) : ( +
+ No logs were recorded for this job. +
+ )} +
+
+
+ ); +}; diff --git a/packages/web/src/app/(app)/settings/connections/connectionsList.tsx b/packages/web/src/app/(app)/settings/connections/connectionsList.tsx new file mode 100644 index 000000000..49acc679b --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/connectionsList.tsx @@ -0,0 +1,282 @@ +"use client"; + +import { DisplayDate } from "@/app/(app)/components/DisplayDate"; +import { useToast } from "@/components/hooks/use-toast"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { Input } from "@/components/ui/input"; +import { syncConnection } from "@/features/workerApi/actions"; +import { cn, getCodeHostIcon, isServiceError } from "@/lib/utils"; +import { ConnectionType } from "@sourcebot/db"; +import { AlertCircle, CheckCircle2, CircleDashed, FileText, MoreHorizontal, RefreshCw, Search } from "lucide-react"; +import Image from "next/image"; +import { useRouter } from "next/navigation"; +import { useEffect, useMemo, useState } from "react"; +import { ConnectionSyncLogsDialog } from "./connectionSyncLogsDialog"; +import { WorkloadJob } from "@sourcebot/shared"; + +const SYNC_STATUS_POLL_INTERVAL_MS = 1000; + +export type ConnectionV2 = { + id: number; + name: string; + connectionType: ConnectionType; + syncedAt: Date | null; + currentJob: WorkloadJob<'connection'> | null; +}; + +const getConnectionStatus = (connection: ConnectionV2) => { + switch (connection.currentJob?.status) { + case "PENDING": + return { + key: "syncing", + label: "Queued", + icon: , + className: "border-blue-200 bg-blue-50 text-blue-700 dark:border-blue-900 dark:bg-blue-950 dark:text-blue-300", + }; + case "IN_PROGRESS": + return { + key: "syncing", + label: "Syncing", + icon: , + className: "border-blue-200 bg-blue-50 text-blue-700 dark:border-blue-900 dark:bg-blue-950 dark:text-blue-300", + }; + case "FAILED": + return { + key: "failed", + label: "Sync failed", + icon: , + className: "border-destructive/30 bg-destructive/10 text-destructive", + }; + case "COMPLETED": + return { + key: "healthy", + label: "Healthy", + icon: , + className: "border-green-200 bg-green-50 text-green-700 dark:border-green-900 dark:bg-green-950 dark:text-green-300", + }; + default: + return connection.syncedAt + ? { + key: "healthy", + label: "Healthy", + icon: , + className: "border-green-200 bg-green-50 text-green-700 dark:border-green-900 dark:bg-green-950 dark:text-green-300", + } + : { + key: "not-synced", + label: "Never synced", + icon: , + className: "border-border bg-muted text-muted-foreground", + }; + } +}; + +const ConnectionRow = ({ connection }: { connection: ConnectionV2 }) => { + const [isSubmitting, setIsSubmitting] = useState(false); + const [submittedJobId, setSubmittedJobId] = useState(null); + const [isLogsOpen, setIsLogsOpen] = useState(false); + const router = useRouter(); + const { toast } = useToast(); + const codeHostIcon = getCodeHostIcon(connection.connectionType); + const isRunning = connection.currentJob?.status === "PENDING" || + connection.currentJob?.status === "IN_PROGRESS"; + const isSubmittedJobSettled = connection.currentJob?.id === submittedJobId && + (connection.currentJob.status === "COMPLETED" || connection.currentJob.status === "FAILED"); + const isSyncRunning = isSubmitting || isRunning || (submittedJobId !== null && !isSubmittedJobSettled); + const status = isSyncRunning && !isRunning + ? { + key: "syncing", + label: "Syncing", + icon: , + className: "border-blue-200 bg-blue-50 text-blue-700 dark:border-blue-900 dark:bg-blue-950 dark:text-blue-300", + } + : getConnectionStatus(connection); + + useEffect(() => { + if (!isSyncRunning) { + return; + } + + const interval = window.setInterval(() => { + router.refresh(); + }, SYNC_STATUS_POLL_INTERVAL_MS); + + return () => { + window.clearInterval(interval); + }; + }, [isSyncRunning, router]); + + const onSync = async () => { + setIsSubmitting(true); + try { + const result = await syncConnection(connection.id); + if (isServiceError(result)) { + toast({ + description: `❌ Failed to sync connection. ${result.message}`, + }); + return; + } + + setSubmittedJobId(result.jobId); + toast({ + description: `✅ Connection sync triggered. Job ID: ${result.jobId}`, + }); + router.refresh(); + } catch { + toast({ + description: "❌ Failed to sync connection.", + }); + } finally { + setIsSubmitting(false); + } + }; + + return ( + <> +
+
+
+ {`${connection.connectionType} +
+
+

{connection.name}

+

{connection.connectionType}

+
+
+ +
+

Current status

+ + {status.icon} + {status.label} + + {connection.currentJob?.status === "FAILED" && connection.currentJob.errorMessage && ( +

{connection.currentJob.errorMessage}

+ )} +
+ +
+

Last successful sync

+

+ {connection.syncedAt + ? + : Not yet synced} +

+
+ + + + + + + { + setIsLogsOpen(true); + }} + > + + View logs + + { + void onSync(); + }} + > + + {isSyncRunning ? "Sync in progress" : "Trigger sync"} + + + +
+ + + + ); +}; + +export const ConnectionsList = ({ data }: { data: ConnectionV2[] }) => { + const [query, setQuery] = useState(""); + const healthyCount = useMemo( + () => data.filter((connection) => getConnectionStatus(connection).key === "healthy").length, + [data], + ); + const filteredConnections = useMemo(() => { + const normalizedQuery = query.trim().toLowerCase(); + + return data.filter((connection) => ( + normalizedQuery.length === 0 || + connection.name.toLowerCase().includes(normalizedQuery) || + connection.connectionType.toLowerCase().includes(normalizedQuery) + )); + }, [data, query]); + + return ( +
+
+
+ + setQuery(event.target.value)} + placeholder="Search connections..." + className="pl-9" + /> +
+

+ {healthyCount} healthy · {data.length} total +

+
+ +
+ {filteredConnections.length > 0 ? filteredConnections.map((connection, index) => ( +
0 ? "border-t" : undefined}> + +
+ )) : ( +
+

{data.length === 0 ? "No code host connections" : "No matching connections"}

+

+ {data.length === 0 + ? "Add a connection to begin syncing repositories." + : "Try changing your search."} +

+
+ )} +
+ + {filteredConnections.length > 0 && ( +

+ Showing {filteredConnections.length} of {data.length} {data.length === 1 ? "connection" : "connections"} +

+ )} +
+ ); +}; diff --git a/packages/web/src/app/(app)/settings/connections/page.tsx b/packages/web/src/app/(app)/settings/connections/page.tsx index fdce7b08f..a1d99cd38 100644 --- a/packages/web/src/app/(app)/settings/connections/page.tsx +++ b/packages/web/src/app/(app)/settings/connections/page.tsx @@ -1,77 +1,72 @@ -import { sew } from "@/middleware/sew"; import { ServiceErrorException } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; +import { sew } from "@/middleware/sew"; import { withAuth } from "@/middleware/withAuth"; +import { getBullMQClient } from "@/lib/bullmqClient"; import Link from "next/link"; -import { ConnectionsTable } from "./components/connectionsTable"; -import { ConnectionSyncJobStatus } from "@prisma/client"; +import { ConnectionsList } from "./connectionsList"; +import { CONNECTION_QUEUE } from "@sourcebot/shared"; const DOCS_URL = "https://docs.sourcebot.dev/docs/connections/indexing-your-code"; -export default async function ConnectionsPage() { - const _connections = await getConnectionsWithLatestJob(); - if (isServiceError(_connections)) { - throw new ServiceErrorException(_connections); +export default async function ConnectionsV2Page() { + const connections = await getConnectionsWithCurrentStatus(); + if (isServiceError(connections)) { + throw new ServiceErrorException(connections); } - // Sort connections so that first time syncs are at the top. - const connections = _connections - .map((connection) => ({ - ...connection, - isFirstTimeSync: connection.syncedAt === null && connection.syncJobs.filter((job) => job.status === ConnectionSyncJobStatus.PENDING || job.status === ConnectionSyncJobStatus.IN_PROGRESS).length > 0, - latestJobStatus: connection.syncJobs.length > 0 ? connection.syncJobs[0].status : null, - })) - .sort((a, b) => { - if (a.isFirstTimeSync && !b.isFirstTimeSync) { - return -1; - } - if (!a.isFirstTimeSync && b.isFirstTimeSync) { - return 1; - } - return a.name.localeCompare(b.name); - }); - return (
-

Code Host Connections

-

Manage your connections to external code hosts. Learn more

+
+

Code Host Connections

+ + Prototype + +
+

+ Monitor and sync your external code hosts.{" "} + + Learn more + +

- ({ - id: connection.id, - name: connection.name, - connectionType: connection.connectionType, - syncedAt: connection.syncedAt, - latestJobStatus: connection.latestJobStatus, - isFirstTimeSync: connection.isFirstTimeSync, - }))} /> + +
- ) + ); } -const getConnectionsWithLatestJob = async () => sew(() => +const getConnectionsWithCurrentStatus = async () => sew(() => withAuth(async ({ prisma, org }) => { const connections = await prisma.connection.findMany({ where: { orgId: org.id, }, - include: { - _count: { - select: { - syncJobs: true, - } - }, - syncJobs: { - orderBy: { - createdAt: 'desc' - }, - take: 1 - }, + select: { + id: true, + name: true, + connectionType: true, + syncedAt: true, + latestSyncJobId: true, }, orderBy: { - name: 'asc' + name: 'asc', }, }); - return connections; - })); \ No newline at end of file + return Promise.all(connections.map(async ({ + latestSyncJobId, + ...connection + }) => { + const job = latestSyncJobId + ? await getBullMQClient().getJob(CONNECTION_QUEUE, latestSyncJobId) + : null; + + return { + ...connection, + currentJob: job + }; + })); + }) +); diff --git a/packages/web/src/app/(app)/settings/layout.tsx b/packages/web/src/app/(app)/settings/layout.tsx index 655ca28f5..e57e4180e 100644 --- a/packages/web/src/app/(app)/settings/layout.tsx +++ b/packages/web/src/app/(app)/settings/layout.tsx @@ -3,7 +3,6 @@ import { Metadata } from "next" import { redirect } from "next/navigation"; import { auth } from "@/auth"; import { isServiceError } from "@/lib/utils"; -import { getConnectionStats } from "@/actions"; import { getOrgAccountRequests } from "@/features/membership/actions"; import { ServiceErrorException } from "@/lib/serviceError"; import { OrgRole } from "@prisma/client"; @@ -48,10 +47,6 @@ export const getSidebarNavGroups = async () => numJoinRequests = requests.length; } - const connectionStats = await getConnectionStats(); - if (isServiceError(connectionStats)) { - throw new ServiceErrorException(connectionStats); - } const hasAskEntitlement = await hasEntitlement("ask"); const groups: NavGroup[] = [ @@ -119,8 +114,6 @@ export const getSidebarNavGroups = async () => { title: "Connections", href: `/settings/connections`, - hrefRegex: `/settings/connections(/[^/]+)?$`, - isNotificationDotVisible: connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0, icon: "plug" as const, }, { diff --git a/packages/web/src/features/workerApi/actions.test.ts b/packages/web/src/features/workerApi/actions.test.ts deleted file mode 100644 index 0a949d983..000000000 --- a/packages/web/src/features/workerApi/actions.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { ConnectionSyncJobStatus, OrgRole } from '@sourcebot/db'; -import type { DataOf, QueueSpec } from '@sourcebot/shared'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; - -const mocks = vi.hoisted(() => ({ - connectionFindUnique: vi.fn(), - connectionSyncJobCreateMany: vi.fn(), - enqueue: vi.fn(), -})); - -vi.mock('@/middleware/sew', () => ({ - sew: (fn: () => Promise) => fn(), -})); - -vi.mock('@/middleware/withAuth', () => ({ - withAuth: (fn: (context: unknown) => Promise) => fn({ - org: { id: 7 }, - prisma: { - connection: { - findUnique: mocks.connectionFindUnique, - }, - connectionSyncJob: { - createMany: mocks.connectionSyncJobCreateMany, - }, - }, - role: 'OWNER', - }), - withOptionalAuth: vi.fn(), -})); - -vi.mock('@/middleware/withMinimumOrgRole', () => ({ - withMinimumOrgRole: ( - _role: OrgRole, - _minimumRole: OrgRole, - fn: () => Promise, - ) => fn(), -})); - -vi.mock('@/lib/jobProducer', () => ({ - getJobProducer: () => ({ - enqueue: mocks.enqueue, - }), -})); - -vi.mock('@sourcebot/shared', () => ({ - CONNECTION_QUEUE: { - name: 'connection', - dedupKey: ({ connectionId }: { connectionId: number }) => `connection:${connectionId}`, - jobOptions: { - attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - }, - }, - env: { - WORKER_API_URL: 'http://localhost:3060', - }, -})); - -import { syncConnection } from './actions'; - -describe('syncConnection', () => { - beforeEach(() => { - vi.clearAllMocks(); - mocks.connectionSyncJobCreateMany.mockResolvedValue({ count: 1 }); - mocks.enqueue.mockImplementation(async ( - spec: QueueSpec<'connection'>, - data: DataOf<'connection'>, - ) => { - const jobId = 'job-1'; - await spec.onEnqueued?.({ - data, - jobId, - attemptsMade: 0, - maxAttempts: spec.jobOptions.attempts, - }); - return jobId; - }); - }); - - test('enqueues an org-scoped connection sync and creates its pending job record', async () => { - mocks.connectionFindUnique.mockResolvedValue({ - id: 42, - orgId: 7, - }); - - const result = await syncConnection(42); - - expect(mocks.connectionFindUnique).toHaveBeenCalledWith({ - where: { - id: 42, - orgId: 7, - }, - select: { - id: true, - orgId: true, - }, - }); - expect(mocks.enqueue).toHaveBeenCalledWith( - expect.objectContaining({ - name: 'connection', - onEnqueued: expect.any(Function), - }), - { - connectionId: 42, - orgId: 7, - }, - ); - expect(mocks.connectionSyncJobCreateMany).toHaveBeenCalledWith({ - data: [{ - id: 'job-1', - connectionId: 42, - status: ConnectionSyncJobStatus.PENDING, - warningMessages: [], - }], - skipDuplicates: true, - }); - expect(result).toEqual({ jobId: 'job-1' }); - }); - - test('does not enqueue a missing connection', async () => { - mocks.connectionFindUnique.mockResolvedValue(null); - - const result = await syncConnection(42); - - expect(mocks.enqueue).not.toHaveBeenCalled(); - expect(result).toEqual(expect.objectContaining({ - statusCode: 404, - message: 'Connection not found', - })); - }); -}); diff --git a/packages/web/src/features/workerApi/actions.ts b/packages/web/src/features/workerApi/actions.ts index b0fa45ac5..54f0c7f24 100644 --- a/packages/web/src/features/workerApi/actions.ts +++ b/packages/web/src/features/workerApi/actions.ts @@ -4,10 +4,10 @@ import { sew } from "@/middleware/sew"; import { notFound, repositoryNotFound, unexpectedError } from "@/lib/serviceError"; import { withAuth, withOptionalAuth } from "@/middleware/withAuth"; import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; -import { ConnectionSyncJobStatus, OrgRole } from "@sourcebot/db"; +import { OrgRole } from "@sourcebot/db"; import { CONNECTION_QUEUE, env } from "@sourcebot/shared"; import z from "zod"; -import { getJobProducer } from "@/lib/jobProducer"; +import { getBullMQClient } from "@/lib/bullmqClient"; const WORKER_API_URL = env.WORKER_API_URL; @@ -29,7 +29,7 @@ export const syncConnection = async (connectionId: number) => sew(() => return notFound('Connection not found'); } - const jobId = await getJobProducer().enqueue(CONNECTION_QUEUE, { + const jobId = await getBullMQClient().enqueue(CONNECTION_QUEUE, { connectionId: connection.id, orgId: connection.orgId, }); @@ -39,6 +39,45 @@ export const syncConnection = async (connectionId: number) => sew(() => ) ); +export const getConnectionSyncJobLogs = async ( + connectionId: number, + jobId: string, + start = 0, +) => sew(() => + withAuth(({ org, prisma, role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + const connection = await prisma.connection.findUnique({ + where: { + id: connectionId, + orgId: org.id, + }, + select: { + id: true, + }, + }); + + if (!connection) { + return notFound('Connection not found'); + } + + const client = getBullMQClient(); + const job = await client.getJob(CONNECTION_QUEUE, jobId); + if ( + !job || + job.data.connectionId !== connection.id || + job.data.orgId !== org.id + ) { + return notFound('Connection sync job not found'); + } + + return client.getJobLogs(CONNECTION_QUEUE, jobId, { + start: Number.isInteger(start) && start >= 0 ? start : 0, + ascending: true, + }); + }) + ) +); + export const indexRepo = async (repoId: number) => sew(() => withAuth(({ role }) => withMinimumOrgRole(role, OrgRole.OWNER, async () => { diff --git a/packages/web/src/lib/bullmqClient.ts b/packages/web/src/lib/bullmqClient.ts new file mode 100644 index 000000000..ef5d9524f --- /dev/null +++ b/packages/web/src/lib/bullmqClient.ts @@ -0,0 +1,12 @@ +import 'server-only'; + +import { BullMQClient } from '@sourcebot/shared'; +import { getRedisClient } from './redis'; +import { __unsafePrisma } from '@/prisma'; + +let client: BullMQClient | undefined; + +export function getBullMQClient() { + client ??= new BullMQClient(getRedisClient(), __unsafePrisma); + return client; +} diff --git a/packages/web/src/lib/jobProducer.ts b/packages/web/src/lib/jobProducer.ts deleted file mode 100644 index 1d50ebe6d..000000000 --- a/packages/web/src/lib/jobProducer.ts +++ /dev/null @@ -1,11 +0,0 @@ -import 'server-only'; - -import { BullMQJobProducer } from '@sourcebot/shared'; -import { getRedisClient } from './redis'; - -let jobProducer: BullMQJobProducer | undefined; - -export function getJobProducer() { - jobProducer ??= new BullMQJobProducer(getRedisClient()); - return jobProducer; -} From c7108d668dc81ff3227c996f3f976961256080a8 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Thu, 30 Jul 2026 16:39:29 -0700 Subject: [PATCH 05/15] migrated repo indexing to workload --- packages/backend/package.json | 1 - packages/backend/src/api.ts | 163 +--- packages/backend/src/index.ts | 51 +- packages/backend/src/jobManager.test.ts | 163 +++- .../backend/src/jobManagerLifecycle.test.ts | 176 ---- .../src/reconciliationWorkload.test.ts | 114 ++- .../backend/src/reconciliationWorkload.ts | 52 +- packages/backend/src/repoIndexManager.test.ts | 912 ------------------ packages/backend/src/repoIndexManager.ts | 769 --------------- packages/backend/src/repoIndexWorkload.ts | 351 +++++++ packages/backend/src/types/redlock.d.ts | 95 -- packages/backend/src/utils.ts | 4 +- .../migration.sql | 22 + packages/db/prisma/schema.prisma | 33 +- packages/db/tools/scripts/inject-repo-data.ts | 22 +- packages/shared/src/bullmqClient.test.ts | 34 + packages/shared/src/index.server.ts | 3 +- packages/shared/src/queue.ts | 29 + packages/shared/src/types.ts | 9 - .../src/app/(app)/chat/chatLandingPage.tsx | 11 +- packages/web/src/app/(app)/repos/layout.tsx | 9 - .../search/components/searchLandingPage.tsx | 8 +- yarn.lock | 12 +- 23 files changed, 766 insertions(+), 2277 deletions(-) delete mode 100644 packages/backend/src/jobManagerLifecycle.test.ts delete mode 100644 packages/backend/src/repoIndexManager.test.ts delete mode 100644 packages/backend/src/repoIndexManager.ts create mode 100644 packages/backend/src/repoIndexWorkload.ts delete mode 100644 packages/backend/src/types/redlock.d.ts create mode 100644 packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql diff --git a/packages/backend/package.json b/packages/backend/package.json index 13640cef7..8d9f61aaf 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -52,7 +52,6 @@ "p-limit": "^7.2.0", "posthog-node": "^5.24.15", "prom-client": "^15.1.3", - "redlock": "5.0.0-beta.2", "simple-git": "^3.36.0", "zod": "^3.25.74" } diff --git a/packages/backend/src/api.ts b/packages/backend/src/api.ts index 33f854ada..fd1d91ba5 100644 --- a/packages/backend/src/api.ts +++ b/packages/backend/src/api.ts @@ -1,18 +1,8 @@ -import { PrismaClient, RepoIndexingJobType } from '@sourcebot/db'; -import { hasEntitlement } from './entitlements.js'; -import { createLogger, doesIdpSupportPermissionSyncing, env } from '@sourcebot/shared'; +import { createLogger, env } from '@sourcebot/shared'; import express, { Request, Response } from 'express'; import 'express-async-errors'; import * as http from "http"; -import { ConnectionManager } from './connectionManager.js'; -import { AccountPermissionSyncer } from './ee/accountPermissionSyncer.js'; import { PromClient } from './promClient.js'; -import { RepoIndexManager } from './repoIndexManager.js'; -import { createGitHubRepoRecord } from './repoCompileUtils.js'; -import { isNotFound } from './errors.js'; -import { Octokit } from '@octokit/rest'; -import { SINGLE_TENANT_ORG_ID } from './constants.js'; -import z from 'zod'; const logger = createLogger('api'); @@ -22,13 +12,7 @@ const PORT = Number(workerApiUrl.port) || (workerApiUrl.protocol === "https:" ? export class Api { private server: http.Server; - constructor( - promClient: PromClient, - private prisma: PrismaClient, - private connectionManager: ConnectionManager, - private repoIndexManager: RepoIndexManager, - private accountPermissionSyncer: AccountPermissionSyncer, - ) { + constructor(promClient: PromClient) { const app = express(); app.use(express.json()); app.use(express.urlencoded({ extended: true })); @@ -40,154 +24,11 @@ export class Api { res.end(metrics); }); - app.post('/api/sync-connection', this.syncConnection.bind(this)); - app.post('/api/index-repo', this.indexRepo.bind(this)); - app.post('/api/trigger-account-permission-sync', this.triggerAccountPermissionSync.bind(this)); - app.post(`/api/experimental/add-github-repo`, this.experimental_addGithubRepo.bind(this)); - this.server = app.listen(PORT, () => { logger.debug(`API server is running on port ${PORT}`); }); } - private async syncConnection(req: Request, res: Response) { - const schema = z.object({ - connectionId: z.number(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const { connectionId } = parsed.data; - const connection = await this.prisma.connection.findUnique({ - where: { - id: connectionId, - } - }); - - if (!connection) { - res.status(404).json({ error: 'Connection not found' }); - return; - } - - const [jobId] = await this.connectionManager.createJobs([connection]); - - res.status(200).json({ jobId }); - } - - private async indexRepo(req: Request, res: Response) { - const schema = z.object({ - repoId: z.number(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const { repoId } = parsed.data; - const repo = await this.prisma.repo.findUnique({ - where: { id: repoId }, - }); - - if (!repo) { - res.status(404).json({ error: 'Repo not found' }); - return; - } - - const [jobId] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX); - res.status(200).json({ jobId }); - } - - private async triggerAccountPermissionSync(req: Request, res: Response) { - if (env.PERMISSION_SYNC_ENABLED !== 'true' || !await hasEntitlement('permission-syncing')) { - res.status(403).json({ error: 'Permission syncing is not enabled.' }); - return; - } - - const schema = z.object({ - accountId: z.string(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const { accountId } = parsed.data; - const account = await this.prisma.account.findUnique({ - where: { id: accountId }, - }); - - if (!account) { - res.status(404).json({ error: 'Account not found' }); - return; - } - - if (!doesIdpSupportPermissionSyncing(account.providerType)) { - res.status(400).json({ error: `Provider '${account.providerType}' does not support permission syncing.` }); - return; - } - - const jobId = await this.accountPermissionSyncer.schedulePermissionSyncForAccount(account); - res.status(200).json({ jobId }); - } - - private async experimental_addGithubRepo(req: Request, res: Response) { - const schema = z.object({ - owner: z.string(), - repo: z.string(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const octokit = new Octokit(); - let response; - try { - response = await octokit.rest.repos.get({ - owner: parsed.data.owner, - repo: parsed.data.repo, - }); - } catch (error) { - if (isNotFound(error)) { - res.status(404).json({ error: 'Repository not found on GitHub' }); - return; - } - throw error; - } - - const record = createGitHubRepoRecord({ - repo: response.data, - hostUrl: 'https://github.com', - isAutoCleanupDisabled: true, - }); - - const repo = await this.prisma.repo.upsert({ - where: { - external_id_external_codeHostUrl_orgId: { - external_id: record.external_id, - external_codeHostUrl: record.external_codeHostUrl, - orgId: SINGLE_TENANT_ORG_ID, - } - }, - update: record, - create: record, - }); - - const [jobId ] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX); - - res.status(200).json({ jobId, repoId: repo.id }); - } - public async dispose() { return new Promise((resolve, reject) => { this.server.close((err) => { diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 9efa68768..73db4fc3f 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -16,6 +16,8 @@ import { PromClient } from './promClient.js'; import { createReconciliationWorkload } from "./reconciliationWorkload.js"; import { redis } from "./redis.js"; import { connectionWorkload } from "./connectionWorkload.js"; +import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; +import { Api } from "./api.js"; const logger = createLogger('backend-entrypoint'); @@ -38,7 +40,6 @@ try { process.exit(1); } -const promClient = new PromClient(); const settings = await getConfigSettings(env.CONFIG_PATH); @@ -46,51 +47,27 @@ if (await hasEntitlement('github-app')) { await GithubAppManager.getInstance().init(prisma); } -// const connectionManager = new ConnectionManager(prisma, settings, redis, promClient); -// const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); -// const accountPermissionSyncer = new AccountPermissionSyncer(prisma, settings, redis); -// const repoIndexManager = new RepoIndexManager(prisma, settings, redis, promClient); -// const auditLogPruner = new AuditLogPruner(prisma); -// const attachmentPruner = new AttachmentPruner(prisma); - -// connectionManager.startScheduler(); -// await repoIndexManager.startScheduler(); -// auditLogPruner.startScheduler(); -// attachmentPruner.startScheduler(); - -// if (env.PERMISSION_SYNC_ENABLED === 'true' && !await hasEntitlement('permission-syncing')) { -// logger.warn('Permission syncing is not supported in current plan. Please contact team@sourcebot.dev for assistance.'); -// } -// else if (env.PERMISSION_SYNC_ENABLED === 'true' && await hasEntitlement('permission-syncing')) { -// if (env.PERMISSION_SYNC_REPO_DRIVEN_ENABLED === 'true') { -// await repoPermissionSyncer.startScheduler(); -// } -// await accountPermissionSyncer.startScheduler(); -// } - -// const api = new Api( -// promClient, -// prisma, -// connectionManager, -// repoIndexManager, -// accountPermissionSyncer, -// ); +const promClient = new PromClient(); +const api = new Api(promClient); logger.info('Worker started.'); -// Background jobs run through the JobManager (BullMQ/Redis as the source of truth). Phase 0 -// wires the framework here in place of the old per-manager pollers; the real workloads -// (repo-index, connection-sync, permission syncers) are ported onto it in subsequent phases. const jobManager = new BullMQJobManager(redis); const reconciliationWorkload = createReconciliationWorkload({ db: prisma, settings, }); +const repoIndexWorkload = createRepoIndexWorkload({ + db: prisma, + settings, +}); jobManager.register(reconciliationWorkload); jobManager.register(connectionWorkload); +jobManager.register(repoIndexWorkload); +await cleanupOrphanedRepoResources(prisma); await jobManager.start(); const configManager = new ConfigManager(jobManager, env.CONFIG_PATH); @@ -110,18 +87,12 @@ const listenToShutdownSignals = () => { logger.info(`Received ${signal}, cleaning up...`); - // await repoIndexManager.dispose() - // await connectionManager.dispose() - // await repoPermissionSyncer.dispose() - // await accountPermissionSyncer.dispose() - // await auditLogPruner.dispose() - // await attachmentPruner.dispose() await configManager.dispose() await jobManager.stop(); await prisma.$disconnect(); await redis.quit(); - // await api.dispose(); + await api.dispose(); await shutdownPosthog(); logger.info('All workers shut down gracefully'); diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts index 5f01538a5..894efdade 100644 --- a/packages/backend/src/jobManager.test.ts +++ b/packages/backend/src/jobManager.test.ts @@ -1,4 +1,23 @@ -import { describe, expect, test, vi } from 'vitest'; +import { Redis } from 'ioredis'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { ProcessContext, Workload } from './types.js'; + +const mocks = vi.hoisted(() => ({ + enqueue: vi.fn(), + producerClose: vi.fn(), + workerClose: vi.fn(), + jobLogger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + flush: vi.fn(), + }, + workers: [] as Array<{ + processor: (job: unknown) => Promise; + handlers: Map void>; + }>, +})); // The module under test creates a logger at import time; stub it so importing pure helpers // has no side effects (mirrors repoIndexManager.test.ts). @@ -9,6 +28,15 @@ vi.mock('@sourcebot/shared', () => ({ error: vi.fn(), debug: vi.fn(), })), + createBullMQJobLogger: vi.fn(() => mocks.jobLogger), + BullMQClient: class { + enqueue = mocks.enqueue; + close = mocks.producerClose; + getQueue = vi.fn(() => ({ + getJobCounts: vi.fn(), + upsertJobScheduler: vi.fn(), + })); + }, })); // Mock the constants module directly so its env-derived cache-dir paths don't load. @@ -16,7 +44,28 @@ vi.mock('./constants.js', () => ({ WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, })); -import { normalizeJobState, parseDuration } from './jobManager.js'; +vi.mock('@sentry/node', () => ({ + captureException: vi.fn(), +})); + +vi.mock('bullmq', () => ({ + Worker: class { + private readonly record: (typeof mocks.workers)[number]; + + constructor(_name: string, processor: (job: unknown) => Promise) { + this.record = { processor, handlers: new Map() }; + mocks.workers.push(this.record); + } + + on(event: string, handler: (...args: unknown[]) => void) { + this.record.handlers.set(event, handler); + } + + close = mocks.workerClose; + }, +})); + +import { BullMQJobManager, normalizeJobState, parseDuration } from './jobManager.js'; describe('parseDuration', () => { test.each([ @@ -61,3 +110,113 @@ describe('normalizeJobState', () => { }); }); +const createWorkload = ( + overrides: Partial> = {}, +): Workload<'connection', { repoCount: number }> => ({ + queueSpec: { + name: 'connection', + dedupKey: ({ connectionId }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: 500, + }, + }, + concurrency: 2, + process: vi.fn(async () => ({ repoCount: 3 })), + ...overrides, +}); + +const data = { connectionId: 42, orgId: 1 }; +const job = { + id: 'job-1', + data, + attemptsMade: 2, + opts: { attempts: 2 }, + log: vi.fn(), + updateProgress: vi.fn(), +}; + +describe('BullMQJobManager lifecycle', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.workers.length = 0; + mocks.enqueue.mockResolvedValue('job-1'); + }); + + test('delegates enqueueing to BullMQClient and returns its job id', async () => { + const manager = new BullMQJobManager({} as Redis); + const workload = createWorkload(); + manager.register(workload); + + const result = await manager.trigger('connection', data); + + expect(result).toBe('job-1'); + expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data); + }); + + test('calls onStarted before processing and onCompleted after completion', async () => { + const calls: string[] = []; + const workload = createWorkload({ + onStarted: vi.fn(async () => { calls.push('started'); }), + process: vi.fn(async () => { + calls.push('processed'); + return { repoCount: 3 }; + }), + onCompleted: vi.fn(async () => { calls.push('completed'); }), + }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + const result = await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); + expect(calls).toEqual(['started', 'processed']); + + mocks.workers[0].handlers.get('completed')?.(job, result); + await vi.waitFor(() => expect(calls).toEqual(['started', 'processed', 'completed'])); + expect(workload.onCompleted).toHaveBeenCalledWith( + expect.objectContaining({ data, jobId: 'job-1', maxAttempts: 2 }), + { repoCount: 3 }, + ); + expect(mocks.jobLogger.flush).toHaveBeenCalled(); + }); + + test('provides the structured job logger to the workload processor', async () => { + const process = vi.fn(async (context: ProcessContext<'connection'>) => { + context.logger.info('Processing connection'); + return { repoCount: 3 }; + }); + const manager = new BullMQJobManager({} as Redis); + manager.register(createWorkload({ process })); + await manager.start(); + + await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); + + expect(process).toHaveBeenCalledWith(expect.objectContaining({ + logger: mocks.jobLogger, + })); + expect(mocks.jobLogger.info).toHaveBeenCalledWith('Processing connection'); + expect(mocks.jobLogger.flush).toHaveBeenCalled(); + }); + + test('reports lifecycle metadata after terminal failure', async () => { + const onTerminalFailure = vi.fn(); + const workload = createWorkload({ onTerminalFailure }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + const error = new Error('failed'); + mocks.workers[0].handlers.get('failed')?.(job, error); + + await vi.waitFor(() => { + expect(onTerminalFailure).toHaveBeenCalledWith(expect.objectContaining({ + data, + jobId: 'job-1', + attemptsMade: 2, + maxAttempts: 2, + }), error); + }); + }); +}); diff --git a/packages/backend/src/jobManagerLifecycle.test.ts b/packages/backend/src/jobManagerLifecycle.test.ts deleted file mode 100644 index 86a1120f4..000000000 --- a/packages/backend/src/jobManagerLifecycle.test.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { Redis } from 'ioredis'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; -import { ProcessContext, Workload } from './types.js'; - -const mocks = vi.hoisted(() => ({ - enqueue: vi.fn(), - producerClose: vi.fn(), - workerClose: vi.fn(), - jobLogger: { - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - flush: vi.fn(), - }, - workers: [] as Array<{ - processor: (job: unknown) => Promise; - handlers: Map void>; - }>, -})); - -vi.mock('@sentry/node', () => ({ - captureException: vi.fn(), -})); - -vi.mock('@sourcebot/shared', () => ({ - createLogger: vi.fn(() => ({ - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - })), - createBullMQJobLogger: vi.fn(() => mocks.jobLogger), - BullMQClient: class { - enqueue = mocks.enqueue; - close = mocks.producerClose; - queue = vi.fn(() => ({ - getJobCounts: vi.fn(), - upsertJobScheduler: vi.fn(), - })); - }, -})); - -vi.mock('./constants.js', () => ({ - WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, -})); - -vi.mock('bullmq', () => ({ - Worker: class { - private readonly record: (typeof mocks.workers)[number]; - - constructor(_name: string, processor: (job: unknown) => Promise) { - this.record = { processor, handlers: new Map() }; - mocks.workers.push(this.record); - } - - on(event: string, handler: (...args: unknown[]) => void) { - this.record.handlers.set(event, handler); - } - - close = mocks.workerClose; - }, -})); - -import { BullMQJobManager } from './jobManager.js'; - -const createWorkload = ( - overrides: Partial> = {}, -): Workload<'connection', { repoCount: number }> => ({ - queueSpec: { - name: 'connection', - dedupKey: ({ connectionId }) => `connection:${connectionId}`, - jobOptions: { - attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - keepLogs: 500, - }, - }, - concurrency: 2, - process: vi.fn(async () => ({ repoCount: 3 })), - ...overrides, -}); - -const data = { connectionId: 42, orgId: 1 }; -const job = { - id: 'job-1', - data, - attemptsMade: 2, - opts: { attempts: 2 }, - log: vi.fn(), - updateProgress: vi.fn(), -}; - -describe('BullMQJobManager lifecycle', () => { - beforeEach(() => { - vi.clearAllMocks(); - mocks.workers.length = 0; - mocks.enqueue.mockResolvedValue('job-1'); - }); - - test('delegates enqueueing to BullMQClient and returns its job id', async () => { - const manager = new BullMQJobManager({} as Redis); - const workload = createWorkload(); - manager.register(workload); - - const result = await manager.trigger('connection', data); - - expect(result).toBe('job-1'); - expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data); - }); - - test('calls onStarted before processing and onCompleted after completion', async () => { - const calls: string[] = []; - const workload = createWorkload({ - onStarted: vi.fn(async () => { calls.push('started'); }), - process: vi.fn(async () => { - calls.push('processed'); - return { repoCount: 3 }; - }), - onCompleted: vi.fn(async () => { calls.push('completed'); }), - }); - const manager = new BullMQJobManager({} as Redis); - manager.register(workload); - await manager.start(); - - const result = await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); - expect(calls).toEqual(['started', 'processed']); - - mocks.workers[0].handlers.get('completed')?.(job, result); - await vi.waitFor(() => expect(calls).toEqual(['started', 'processed', 'completed'])); - expect(workload.onCompleted).toHaveBeenCalledWith( - expect.objectContaining({ data, jobId: 'job-1', maxAttempts: 2 }), - { repoCount: 3 }, - ); - expect(mocks.jobLogger.flush).toHaveBeenCalled(); - }); - - test('provides the structured job logger to the workload processor', async () => { - const process = vi.fn(async (context: ProcessContext<'connection'>) => { - context.logger.info('Processing connection'); - return { repoCount: 3 }; - }); - const manager = new BullMQJobManager({} as Redis); - manager.register(createWorkload({ process })); - await manager.start(); - - await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); - - expect(process).toHaveBeenCalledWith(expect.objectContaining({ - logger: mocks.jobLogger, - })); - expect(mocks.jobLogger.info).toHaveBeenCalledWith('Processing connection'); - expect(mocks.jobLogger.flush).toHaveBeenCalled(); - }); - - test('reports lifecycle metadata after terminal failure', async () => { - const onTerminalFailure = vi.fn(); - const workload = createWorkload({ onTerminalFailure }); - const manager = new BullMQJobManager({} as Redis); - manager.register(workload); - await manager.start(); - - const error = new Error('failed'); - mocks.workers[0].handlers.get('failed')?.(job, error); - - await vi.waitFor(() => { - expect(onTerminalFailure).toHaveBeenCalledWith(expect.objectContaining({ - data, - jobId: 'job-1', - attemptsMade: 2, - maxAttempts: 2, - }), error); - }); - }); -}); diff --git a/packages/backend/src/reconciliationWorkload.test.ts b/packages/backend/src/reconciliationWorkload.test.ts index 3aea21082..949af1c50 100644 --- a/packages/backend/src/reconciliationWorkload.test.ts +++ b/packages/backend/src/reconciliationWorkload.test.ts @@ -1,4 +1,5 @@ import type { PrismaClient } from '@sourcebot/db'; +import type { JobLogger } from '@sourcebot/shared'; import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; vi.mock('@sourcebot/shared', () => ({ @@ -11,25 +12,41 @@ vi.mock('@sourcebot/shared', () => ({ keepLogs: 500, }, }, - createLogger: vi.fn(() => ({ - debug: vi.fn(), - })), })); import { createReconciliationWorkload } from './reconciliationWorkload.js'; +const settings = { + resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, + reindexIntervalMs: 60 * 60 * 1000, + repoGarbageCollectionGracePeriodMs: 10 * 1000, +}; + +const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), +} satisfies JobLogger; + describe('reconciliationWorkload', () => { - const findMany = vi.fn(); + const connectionFindMany = vi.fn(); + const repoFindMany = vi.fn(); const db = { connection: { - findMany, + findMany: connectionFindMany, + }, + repo: { + findMany: repoFindMany, }, } as unknown as PrismaClient; beforeEach(() => { vi.useFakeTimers(); vi.setSystemTime(new Date('2026-07-27T12:00:00.000Z')); - findMany.mockReset(); + connectionFindMany.mockReset().mockResolvedValue([]); + repoFindMany.mockReset().mockResolvedValue([]); }); afterEach(() => { @@ -39,9 +56,7 @@ describe('reconciliationWorkload', () => { test('runs every 15 minutes on the reconciliation queue', () => { const workload = createReconciliationWorkload({ db, - settings: { - resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, - }, + settings, }); expect(workload.queueSpec.name).toBe('reconciliation'); @@ -50,16 +65,14 @@ describe('reconciliationWorkload', () => { }); test('triggers connection syncs for connections that are due', async () => { - findMany.mockResolvedValue([ + connectionFindMany.mockResolvedValue([ { id: 42, orgId: 1 }, { id: 84, orgId: 2 }, ]); const trigger = vi.fn().mockResolvedValue('job-id'); const workload = createReconciliationWorkload({ db, - settings: { - resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, - }, + settings, }); await workload.process({ @@ -67,19 +80,14 @@ describe('reconciliationWorkload', () => { jobId: 'reconciliation-job', attemptsMade: 0, maxAttempts: 2, + prisma: db, signal: new AbortController().signal, - logger: { - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - flush: vi.fn(), - }, + logger, updateProgress: vi.fn(), trigger, }); - expect(findMany).toHaveBeenCalledWith({ + expect(connectionFindMany).toHaveBeenCalledWith({ where: { OR: [ { syncedAt: null }, @@ -101,4 +109,68 @@ describe('reconciliationWorkload', () => { orgId: 2, }); }); + + test('submits all due repos for indexing', async () => { + repoFindMany + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { id: 1 }, + { id: 2 }, + { id: 3 }, + { id: 4 }, + ]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger, + }); + + expect(trigger).toHaveBeenCalledTimes(4); + for (const repoId of [1, 2, 3, 4]) { + expect(trigger).toHaveBeenCalledWith('repo-index', { + repoId, + type: 'INDEX', + }); + } + }); + + test('schedules orphaned repos for cleanup', async () => { + repoFindMany + .mockResolvedValueOnce([{ id: 42 }]) + .mockResolvedValueOnce([]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger, + }); + + expect(trigger).toHaveBeenCalledWith('repo-index', { + repoId: 42, + type: 'CLEANUP', + }); + }); }); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts index 73209b26c..e36e7c975 100644 --- a/packages/backend/src/reconciliationWorkload.ts +++ b/packages/backend/src/reconciliationWorkload.ts @@ -4,7 +4,7 @@ import { Settings, Workload } from "./types.js"; interface ReconciliationWorkloadDependencies { db: PrismaClient; - settings: Pick; + settings: Settings; } export const createReconciliationWorkload = ({ @@ -15,12 +15,12 @@ export const createReconciliationWorkload = ({ schedule: { every: '15m' }, queueSpec: RECONCILIATION_QUEUE, process: async ({ logger, trigger }) => { - const thresholdDate = new Date(Date.now() - settings.resyncConnectionIntervalMs); + const connectionThreshold = new Date(Date.now() - settings.resyncConnectionIntervalMs); const connections = await db.connection.findMany({ where: { OR: [ { syncedAt: null }, - { syncedAt: { lt: thresholdDate } }, + { syncedAt: { lt: connectionThreshold } }, ], }, select: { @@ -36,5 +36,51 @@ export const createReconciliationWorkload = ({ orgId: connection.orgId, }); })); + + const cleanupThreshold = new Date(Date.now() - settings.repoGarbageCollectionGracePeriodMs); + const reposToCleanup = await db.repo.findMany({ + where: { + connections: { + none: {}, + }, + isAutoCleanupDisabled: false, + OR: [ + { indexedAt: null }, + { indexedAt: { lt: cleanupThreshold } }, + ], + }, + select: { + id: true, + }, + }); + + await Promise.all(reposToCleanup.map(async ({ id }) => { + logger.debug(`Scheduling cleanup for repo ${id}`); + await trigger('repo-index', { + repoId: id, + type: 'CLEANUP', + }); + })); + + const indexThreshold = new Date(Date.now() - settings.reindexIntervalMs); + const reposToIndex = await db.repo.findMany({ + where: { + OR: [ + { indexedAt: null }, + { indexedAt: { lt: indexThreshold } }, + ], + }, + select: { + id: true, + }, + }); + + await Promise.all(reposToIndex.map(async ({ id }) => { + logger.debug(`Scheduling index for repo ${id}`); + await trigger('repo-index', { + repoId: id, + type: 'INDEX', + }); + })); }, }); diff --git a/packages/backend/src/repoIndexManager.test.ts b/packages/backend/src/repoIndexManager.test.ts deleted file mode 100644 index 684f1a826..000000000 --- a/packages/backend/src/repoIndexManager.test.ts +++ /dev/null @@ -1,912 +0,0 @@ -import type { PrismaClient, Repo } from '@sourcebot/db'; -import { RepoIndexingJobStatus, RepoIndexingJobType } from '@sourcebot/db'; -import type { Job } from 'bullmq'; -import type { Redis } from 'ioredis'; -import { afterEach, beforeEach, describe, expect, Mock, test, vi } from 'vitest'; -import type { RepoWithConnections, Settings } from './types.js'; - -// Mock modules before importing the class under test -vi.mock('@sentry/node', () => ({ - captureException: vi.fn(), -})); - -vi.mock('@sourcebot/shared', () => ({ - createLogger: vi.fn(() => ({ - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - })), - env: { - DATA_CACHE_DIR: 'test-data', - REDIS_REMOVE_ON_COMPLETE: true, - REDIS_REMOVE_ON_FAIL: true, - }, - getRepoPath: vi.fn((repo: Repo) => ({ - path: `/test-data/repos/${repo.id}`, - isReadOnly: false, - })), - repoMetadataSchema: { - parse: vi.fn((metadata: unknown) => metadata ?? {}), - }, - repoIndexingJobMetadataSchema: { - parse: vi.fn((metadata: unknown) => metadata ?? {}), - }, -})); - -vi.mock('./constants.js', () => ({ - WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, - INDEX_CACHE_DIR: 'test-data/index', -})); - -vi.mock('./git.js', () => ({ - cloneRepository: vi.fn(), - fetchRepository: vi.fn(), - getBranches: vi.fn().mockResolvedValue([]), - getTags: vi.fn().mockResolvedValue([]), - getLocalDefaultBranch: vi.fn().mockResolvedValue('main'), - getCommitHashForRefName: vi.fn().mockResolvedValue('abc123'), - getLatestCommitTimestamp: vi.fn().mockResolvedValue(new Date()), - isPathAValidGitRepoRoot: vi.fn().mockResolvedValue(true), - isRepoEmpty: vi.fn().mockResolvedValue(false), - unsetGitConfig: vi.fn(), - upsertGitConfig: vi.fn(), - writeCommitGraph: vi.fn(), -})); - -vi.mock('./zoekt.js', () => ({ - indexGitRepository: vi.fn().mockResolvedValue({ stdout: '', stderr: '' }), -})); - -vi.mock('./posthog.js', () => ({ - captureEvent: vi.fn(), -})); - -vi.mock('./utils.js', () => ({ - getAuthCredentialsForRepo: vi.fn().mockResolvedValue(null), - getShardPrefix: vi.fn((orgId: number, repoId: number) => `${orgId}_${repoId}`), - measure: vi.fn(async (cb: () => Promise) => { - const data = await cb(); - return { data, durationMs: 100 }; - }), - setIntervalAsync: vi.fn((cb: () => void, _interval: number) => { - // Return a mock interval ID - return { unref: vi.fn() } as unknown as NodeJS.Timeout; - }), -})); - -vi.mock('fs', () => ({ - existsSync: vi.fn().mockReturnValue(false), -})); - -vi.mock('fs/promises', () => ({ - rm: vi.fn(), - readdir: vi.fn().mockResolvedValue([]), -})); - -// Mock BullMQ -const mockQueueAdd = vi.fn().mockResolvedValue(undefined); -const mockQueueClose = vi.fn().mockResolvedValue(undefined); -const mockWorkerClose = vi.fn().mockResolvedValue(undefined); -const mockWorkerOn = vi.fn(); - -vi.mock('bullmq', () => ({ - Queue: vi.fn().mockImplementation(function () { - return { - add: mockQueueAdd, - close: mockQueueClose, - }; - }), - Worker: vi.fn().mockImplementation(function (_name: string, processor: unknown) { - return { - on: mockWorkerOn, - close: mockWorkerClose, - processJob: processor, - }; - }), - DelayedError: class DelayedError extends Error { - constructor(message: string) { - super(message); - this.name = 'DelayedError'; - } - }, -})); - -// Mock Redlock -const mockRedlockUsing = vi.fn(); -vi.mock('redlock', () => ({ - default: vi.fn().mockImplementation(function () { - return { - using: mockRedlockUsing, - }; - }), - ExecutionError: class ExecutionError extends Error { - constructor(message: string) { - super(message); - this.name = 'ExecutionError'; - } - }, -})); - -// Import after mocks are set up -import { existsSync } from 'fs'; -import { readdir, rm } from 'fs/promises'; -import { ExecutionError } from 'redlock'; -import { - cloneRepository, - fetchRepository, - getBranches, - getTags, - isPathAValidGitRepoRoot, -} from './git.js'; -import { RepoIndexManager } from './repoIndexManager.js'; -import { indexGitRepository } from './zoekt.js'; - -// Helper to create mock Prisma client -const createMockPrisma = () => { - return { - repo: { - findMany: vi.fn().mockResolvedValue([]), - update: vi.fn(), - delete: vi.fn(), - }, - repoIndexingJob: { - createManyAndReturn: vi.fn().mockResolvedValue([]), - findUniqueOrThrow: vi.fn().mockResolvedValue({ status: RepoIndexingJobStatus.PENDING }), - update: vi.fn(), - }, - } as unknown as PrismaClient; -}; - -// Helper to create mock Redis -const createMockRedis = () => { - return {} as Redis; -}; - -// Helper to create mock Settings -const createMockSettings = (): Settings => ({ - maxFileSize: 2 * 1024 * 1024, - maxTrigramCount: 20000, - reindexIntervalMs: 1000 * 60 * 60, - resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, - resyncConnectionPollingIntervalMs: 1000 * 1, - reindexRepoPollingIntervalMs: 1000 * 1, - maxConnectionSyncJobConcurrency: 8, - maxRepoIndexingJobConcurrency: 8, - maxRepoGarbageCollectionJobConcurrency: 8, - repoGarbageCollectionGracePeriodMs: 10 * 1000, - repoIndexTimeoutMs: 1000 * 60 * 60 * 2, - enablePublicAccess: false, - experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - maxAccountPermissionSyncJobConcurrency: 8, - maxRepoPermissionSyncJobConcurrency: 8, -}); - -// Helper to create mock PromClient -const createMockPromClient = () => ({ - pendingRepoIndexJobs: { inc: vi.fn(), dec: vi.fn() }, - activeRepoIndexJobs: { inc: vi.fn(), dec: vi.fn() }, - repoIndexJobSuccessTotal: { inc: vi.fn() }, - repoIndexJobFailTotal: { inc: vi.fn() }, -}); - -// Helper to create a mock repo -const createMockRepo = (overrides: Partial = {}): Repo => ({ - id: 1, - name: 'test-repo', - cloneUrl: 'https://github.com/test/repo.git', - orgId: 1, - indexedAt: null, - indexedCommitHash: null, - defaultBranch: 'main', - metadata: {}, - repoIndexingStatus: 'CREATED', - latestIndexingJobStatus: null, - latestConnectionSyncJobStatus: null, - external_id: 'test-external-id', - external_codeHostType: 'github', - external_codeHostUrl: 'https://github.com', - pushedAt: null, - createdAt: new Date(), - updatedAt: new Date(), - isFork: false, - isArchived: false, - isAutoCleanupDisabled: false, - ...overrides, -} as Repo); - -// Helper to create a mock repoWithConnections -const createMockRepoWithConnections = (overrides: Partial = {}): RepoWithConnections => ({ - ...createMockRepo(), - connections: [], - ...overrides, -}); - -describe('RepoIndexManager', () => { - let mockPrisma: PrismaClient; - let mockRedis: Redis; - let mockSettings: Settings; - let mockPromClient: ReturnType; - let manager: RepoIndexManager; - - beforeEach(() => { - vi.clearAllMocks(); - mockPrisma = createMockPrisma(); - mockRedis = createMockRedis(); - mockSettings = createMockSettings(); - mockPromClient = createMockPromClient(); - - // Default redlock behavior - execute the callback immediately - mockRedlockUsing.mockImplementation(async (_keys: string[], _ttl: number, cb: (signal: AbortSignal) => Promise) => { - const signal = new AbortController().signal; - return cb(signal); - }); - }); - - afterEach(async () => { - if (manager) { - await manager.dispose(); - } - }); - - describe('Job Processing - Success', () => { - test('clones new repository when directory does not exist', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(false); - (cloneRepository as Mock).mockResolvedValue(undefined); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - // Set up mocks for job processing - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - // Simulate processing a job - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - // Get the worker processor callback - const { Worker } = await import('bullmq'); - const workerCalls = (Worker as unknown as Mock).mock.calls; - expect(workerCalls.length).toBeGreaterThan(0); - const processor = workerCalls[0][1]; - - // Execute the processor - await processor(mockJob); - - expect(cloneRepository).toHaveBeenCalledWith( - expect.objectContaining({ - cloneUrl: repo.cloneUrl, - path: expect.stringContaining(`${repo.id}`), - }) - ); - }); - - test('deletes directory and performs fresh clone when path exists but is not a valid git repo root', async () => { - const repo = createMockRepoWithConnections(); - // Path exists initially but after rm is called, it no longer exists - // First two calls return true (first check + check before delete), then false (after deletion) - (existsSync as Mock) - .mockReturnValueOnce(true) // First existsSync check - path exists - .mockReturnValueOnce(false); // Second existsSync check - path deleted, trigger clone - (isPathAValidGitRepoRoot as Mock).mockResolvedValue(false); - (cloneRepository as Mock).mockResolvedValue(undefined); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - // Should delete the invalid directory - expect(rm).toHaveBeenCalledWith( - expect.stringContaining(`${repo.id}`), - { recursive: true, force: true } - ); - - // Should perform a fresh clone after deletion - expect(cloneRepository).toHaveBeenCalledWith( - expect.objectContaining({ - cloneUrl: repo.cloneUrl, - path: expect.stringContaining(`${repo.id}`), - }) - ); - }); - - test('fetches existing repository when directory exists', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - (fetchRepository as Mock).mockResolvedValue(undefined); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - expect(fetchRepository).toHaveBeenCalledWith( - expect.objectContaining({ - cloneUrl: repo.cloneUrl, - path: expect.stringContaining(`${repo.id}`), - }) - ); - }); - - test('invokes zoekt-git-index with correct arguments', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - expect(indexGitRepository).toHaveBeenCalledWith( - repo, - mockSettings, - expect.arrayContaining(['refs/heads/main']), - expect.any(Object) - ); - }); - - test('keeps default branch and truncates to the first 63 matching tags', async () => { - const newestTagsFirst = Array.from( - { length: 70 }, - (_, index) => `v${70 - index}.0.0`, - ); - const repo = createMockRepoWithConnections({ - metadata: { - tags: ['**'], - }, - }); - (existsSync as Mock).mockReturnValue(true); - (getTags as Mock).mockResolvedValue(newestTagsFirst); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - expect(indexGitRepository).toHaveBeenCalledWith( - repo, - mockSettings, - [ - 'refs/heads/main', - ...newestTagsFirst - .slice(0, 63) - .map((tag) => `refs/tags/${tag}`), - ], - expect.any(Object) - ); - }); - - test('de-duplicates the default branch before truncating matching branches', async () => { - const newestBranchesFirst = [ - 'feature/newest', - 'main', - ...Array.from( - { length: 68 }, - (_, index) => `feature/${68 - index}`, - ), - ]; - const repo = createMockRepoWithConnections({ - metadata: { - branches: ['main', 'feature/**'], - }, - }); - (existsSync as Mock).mockReturnValue(true); - (getBranches as Mock).mockResolvedValue(newestBranchesFirst); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - const revisions = (indexGitRepository as Mock).mock.calls.at(-1)?.[2] as string[]; - - expect(revisions).toHaveLength(64); - expect(revisions.filter((revision) => revision === 'refs/heads/main')).toHaveLength(1); - expect(revisions[0]).toBe('refs/heads/main'); - expect(revisions).toContain('refs/heads/feature/newest'); - expect(revisions).not.toContain('refs/heads/feature/6'); - }); - - test('updates repo.indexedAt and indexedCommitHash on completion', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - // The onJobCompleted handler reads the job via findUniqueOrThrow, then marks it - // COMPLETED and updates the repo (indexedAt, etc.) in a single repoIndexingJob.update - // with a nested repo update. - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repoId: repo.id, - repo, - metadata: {}, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - - // Get the onCompleted handler - const onCompletedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'completed')?.[1]; - expect(onCompletedHandler).toBeDefined(); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - } as unknown as Job; - - await onCompletedHandler(mockJob); - - // The job status and indexedAt must be written together (single transaction) to - // close the race where the scheduler sees a completed job but a stale indexedAt. - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.COMPLETED, - completedAt: expect.any(Date), - repo: { - update: expect.objectContaining({ - indexedAt: expect.any(Date), - indexedCommitHash: 'abc123', - }), - }, - }), - }) - ); - }); - }); - - describe('Job Processing - Failure', () => { - test('marks job as FAILED when git clone throws', async () => { - const repo = createMockRepoWithConnections(); - const cloneError = new Error('Clone failed: authentication error'); - (existsSync as Mock).mockReturnValue(false); - (cloneRepository as Mock).mockRejectedValue(cloneError); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock) - .mockResolvedValueOnce({ type: RepoIndexingJobType.INDEX, repo }) - .mockResolvedValueOnce({ repo }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - getState: vi.fn().mockResolvedValue('failed'), - } as unknown as Job; - - // Get the onFailed handler - const onFailedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'failed')?.[1]; - expect(onFailedHandler).toBeDefined(); - - await onFailedHandler(mockJob, cloneError); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.FAILED, - errorMessage: cloneError.message, - }), - }) - ); - - expect(mockPromClient.repoIndexJobFailTotal.inc).toHaveBeenCalledWith({ - repo: repo.name, - type: 'index', - }); - }); - - test('marks job as FAILED when zoekt-git-index fails', async () => { - const repo = createMockRepoWithConnections(); - const indexError = new Error('zoekt-git-index: failed to index'); - (existsSync as Mock).mockReturnValue(true); - (indexGitRepository as Mock).mockRejectedValue(indexError); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock) - .mockResolvedValueOnce({ type: RepoIndexingJobType.INDEX, repo }) - .mockResolvedValueOnce({ repo }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - getState: vi.fn().mockResolvedValue('failed'), - } as unknown as Job; - - const onFailedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'failed')?.[1]; - await onFailedHandler(mockJob, indexError); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.FAILED, - errorMessage: indexError.message, - }), - }) - ); - }); - }); - - describe('Concurrency Control', () => { - test('prevents concurrent jobs for the same repo via redlock', async () => { - const repo = createMockRepoWithConnections(); - - // Simulate lock acquisition failure - mockRedlockUsing.mockRejectedValue(new ExecutionError('Lock already held')); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn().mockResolvedValue(undefined), - token: 'test-token', - } as unknown as Job; - - const { Worker, DelayedError } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - - // The processor should throw a DelayedError when lock cannot be acquired - await expect(processor(mockJob)).rejects.toThrow('locked'); - - // Verify moveToDelayed was called to retry later - expect(mockJob.moveToDelayed).toHaveBeenCalled(); - }); - }); - - describe('Cleanup Jobs', () => { - test('deletes repo directory and index shards', async () => { - const repo = createMockRepoWithConnections({ id: 5, orgId: 2 }); - (existsSync as Mock).mockReturnValue(true); - (readdir as Mock).mockResolvedValue(['2_5_v1.zoekt', '2_5_v2.zoekt', 'other_file.zoekt']); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.CLEANUP, - repo, - }); - - const mockJob = { - data: { - jobId: 'cleanup-job-1', - type: 'CLEANUP', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - // Should delete the repo directory - expect(rm).toHaveBeenCalledWith( - expect.stringContaining(`${repo.id}`), - { recursive: true, force: true } - ); - - // Should delete shard files matching the prefix - expect(rm).toHaveBeenCalledWith( - expect.stringContaining('2_5_v1.zoekt'), - { force: true } - ); - expect(rm).toHaveBeenCalledWith( - expect.stringContaining('2_5_v2.zoekt'), - { force: true } - ); - }); - - test('removes repo from database after cleanup', async () => { - const repo = createMockRepoWithConnections({ id: 3 }); - (existsSync as Mock).mockReturnValue(false); - (readdir as Mock).mockResolvedValue([]); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - type: RepoIndexingJobType.CLEANUP, - repoId: repo.id, - repo, - metadata: {}, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - (mockPrisma.repo.delete as Mock).mockResolvedValue(repo); - - const onCompletedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'completed')?.[1]; - - const mockJob = { - data: { - jobId: 'cleanup-job-1', - type: 'CLEANUP', - repoId: repo.id, - repoName: repo.name, - }, - } as unknown as Job; - - await onCompletedHandler(mockJob); - - expect(mockPrisma.repo.delete).toHaveBeenCalledWith({ - where: { id: repo.id }, - }); - - expect(mockPromClient.repoIndexJobSuccessTotal.inc).toHaveBeenCalledWith({ - repo: repo.name, - type: 'cleanup', - }); - }); - }); - - describe('latestIndexingJobStatus Updates', () => { - test('sets latestIndexingJobStatus to IN_PROGRESS when job starts', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - // Ensure indexGitRepository resolves for this test - (indexGitRepository as Mock).mockResolvedValue({ stdout: '', stderr: '' }); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - // Verify the first update call sets latestIndexingJobStatus to IN_PROGRESS - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.IN_PROGRESS, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, - }, - }, - }), - }) - ); - }); - - test('sets latestIndexingJobStatus to COMPLETED when job succeeds', async () => { - const repo = createMockRepoWithConnections(); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repoId: repo.id, - repo, - metadata: {}, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - - const onCompletedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'completed')?.[1]; - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - } as unknown as Job; - - await onCompletedHandler(mockJob); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.COMPLETED, - repo: { - update: expect.objectContaining({ - latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, - }), - }, - }), - }) - ); - }); - - test('sets latestIndexingJobStatus to FAILED when job fails', async () => { - const repo = createMockRepoWithConnections(); - const error = new Error('Job processing failed'); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - - const onFailedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'failed')?.[1]; - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - getState: vi.fn().mockResolvedValue('failed'), - } as unknown as Job; - - await onFailedHandler(mockJob, error); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.FAILED, - errorMessage: error.message, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, - }, - }, - }), - }) - ); - }); - }); -}); diff --git a/packages/backend/src/repoIndexManager.ts b/packages/backend/src/repoIndexManager.ts deleted file mode 100644 index aea1291dc..000000000 --- a/packages/backend/src/repoIndexManager.ts +++ /dev/null @@ -1,769 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; -import { createLogger, env, getRepoPath, Logger, getRepoIdFromPath, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata, repoMetadataSchema } from "@sourcebot/shared"; -import { DelayedError, Job, Queue, Worker } from "bullmq"; -import { existsSync } from 'fs'; -import { readdir, rm } from 'fs/promises'; -import { Redis } from 'ioredis'; -import micromatch from 'micromatch'; -import Redlock, { ExecutionError } from 'redlock'; -import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, WORKER_STOP_GRACEFUL_TIMEOUT_MS } from './constants.js'; -import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getLatestCommitTimestamp, getLocalDefaultBranch, getTags, isPathAValidGitRepoRoot, isRepoEmpty, unsetGitConfig, upsertGitConfig, writeCommitGraph } from './git.js'; -import { captureEvent } from './posthog.js'; -import { PromClient } from './promClient.js'; -import { RepoWithConnections, Settings } from "./types.js"; -import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, getShardPrefix, measure, setIntervalAsync } from './utils.js'; -import { cleanupTempShards, indexGitRepository } from './zoekt.js'; - -const LOG_TAG = 'repo-index-manager'; -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); -const QUEUE_NAME = 'repo-index-queue'; - -type JobPayload = { - type: 'INDEX' | 'CLEANUP'; - jobId: string; - repoId: number; - repoName: string; -}; - -// Lock TTL with auto-extension - minimizes dead lock time after crashes -const LOCK_TTL_MS = 60 * 1000; // 1 minute -const LOCK_PREFIX = `bullmq:${QUEUE_NAME}:lock:`; - -// Delay before retrying a job when the group lock cannot be acquired -const LOCK_RETRY_DELAY_MS = 5000; - -/** - * Manages the lifecycle of repository data on disk, including git working copies - * and search index shards. Handles both indexing operations (cloning/fetching repos - * and building search indexes) and cleanup operations (removing orphaned repos and - * their associated data). - * - * Uses a job queue system to process indexing and cleanup tasks asynchronously, - * with configurable concurrency limits and retry logic. Automatically schedules - * re-indexing of repos based on configured intervals and manages garbage collection - * of repos that are no longer connected to any source. - */ -export class RepoIndexManager { - private interval?: NodeJS.Timeout; - private queue: Queue; - private worker: Worker; - private redlock: Redlock; - private abortController: AbortController; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - private promClient: PromClient, - ) { - this.abortController = new AbortController(); - - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - defaultJobOptions: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - attempts: 2, - }, - }); - - this.redlock = new Redlock([redis], { - retryCount: 0, // Don't retry - we'll delay the job instead - automaticExtensionThreshold: LOCK_TTL_MS / 2, // Extend when 50% of TTL remains - }); - - this.worker = new Worker( - QUEUE_NAME, - this.processJob.bind(this), - { - connection: redis, - concurrency: this.settings.maxRepoIndexingJobConcurrency, - maxStalledCount: 1, - } - ); - - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobMaybeFailed.bind(this)); - this.worker.on('stalled', (jobId) => { - // Just log - BullMQ will automatically retry the job (up to maxStalledCount times). - // If all retries fail, onJobMaybeFailed will handle marking it as failed. - logger.warn(`Job ${jobId} stalled - BullMQ will retry`); - }); - this.worker.on('error', (error) => { - logger.error(`Index syncer worker error:`, error); - }); - } - - public async startScheduler() { - logger.debug('Starting scheduler'); - // Cleanup any orphaned disk resources on startup - await this.cleanupOrphanedDiskResources(); - this.interval = setIntervalAsync(async () => { - await this.scheduleIndexJobs(); - await this.scheduleCleanupJobs(); - }, this.settings.reindexRepoPollingIntervalMs); - } - - private async scheduleIndexJobs() { - const thresholdDate = new Date(Date.now() - this.settings.reindexIntervalMs); - const timeoutDate = new Date(Date.now() - this.settings.repoIndexTimeoutMs); - - const reposToIndex = await this.db.repo.findMany({ - where: { - AND: [ - { - OR: [ - { indexedAt: null }, - { indexedAt: { lt: thresholdDate } }, - ] - }, - { - NOT: { - jobs: { - some: { - AND: [ - { - type: RepoIndexingJobType.INDEX, - }, - { - OR: [ - // Don't schedule if there are active jobs that were created within the threshold date. - // This handles the case where a job is stuck in a pending state and will never be scheduled. - { - AND: [ - { - status: { - in: [ - RepoIndexingJobStatus.PENDING, - RepoIndexingJobStatus.IN_PROGRESS, - ] - }, - }, - { - createdAt: { - gt: timeoutDate, - } - } - ] - }, - // Don't schedule if there are recent failed jobs (within the threshold date). - { - AND: [ - { status: RepoIndexingJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - ] - } - } - } - } - ], - }, - }); - - if (reposToIndex.length > 0) { - await this.createJobs(reposToIndex, RepoIndexingJobType.INDEX); - } - } - - private async scheduleCleanupJobs() { - const gcGracePeriodMs = new Date(Date.now() - this.settings.repoGarbageCollectionGracePeriodMs); - const timeoutDate = new Date(Date.now() - this.settings.repoIndexTimeoutMs); - - const reposToCleanup = await this.db.repo.findMany({ - where: { - connections: { - none: {} - }, - isAutoCleanupDisabled: false, - OR: [ - { indexedAt: null }, - { indexedAt: { lt: gcGracePeriodMs } }, - ], - NOT: { - jobs: { - some: { - AND: [ - { - type: RepoIndexingJobType.CLEANUP, - }, - { - status: { - in: [ - RepoIndexingJobStatus.PENDING, - RepoIndexingJobStatus.IN_PROGRESS, - ] - }, - }, - { - createdAt: { - gt: timeoutDate, - } - } - ] - } - } - } - } - }); - - if (reposToCleanup.length > 0) { - await this.createJobs(reposToCleanup, RepoIndexingJobType.CLEANUP); - } - } - - public async createJobs(repos: Repo[], type: RepoIndexingJobType) { - // @note: we don't perform this in a transaction because - // we want to avoid the situation where a job is created and run - // prior to the transaction being committed. - const jobs = await this.db.repoIndexingJob.createManyAndReturn({ - data: repos.map(repo => ({ - type, - repoId: repo.id, - })), - include: { - repo: true, - } - }); - - for (const job of jobs) { - await this.queue.add( - 'repo-index-job', - { - jobId: job.id, - type, - repoName: job.repo.name, - repoId: job.repo.id, - }, - { jobId: job.id } - ); - - const jobTypeLabel = getJobTypePrometheusLabel(type); - this.promClient.pendingRepoIndexJobs.inc({ repo: job.repo.name, type: jobTypeLabel }); - } - - return jobs.map(job => job.id); - } - - private async processJob(job: Job): Promise { - const groupId = `repo:${job.data.repoId}`; - const lockKey = `${LOCK_PREFIX}${groupId}`; - - try { - return await this.redlock.using([lockKey], LOCK_TTL_MS, async (lockSignal: AbortSignal) => { - const signal = AbortSignal.any([ - this.abortController.signal, - lockSignal, - ]); - - return await this.runJob(job, signal); - }); - } catch (error) { - if (error instanceof ExecutionError) { - // Lock could not be acquired - another job for this group is running - // Delay this job and let BullMQ retry later - // DelayedError tells BullMQ to delay without counting as a failed attempt - logger.debug(`Group ${groupId} locked, delaying job ${job.id}`); - await job.moveToDelayed(Date.now() + LOCK_RETRY_DELAY_MS, job.token); - throw new DelayedError(`Group ${groupId} locked, delaying job`); - } - throw error; - } - } - - private async runJob(job: Job, signal: AbortSignal) { - const id = job.data.jobId; - const logger = createJobLogger(id); - logger.debug(`Running ${job.data.type} job ${id} for repo ${job.data.repoName} (id: ${job.data.repoId})`); - - const currentStatus = await this.db.repoIndexingJob.findUniqueOrThrow({ - where: { - id, - }, - select: { - status: true, - } - }); - - // Fail safe: if the job is not PENDING (first run) or IN_PROGRESS (retry), it indicates the job - // is in an invalid state and should be skipped. - if ( - currentStatus.status !== RepoIndexingJobStatus.PENDING && - currentStatus.status !== RepoIndexingJobStatus.IN_PROGRESS - ) { - throw new Error(`Job ${id} is not in a valid state. Expected: ${RepoIndexingJobStatus.PENDING} or ${RepoIndexingJobStatus.IN_PROGRESS}. Actual: ${currentStatus.status}. Skipping.`); - } - - const { repo, type: jobType } = await this.db.repoIndexingJob.update({ - where: { - id, - }, - data: { - status: RepoIndexingJobStatus.IN_PROGRESS, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, - } - } - }, - select: { - type: true, - repo: { - include: { - connections: { - include: { - connection: true, - } - } - } - } - } - }); - - const jobTypeLabel = getJobTypePrometheusLabel(jobType); - this.promClient.pendingRepoIndexJobs.dec({ repo: job.data.repoName, type: jobTypeLabel }); - this.promClient.activeRepoIndexJobs.inc({ repo: job.data.repoName, type: jobTypeLabel }); - - if (jobType === RepoIndexingJobType.INDEX) { - const revisions = await this.indexRepository(repo, logger, signal); - - await this.db.repoIndexingJob.update({ - where: { id }, - data: { - metadata: { - indexedRevisions: revisions, - } satisfies RepoIndexingJobMetadata, - }, - }); - } else if (jobType === RepoIndexingJobType.CLEANUP) { - await this.cleanupRepository(repo, logger); - } - - } - - private async indexRepository(repo: RepoWithConnections, logger: Logger, signal: AbortSignal) { - const { path: repoPath, isReadOnly } = getRepoPath(repo); - - const metadata = repoMetadataSchema.parse(repo.metadata); - - const credentials = await getAuthCredentialsForRepo(repo, logger); - const cloneUrlMaybeWithToken = credentials?.cloneUrlWithToken ?? repo.cloneUrl; - const authHeader = credentials?.authHeader ?? undefined; - - // If the repo path exists but it is not a valid git repository root, this indicates - // that the repository is in a bad state. To fix, we remove the directory and perform - // a fresh clone. - if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) { - const isValidGitRepo = await isPathAValidGitRepoRoot({ - path: repoPath, - signal, - }); - - if (!isValidGitRepo && !isReadOnly) { - logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`); - await rm(repoPath, { recursive: true, force: true }); - } - } - - if (existsSync(repoPath) && !isReadOnly) { - // @NOTE: in #483, we changed the cloning method s.t., we _no longer_ - // write the clone URL (which could contain a auth token) to the - // `remote.origin.url` entry. For the upgrade scenario, we want - // to unset this key since it is no longer needed, hence this line. - // This will no-op if the key is already unset. - // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 - await unsetGitConfig({ - path: repoPath, - keys: ["remote.origin.url"], - signal, - }); - - logger.debug(`Fetching ${repo.name} (id: ${repo.id})...`); - const { durationMs } = await measure(() => fetchRepository({ - cloneUrl: cloneUrlMaybeWithToken, - authHeader, - path: repoPath, - onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) - }, - signal, - })); - const fetchDuration_s = durationMs / 1000; - - logger.debug(`Fetched ${repo.name} (id: ${repo.id}) in ${fetchDuration_s}s`); - - // Update the commit-graph after fetch. Force a full backfill the first time we - // see this repo after the --changed-paths rollout, so historical commits get - // Bloom filters. Subsequent fetches do a cheap incremental write. - const needsBackfill = !metadata.commitGraphChangedPathsBackfilledAt; - if (needsBackfill) { - logger.debug(`Backfilling changed-path Bloom filters for ${repo.name} (id: ${repo.id})...`); - } - await writeCommitGraph({ - path: repoPath, - forceBackfill: needsBackfill, - signal, - }); - } else if (!isReadOnly) { - logger.debug(`Cloning ${repo.name} (id: ${repo.id})...`); - - const { durationMs } = await measure(() => cloneRepository({ - cloneUrl: cloneUrlMaybeWithToken, - authHeader, - path: repoPath, - onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) - }, - signal - })); - const cloneDuration_s = durationMs / 1000; - - logger.debug(`Cloned ${repo.name} (id: ${repo.id}) in ${cloneDuration_s}s`); - - // Write the commit-graph for the freshly cloned repo. - await writeCommitGraph({ - path: repoPath, - signal, - }); - } - - // Record that this repo's commit-graph now includes changed-path Bloom filters - // for its full history (either freshly written during clone, or backfilled above - // during fetch). - if (!isReadOnly && !metadata.commitGraphChangedPathsBackfilledAt) { - await this.db.repo.update({ - where: { id: repo.id }, - data: { - metadata: { - ...metadata, - commitGraphChangedPathsBackfilledAt: new Date().toISOString(), - } satisfies RepoMetadata, - }, - }); - } - - // Regardless of clone or fetch, always upsert the git config for the repo. - // This ensures that the git config is always up to date for whatever we - // have in the DB. - if (metadata.gitConfig && !isReadOnly) { - await upsertGitConfig({ - path: repoPath, - gitConfig: metadata.gitConfig, - signal, - }); - } - - const defaultBranch = await getLocalDefaultBranch({ - path: repoPath, - }); - - // Ensure defaultBranch has refs/heads/ prefix for consistent searching - const defaultBranchWithPrefix = defaultBranch && !defaultBranch.startsWith('refs/') - ? `refs/heads/${defaultBranch}` - : defaultBranch; - - let revisions = defaultBranchWithPrefix ? [defaultBranchWithPrefix] : ['HEAD']; - - if (metadata.branches) { - const branchGlobs = metadata.branches - const allBranches = await getBranches(repoPath); - const matchingBranches = - allBranches - .filter((branch) => micromatch.isMatch(branch, branchGlobs)) - .map((branch) => `refs/heads/${branch}`); - - revisions = [ - ...revisions, - ...matchingBranches - ]; - } - - if (metadata.tags) { - const tagGlobs = metadata.tags; - const allTags = await getTags(repoPath); - const matchingTags = - allTags - .filter((tag) => micromatch.isMatch(tag, tagGlobs)) - .map((tag) => `refs/tags/${tag}`); - - revisions = [ - ...revisions, - ...matchingTags - ]; - } - - // De-duplicate revisions to ensure we don't have duplicate branches/tags - revisions = [...new Set(revisions)]; - - // zoekt has a limit of 64 branches/tags to index. - if (revisions.length > 64) { - logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`); - captureEvent('backend_revisions_truncated', { - repoId: repo.id, - revisionCount: revisions.length, - }); - revisions = revisions.slice(0, 64); - } - - logger.debug(`Indexing ${repo.name} (id: ${repo.id})...`); - try { - const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal)); - const indexDuration_s = durationMs / 1000; - logger.debug(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`); - } catch (error) { - // Clean up any temporary shard files left behind by the failed indexing operation. - // Zoekt creates .tmp files during indexing which can accumulate if indexing fails repeatedly. - logger.warn(`Indexing failed for ${repo.name} (id: ${repo.id}), cleaning up temp shard files...`); - await cleanupTempShards(repo); - throw error; - } - - return revisions; - } - - private async cleanupRepository(repo: Repo, logger: Logger) { - const { path: repoPath, isReadOnly } = getRepoPath(repo); - if (existsSync(repoPath) && !isReadOnly) { - logger.debug(`Deleting repo directory ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - - const shardPrefix = getShardPrefix(repo.orgId, repo.id); - const files = (await readdir(INDEX_CACHE_DIR)).filter(file => file.startsWith(shardPrefix)); - for (const file of files) { - const filePath = `${INDEX_CACHE_DIR}/${file}`; - logger.debug(`Deleting shard file ${filePath}`); - await rm(filePath, { force: true }); - } - } - - private async onJobCompleted(job: Job) { - try { - const logger = createJobLogger(job.data.jobId); - const jobData = await this.db.repoIndexingJob.findUniqueOrThrow({ - where: { id: job.data.jobId }, - include: { - repo: true, - } - }); - - const jobTypeLabel = getJobTypePrometheusLabel(jobData.type); - // @note: capture this before the update below, since the update sets indexedAt. - const isFirstIndex = jobData.repo.indexedAt === null; - - if (jobData.type === RepoIndexingJobType.INDEX) { - const { path: repoPath } = getRepoPath(jobData.repo); - const isEmpty = await isRepoEmpty({ path: repoPath }); - const commitHash = isEmpty ? undefined : await getCommitHashForRefName({ - path: repoPath, - refName: 'HEAD', - }); - - const pushedAt = await getLatestCommitTimestamp({ path: repoPath }); - const defaultBranch = await getLocalDefaultBranch({ path: repoPath }); - - const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata); - - const { repo } = await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.COMPLETED, - completedAt: new Date(), - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, - indexedAt: new Date(), - indexedCommitHash: commitHash, - pushedAt: pushedAt, - metadata: { - ...(jobData.repo.metadata as RepoMetadata), - indexedRevisions: jobMetadata.indexedRevisions, - } satisfies RepoMetadata, - // @note: always update the default branch. While this field can be set - // during connection syncing, by setting it here we ensure that a) the - // default branch is as up to date as possible (since repo indexing happens - // more frequently than connection syncing) and b) for hosts where it is - // impossible to determine the default branch from the host's API - // (e.g., generic git url), we still set the default branch here. - defaultBranch: defaultBranch, - } - } - }, - include: { - repo: true, - } - }); - - logger.debug(`Completed index job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); - } - else if (jobData.type === RepoIndexingJobType.CLEANUP) { - await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.COMPLETED, - completedAt: new Date(), - } - }); - - const repo = await this.db.repo.delete({ - where: { id: jobData.repoId }, - }); - - logger.debug(`Completed cleanup job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); - } - - // Track metrics for successful job - this.promClient.activeRepoIndexJobs.dec({ repo: job.data.repoName, type: jobTypeLabel }); - this.promClient.repoIndexJobSuccessTotal.inc({ repo: job.data.repoName, type: jobTypeLabel }); - - if (jobData.type === RepoIndexingJobType.INDEX && isFirstIndex) { - captureEvent('backend_repo_first_indexed', { - repoId: job.data.repoId, - type: jobData.repo.external_codeHostType, - }); - } - } catch (error) { - Sentry.captureException(error); - logger.error(`Exception thrown while executing lifecycle function \`onJobCompleted\`.`, error); - } - } - - private async onJobMaybeFailed(job: Job | undefined, error: Error) { - try { - if (!job) { - logger.error(`Job failed but job object is undefined. Error: ${error.message}`); - return; - } - - const jobLogger = createJobLogger(job.data.jobId); - const jobTypeLabel = getJobTypePrometheusLabel(job.data.type); - - // @note: we need to check the job state to determine if the job failed, - // or if it is being retried. - const jobState = await job.getState(); - if (jobState !== 'failed') { - jobLogger.warn(`Job ${job.id} for repo ${job.data.repoName} (id: ${job.data.repoId}) failed. Retrying... Reason: ${error.message}`); - return; - } - - const { repo } = await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, - } - } - }, - select: { repo: true } - }); - - this.promClient.activeRepoIndexJobs.dec({ repo: job.data.repoName, type: jobTypeLabel }); - this.promClient.repoIndexJobFailTotal.inc({ repo: job.data.repoName, type: jobTypeLabel }); - - jobLogger.error(`Failed job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id}). Reason: ${error.message}`); - - captureEvent('backend_repo_index_job_failed', { - repoId: job.data.repoId, - jobType: job.data.type, - type: repo.external_codeHostType, - }); - } catch (err) { - Sentry.captureException(err); - logger.error(`Exception thrown while executing lifecycle function \`onJobMaybeFailed\`.`, err); - } - } - - // Scans the repos and index directories on disk and removes any entries - // that have no corresponding Repo record in the database. This handles - // edge cases where the DB and disk resources are out of sync. - private async cleanupOrphanedDiskResources() { - // --- Repo directories --- - // Dirs are named by repoId: DATA_CACHE_DIR/repos// - if (existsSync(REPOS_CACHE_DIR)) { - const entries = await readdir(REPOS_CACHE_DIR); - const repoIdToPath = new Map(); - for (const entry of entries) { - const repoPath = `${REPOS_CACHE_DIR}/${entry}`; - const repoId = getRepoIdFromPath(repoPath); - if (repoId !== undefined) { - repoIdToPath.set(repoId, repoPath); - } - } - - if (repoIdToPath.size > 0) { - const existingRepos = await this.db.repo.findMany({ - where: { id: { in: [...repoIdToPath.keys()] } }, - select: { id: true }, - }); - const existingIds = new Set(existingRepos.map(r => r.id)); - for (const [repoId, repoPath] of repoIdToPath) { - if (!existingIds.has(repoId)) { - logger.debug(`Removing orphaned repo directory with no DB record: ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - } - } - } - - // --- Index shards --- - // Shard files are prefixed with _: DATA_CACHE_DIR/index/__*.zoekt - if (existsSync(INDEX_CACHE_DIR)) { - const entries = await readdir(INDEX_CACHE_DIR); - const repoIdToShards = new Map(); - for (const entry of entries) { - const repoId = getRepoIdFromShardFileName(entry); - if (repoId !== undefined) { - const shards = repoIdToShards.get(repoId) ?? []; - shards.push(entry); - repoIdToShards.set(repoId, shards); - } - } - - if (repoIdToShards.size > 0) { - const existingRepos = await this.db.repo.findMany({ - where: { id: { in: [...repoIdToShards.keys()] } }, - select: { id: true }, - }); - const existingIds = new Set(existingRepos.map(r => r.id)); - for (const [repoId, shards] of repoIdToShards) { - if (!existingIds.has(repoId)) { - for (const entry of shards) { - const shardPath = `${INDEX_CACHE_DIR}/${entry}`; - logger.debug(`Removing orphaned index shard with no DB record: ${shardPath}`); - await rm(shardPath, { force: true }); - } - } - } - } - } - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - - // Signal all active jobs to abort - this.abortController.abort(); - - // Wait for worker to finish with timeout - await Promise.race([ - this.worker.close(), - new Promise(resolve => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)) - ]); - - // Locks will auto-expire via TTL, no need to manually release them - await this.queue.close(); - } -} - -const getJobTypePrometheusLabel = (type: RepoIndexingJobType) => type === RepoIndexingJobType.INDEX ? 'index' : 'cleanup'; diff --git a/packages/backend/src/repoIndexWorkload.ts b/packages/backend/src/repoIndexWorkload.ts new file mode 100644 index 000000000..a2a66e77d --- /dev/null +++ b/packages/backend/src/repoIndexWorkload.ts @@ -0,0 +1,351 @@ +import { PrismaClient, Repo } from "@sourcebot/db"; +import { createLogger, getRepoPath, JobLogSink, getRepoIdFromPath, RepoMetadata, repoMetadataSchema, REPO_INDEX_QUEUE } from "@sourcebot/shared"; +import { existsSync } from 'fs'; +import { readdir, rm } from 'fs/promises'; +import micromatch from 'micromatch'; +import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js'; +import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getLatestCommitTimestamp, getLocalDefaultBranch, getTags, isPathAValidGitRepoRoot, isRepoEmpty, unsetGitConfig, upsertGitConfig, writeCommitGraph } from './git.js'; +import { captureEvent } from './posthog.js'; +import { RepoWithConnections, Settings, Workload } from "./types.js"; +import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, getShardPrefix, measure } from './utils.js'; +import { cleanupTempShards, indexGitRepository } from './zoekt.js'; + +const LOG_TAG = 'repo-index-workload'; +const logger = createLogger(LOG_TAG); + +interface RepoIndexWorkloadDependencies { + db: PrismaClient; + settings: Settings; +} + +export const createRepoIndexWorkload = ({ + db, + settings, +}: RepoIndexWorkloadDependencies): Workload<'repo-index'> => ({ + queueSpec: REPO_INDEX_QUEUE, + concurrency: settings.maxRepoIndexingJobConcurrency, + process: async ({ data, logger: jobLogger, signal }) => { + const repo = await db.repo.findUniqueOrThrow({ + where: { id: data.repoId }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + + jobLogger.debug(`Running ${data.type} job for repo ${repo.name} (id: ${repo.id})`); + + if (data.type === 'CLEANUP') { + await cleanupRepository(repo, jobLogger); + await db.repo.delete({ + where: { id: repo.id }, + }); + } else { + const isFirstIndex = repo.indexedAt === null; + const revisions = await indexRepository(db, settings, repo, jobLogger, signal); + const { path: repoPath } = getRepoPath(repo); + const isEmpty = await isRepoEmpty({ path: repoPath }); + const commitHash = isEmpty ? undefined : await getCommitHashForRefName({ + path: repoPath, + refName: 'HEAD', + }); + const pushedAt = await getLatestCommitTimestamp({ path: repoPath }); + const defaultBranch = await getLocalDefaultBranch({ path: repoPath }); + const currentRepo = await db.repo.findUniqueOrThrow({ + where: { id: repo.id }, + select: { metadata: true }, + }); + + await db.repo.update({ + where: { id: repo.id }, + data: { + indexedAt: new Date(), + indexedCommitHash: commitHash, + pushedAt, + metadata: { + ...(currentRepo.metadata as RepoMetadata), + indexedRevisions: revisions, + } satisfies RepoMetadata, + defaultBranch, + }, + }); + + if (isFirstIndex) { + captureEvent('backend_repo_first_indexed', { + repoId: repo.id, + type: repo.external_codeHostType, + }); + } + } + }, +}); + +const indexRepository = async ( + db: PrismaClient, + settings: Settings, + repo: RepoWithConnections, + logger: JobLogSink, + signal: AbortSignal, +) => { + const { path: repoPath, isReadOnly } = getRepoPath(repo); + + const metadata = repoMetadataSchema.parse(repo.metadata); + + const credentials = await getAuthCredentialsForRepo(repo, logger); + const cloneUrlMaybeWithToken = credentials?.cloneUrlWithToken ?? repo.cloneUrl; + const authHeader = credentials?.authHeader ?? undefined; + + // If the repo path exists but it is not a valid git repository root, this indicates + // that the repository is in a bad state. To fix, we remove the directory and perform + // a fresh clone. + if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) { + const isValidGitRepo = await isPathAValidGitRepoRoot({ + path: repoPath, + signal, + }); + + if (!isValidGitRepo && !isReadOnly) { + logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`); + await rm(repoPath, { recursive: true, force: true }); + } + } + + if (existsSync(repoPath) && !isReadOnly) { + // @NOTE: in #483, we changed the cloning method s.t., we _no longer_ + // write the clone URL (which could contain a auth token) to the + // `remote.origin.url` entry. For the upgrade scenario, we want + // to unset this key since it is no longer needed, hence this line. + // This will no-op if the key is already unset. + // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 + await unsetGitConfig({ + path: repoPath, + keys: ["remote.origin.url"], + signal, + }); + + logger.debug(`Fetching ${repo.name} (id: ${repo.id})...`); + const { durationMs } = await measure(() => fetchRepository({ + cloneUrl: cloneUrlMaybeWithToken, + authHeader, + path: repoPath, + onProgress: ({ method, stage, progress }) => { + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) + }, + signal, + })); + const fetchDuration_s = durationMs / 1000; + + logger.debug(`Fetched ${repo.name} (id: ${repo.id}) in ${fetchDuration_s}s`); + + // Update the commit-graph after fetch. Force a full backfill the first time we + // see this repo after the --changed-paths rollout, so historical commits get + // Bloom filters. Subsequent fetches do a cheap incremental write. + const needsBackfill = !metadata.commitGraphChangedPathsBackfilledAt; + if (needsBackfill) { + logger.debug(`Backfilling changed-path Bloom filters for ${repo.name} (id: ${repo.id})...`); + } + await writeCommitGraph({ + path: repoPath, + forceBackfill: needsBackfill, + signal, + }); + } else if (!isReadOnly) { + logger.debug(`Cloning ${repo.name} (id: ${repo.id})...`); + + const { durationMs } = await measure(() => cloneRepository({ + cloneUrl: cloneUrlMaybeWithToken, + authHeader, + path: repoPath, + onProgress: ({ method, stage, progress }) => { + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) + }, + signal + })); + const cloneDuration_s = durationMs / 1000; + + logger.debug(`Cloned ${repo.name} (id: ${repo.id}) in ${cloneDuration_s}s`); + + // Write the commit-graph for the freshly cloned repo. + await writeCommitGraph({ + path: repoPath, + signal, + }); + } + + // Record that this repo's commit-graph now includes changed-path Bloom filters + // for its full history (either freshly written during clone, or backfilled above + // during fetch). + if (!isReadOnly && !metadata.commitGraphChangedPathsBackfilledAt) { + await db.repo.update({ + where: { id: repo.id }, + data: { + metadata: { + ...metadata, + commitGraphChangedPathsBackfilledAt: new Date().toISOString(), + } satisfies RepoMetadata, + }, + }); + } + + // Regardless of clone or fetch, always upsert the git config for the repo. + // This ensures that the git config is always up to date for whatever we + // have in the DB. + if (metadata.gitConfig && !isReadOnly) { + await upsertGitConfig({ + path: repoPath, + gitConfig: metadata.gitConfig, + signal, + }); + } + + const defaultBranch = await getLocalDefaultBranch({ + path: repoPath, + }); + + // Ensure defaultBranch has refs/heads/ prefix for consistent searching + const defaultBranchWithPrefix = defaultBranch && !defaultBranch.startsWith('refs/') + ? `refs/heads/${defaultBranch}` + : defaultBranch; + + let revisions = defaultBranchWithPrefix ? [defaultBranchWithPrefix] : ['HEAD']; + + if (metadata.branches) { + const branchGlobs = metadata.branches + const allBranches = await getBranches(repoPath); + const matchingBranches = + allBranches + .filter((branch) => micromatch.isMatch(branch, branchGlobs)) + .map((branch) => `refs/heads/${branch}`); + + revisions = [ + ...revisions, + ...matchingBranches + ]; + } + + if (metadata.tags) { + const tagGlobs = metadata.tags; + const allTags = await getTags(repoPath); + const matchingTags = + allTags + .filter((tag) => micromatch.isMatch(tag, tagGlobs)) + .map((tag) => `refs/tags/${tag}`); + + revisions = [ + ...revisions, + ...matchingTags + ]; + } + + // De-duplicate revisions to ensure we don't have duplicate branches/tags + revisions = [...new Set(revisions)]; + + // zoekt has a limit of 64 branches/tags to index. + if (revisions.length > 64) { + logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`); + captureEvent('backend_revisions_truncated', { + repoId: repo.id, + revisionCount: revisions.length, + }); + revisions = revisions.slice(0, 64); + } + + logger.debug(`Indexing ${repo.name} (id: ${repo.id})...`); + try { + const { durationMs } = await measure(() => indexGitRepository(repo, settings, revisions, signal)); + const indexDuration_s = durationMs / 1000; + logger.debug(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`); + } catch (error) { + // Clean up any temporary shard files left behind by the failed indexing operation. + // Zoekt creates .tmp files during indexing which can accumulate if indexing fails repeatedly. + logger.warn(`Indexing failed for ${repo.name} (id: ${repo.id}), cleaning up temp shard files...`); + await cleanupTempShards(repo); + throw error; + } + + return revisions; +}; + +const cleanupRepository = async (repo: Repo, logger: JobLogSink) => { + const { path: repoPath, isReadOnly } = getRepoPath(repo); + if (existsSync(repoPath) && !isReadOnly) { + logger.debug(`Deleting repo directory ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + + const shardPrefix = getShardPrefix(repo.orgId, repo.id); + const files = (await readdir(INDEX_CACHE_DIR)).filter(file => file.startsWith(shardPrefix)); + for (const file of files) { + const filePath = `${INDEX_CACHE_DIR}/${file}`; + logger.debug(`Deleting shard file ${filePath}`); + await rm(filePath, { force: true }); + } +}; + +// Scans the repos and index directories on disk and removes any entries +// that have no corresponding Repo record in the database. This handles +// edge cases where the DB and disk resources are out of sync. +export const cleanupOrphanedRepoResources = async (db: PrismaClient) => { + // --- Repo directories --- + // Dirs are named by repoId: DATA_CACHE_DIR/repos// + if (existsSync(REPOS_CACHE_DIR)) { + const entries = await readdir(REPOS_CACHE_DIR); + const repoIdToPath = new Map(); + for (const entry of entries) { + const repoPath = `${REPOS_CACHE_DIR}/${entry}`; + const repoId = getRepoIdFromPath(repoPath); + if (repoId !== undefined) { + repoIdToPath.set(repoId, repoPath); + } + } + + if (repoIdToPath.size > 0) { + const existingRepos = await db.repo.findMany({ + where: { id: { in: [...repoIdToPath.keys()] } }, + select: { id: true }, + }); + const existingIds = new Set(existingRepos.map(r => r.id)); + for (const [repoId, repoPath] of repoIdToPath) { + if (!existingIds.has(repoId)) { + logger.debug(`Removing orphaned repo directory with no DB record: ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + } + } + } + + // --- Index shards --- + // Shard files are prefixed with _: DATA_CACHE_DIR/index/__*.zoekt + if (existsSync(INDEX_CACHE_DIR)) { + const entries = await readdir(INDEX_CACHE_DIR); + const repoIdToShards = new Map(); + for (const entry of entries) { + const repoId = getRepoIdFromShardFileName(entry); + if (repoId !== undefined) { + const shards = repoIdToShards.get(repoId) ?? []; + shards.push(entry); + repoIdToShards.set(repoId, shards); + } + } + + if (repoIdToShards.size > 0) { + const existingRepos = await db.repo.findMany({ + where: { id: { in: [...repoIdToShards.keys()] } }, + select: { id: true }, + }); + const existingIds = new Set(existingRepos.map(r => r.id)); + for (const [repoId, shards] of repoIdToShards) { + if (!existingIds.has(repoId)) { + for (const entry of shards) { + const shardPath = `${INDEX_CACHE_DIR}/${entry}`; + logger.debug(`Removing orphaned index shard with no DB record: ${shardPath}`); + await rm(shardPath, { force: true }); + } + } + } + } + } +}; diff --git a/packages/backend/src/types/redlock.d.ts b/packages/backend/src/types/redlock.d.ts deleted file mode 100644 index 63c6b5a9b..000000000 --- a/packages/backend/src/types/redlock.d.ts +++ /dev/null @@ -1,95 +0,0 @@ -// Type declarations for redlock -// The redlock package's exports field doesn't include types, so TypeScript can't resolve them. -// This file re-exports the types from the actual .d.ts file. - -declare module 'redlock' { - import { EventEmitter } from "events"; - import { Redis as IORedisClient, Cluster as IORedisCluster } from "ioredis"; - - type Client = IORedisClient | IORedisCluster; - - export type ClientExecutionResult = { - client: Client; - vote: "for"; - value: number; - } | { - client: Client; - vote: "against"; - error: Error; - }; - - export type ExecutionStats = { - readonly membershipSize: number; - readonly quorumSize: number; - readonly votesFor: Set; - readonly votesAgainst: Map; - }; - - export type ExecutionResult = { - attempts: ReadonlyArray>; - }; - - export interface Settings { - readonly driftFactor: number; - readonly retryCount: number; - readonly retryDelay: number; - readonly retryJitter: number; - readonly automaticExtensionThreshold: number; - } - - export class ResourceLockedError extends Error { - readonly message: string; - constructor(message: string); - } - - export class ExecutionError extends Error { - readonly message: string; - readonly attempts: ReadonlyArray>; - constructor(message: string, attempts: ReadonlyArray>); - } - - export class Lock { - readonly redlock: Redlock; - readonly resources: string[]; - readonly value: string; - readonly attempts: ReadonlyArray>; - expiration: number; - constructor(redlock: Redlock, resources: string[], value: string, attempts: ReadonlyArray>, expiration: number); - release(): Promise; - extend(duration: number): Promise; - } - - export type RedlockAbortSignal = AbortSignal & { - error?: Error; - }; - - export default class Redlock extends EventEmitter { - readonly clients: Set; - readonly settings: Settings; - readonly scripts: { - readonly acquireScript: { - value: string; - hash: string; - }; - readonly extendScript: { - value: string; - hash: string; - }; - readonly releaseScript: { - value: string; - hash: string; - }; - }; - constructor(clients: Iterable, settings?: Partial, scripts?: { - readonly acquireScript?: string | ((script: string) => string); - readonly extendScript?: string | ((script: string) => string); - readonly releaseScript?: string | ((script: string) => string); - }); - quit(): Promise; - acquire(resources: string[], duration: number, settings?: Partial): Promise; - release(lock: Lock, settings?: Partial): Promise; - extend(existing: Lock, duration: number, settings?: Partial): Promise; - using(resources: string[], duration: number, settings: Partial, routine?: (signal: RedlockAbortSignal) => Promise): Promise; - using(resources: string[], duration: number, routine: (signal: RedlockAbortSignal) => Promise): Promise; - } -} diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index e50ee0ed5..45c91226b 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -1,7 +1,7 @@ import { Logger } from "winston"; import { RepoAuthCredentials, RepoWithConnections } from "./types.js"; import path from 'path'; -import { getTokenFromConfig } from "@sourcebot/shared"; +import { getTokenFromConfig, JobLogSink } from "@sourcebot/shared"; import * as Sentry from "@sentry/node"; import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; import { GithubAppManager } from "./ee/githubAppManager.js"; @@ -114,7 +114,7 @@ export const fetchWithRetry = async ( // fetch the token here using the connections from the repo. Multiple connections could be referencing this repo, and each // may have their own token. This method will just pick the first connection that has a token (if one exists) and uses that. This // may technically cause syncing to fail if that connection's token just so happens to not have access to the repo it's referencing. -export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, logger?: Logger): Promise => { +export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, logger?: JobLogSink): Promise => { // If we have github apps configured we assume that we must use them for github service auth if (repo.external_codeHostType === 'github' && await hasEntitlement('github-app') && GithubAppManager.getInstance().appsConfigured()) { logger?.debug(`Using GitHub App for service auth for repo ${repo.displayName} hosted at ${repo.external_codeHostUrl}`); diff --git a/packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql b/packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql new file mode 100644 index 000000000..ab019a779 --- /dev/null +++ b/packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql @@ -0,0 +1,22 @@ +/* + Warnings: + + - You are about to drop the column `latestIndexingJobStatus` on the `Repo` table. All the data in the column will be lost. + - You are about to drop the `RepoIndexingJob` table. If the table is not empty, all the data it contains will be lost. + +*/ +-- DropForeignKey +ALTER TABLE "RepoIndexingJob" DROP CONSTRAINT "RepoIndexingJob_repoId_fkey"; + +-- AlterTable +ALTER TABLE "Repo" DROP COLUMN "latestIndexingJobStatus", +ADD COLUMN "latestIndexingJobId" TEXT; + +-- DropTable +DROP TABLE "RepoIndexingJob"; + +-- DropEnum +DROP TYPE "RepoIndexingJobStatus"; + +-- DropEnum +DROP TYPE "RepoIndexingJobType"; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 336a306e9..535833061 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -65,10 +65,10 @@ model Repo { permissionSyncJobs RepoPermissionSyncJob[] permissionSyncedAt DateTime? /// When the permissions were last synced successfully. - jobs RepoIndexingJob[] indexedAt DateTime? /// When the repo was last indexed successfully. + latestIndexingJobId String? + indexedCommitHash String? /// The commit hash of the last indexed commit (on HEAD). - latestIndexingJobStatus RepoIndexingJobStatus? /// The status of the latest indexing job. pushedAt DateTime? /// The timestamp of the most recent commit across all branches. external_id String /// The id of the repo in the external service @@ -86,35 +86,6 @@ model Repo { @@index([indexedAt]) } -enum RepoIndexingJobStatus { - PENDING - IN_PROGRESS - COMPLETED - FAILED -} - -enum RepoIndexingJobType { - INDEX - CLEANUP -} - -model RepoIndexingJob { - id String @id @default(cuid()) - type RepoIndexingJobType - status RepoIndexingJobStatus @default(PENDING) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - completedAt DateTime? - metadata Json? /// For schema see repoIndexingJobMetadataSchema in packages/shared/src/types.ts - - errorMessage String? - - repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade) - repoId Int - - @@index([repoId, type, status]) -} - enum RepoPermissionSyncJobStatus { PENDING IN_PROGRESS diff --git a/packages/db/tools/scripts/inject-repo-data.ts b/packages/db/tools/scripts/inject-repo-data.ts index 609bcdcc7..5540f97b4 100644 --- a/packages/db/tools/scripts/inject-repo-data.ts +++ b/packages/db/tools/scripts/inject-repo-data.ts @@ -2,7 +2,6 @@ import { Script } from "../scriptRunner"; import { PrismaClient } from "../../dist"; const NUM_REPOS = 1000; -const NUM_INDEXING_JOBS_PER_REPO = 10000; const NUM_PERMISSION_JOBS_PER_REPO = 10000; export const injectRepoData: Script = { @@ -37,8 +36,6 @@ export const injectRepoData: Script = { console.log(`Creating ${NUM_REPOS} repos...`); const statuses = ['PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'] as const; - const indexingJobTypes = ['INDEX', 'CLEANUP'] as const; - for (let i = 0; i < NUM_REPOS; i++) { const repo = await prisma.repo.create({ data: { @@ -71,23 +68,8 @@ export const injectRepoData: Script = { } }); } - - for (let j = 0; j < NUM_INDEXING_JOBS_PER_REPO; j++) { - const status = statuses[Math.floor(Math.random() * statuses.length)]; - const type = indexingJobTypes[Math.floor(Math.random() * indexingJobTypes.length)]; - await prisma.repoIndexingJob.create({ - data: { - repoId: repo.id, - type, - status, - completedAt: status === 'COMPLETED' || status === 'FAILED' ? new Date() : null, - errorMessage: status === 'FAILED' ? 'Mock indexing error' : null, - metadata: {} - } - }); - } } - console.log(`Created ${NUM_REPOS} repos with associated jobs.`); + console.log(`Created ${NUM_REPOS} repos with associated permission jobs.`); } -}; \ No newline at end of file +}; diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts index ee165d5b8..c73b1fb5d 100644 --- a/packages/shared/src/bullmqClient.test.ts +++ b/packages/shared/src/bullmqClient.test.ts @@ -2,6 +2,7 @@ import type { Redis } from 'ioredis'; import { beforeEach, describe, expect, test, vi } from 'vitest'; import type { QueueSpec } from './queue.js'; import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from './jobLogger.js'; +import { REPO_INDEX_QUEUE } from './queue.js'; const queueMocks = vi.hoisted(() => ({ add: vi.fn(), @@ -96,6 +97,39 @@ describe('BullMQClient', () => { expect(onEnqueued).not.toHaveBeenCalled(); }); + test('stores the latest repo indexing job id when enqueueing a repo job', async () => { + const repoUpdate = vi.fn(); + const client = new BullMQClient(redis, { + repo: { + update: repoUpdate, + }, + } as unknown as PrismaClient); + + const result = await client.enqueue(REPO_INDEX_QUEUE, { + repoId: 42, + type: 'INDEX', + }); + + expect(queueMocks.add).toHaveBeenCalledWith( + 'repo-index', + { + repoId: 42, + type: 'INDEX', + }, + expect.objectContaining({ + deduplication: { id: 'repo:42' }, + }), + ); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestIndexingJobId: result, + }, + }); + }); + test.each([ ['waiting', 'PENDING'], ['waiting-children', 'PENDING'], diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 51d4c9077..f465e677b 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -19,13 +19,11 @@ export type { } from "./entitlements.js"; export type { RepoMetadata, - RepoIndexingJobMetadata, IdentityProviderType, LicenseStatus, } from "./types.js"; export { repoMetadataSchema, - repoIndexingJobMetadataSchema, } from "./types.js"; export { base64Decode, @@ -101,6 +99,7 @@ export type { export { CONNECTION_QUEUE, RECONCILIATION_QUEUE, + REPO_INDEX_QUEUE, } from "./queue.js"; export { BullMQClient, diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index fb375d711..585d1e457 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -12,6 +12,10 @@ interface QueueRegistry { orgId: number }, 'reconciliation': EmptyJobData, + 'repo-index': { + repoId: number, + type: 'INDEX' | 'CLEANUP', + }, } export const CONNECTION_QUEUE: QueueSpec<'connection'> = { @@ -50,6 +54,31 @@ export const RECONCILIATION_QUEUE: QueueSpec<'reconciliation'> = { }, }; +export const REPO_INDEX_QUEUE: QueueSpec<'repo-index'> = { + name: 'repo-index', + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }, + onEnqueued: async ({ + prisma, + data: { repoId }, + jobId, + }) => { + await prisma.repo.update({ + where: { + id: repoId, + }, + data: { + latestIndexingJobId: jobId, + }, + }); + }, + dedupKey: (data) => `repo:${data.repoId}`, +}; + export interface QueueSpec { name: TName; dedupKey?(data: DataOf): string; diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 5951bab6b..9f93637c9 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -63,15 +63,6 @@ export const repoMetadataSchema = z.object({ export type RepoMetadata = z.infer; -export const repoIndexingJobMetadataSchema = z.object({ - /** - * A list of revisions that were indexed for the repo. - */ - indexedRevisions: z.array(z.string()).optional(), -}); - -export type RepoIndexingJobMetadata = z.infer; - export type IdentityProviderType = IdentityProviderConfig['provider']; // @see: https://docs.stripe.com/api/subscriptions/object#subscription_object-status diff --git a/packages/web/src/app/(app)/chat/chatLandingPage.tsx b/packages/web/src/app/(app)/chat/chatLandingPage.tsx index b78e28e10..fdebccf77 100644 --- a/packages/web/src/app/(app)/chat/chatLandingPage.tsx +++ b/packages/web/src/app/(app)/chat/chatLandingPage.tsx @@ -1,4 +1,4 @@ -import { getRepos, getReposStats, getSearchContexts } from "@/actions"; +import { getRepos, getSearchContexts } from "@/actions"; import { SourcebotLogo } from "@/app/components/sourcebotLogo"; import { getConfiguredLanguageModelsInfo } from "@/features/chat/utils.server"; import { CustomSlateEditor } from "@/features/chat/customSlateEditor"; @@ -35,8 +35,6 @@ export async function ChatLandingPage() { take: 10, }); - const repoStats = await getReposStats(); - if (isServiceError(allRepos)) { throw new ServiceErrorException(allRepos); } @@ -49,10 +47,6 @@ export async function ChatLandingPage() { throw new ServiceErrorException(carouselRepos); } - if (isServiceError(repoStats)) { - throw new ServiceErrorException(repoStats); - } - const demoExamples = env.SOURCEBOT_DEMO_EXAMPLES_PATH ? await (async () => { try { return (await measure(() => loadJsonFile(env.SOURCEBOT_DEMO_EXAMPLES_PATH!, demoExamplesSchema), 'loadExamplesJsonFile')).data; @@ -84,7 +78,8 @@ export async function ChatLandingPage() {
diff --git a/packages/web/src/app/(app)/repos/layout.tsx b/packages/web/src/app/(app)/repos/layout.tsx index 88738d22e..61f3cd34e 100644 --- a/packages/web/src/app/(app)/repos/layout.tsx +++ b/packages/web/src/app/(app)/repos/layout.tsx @@ -1,7 +1,3 @@ -import { getReposStats } from "@/actions"; -import { ServiceErrorException } from "@/lib/serviceError"; -import { isServiceError } from "@/lib/utils"; - interface LayoutProps { children: React.ReactNode; } @@ -11,11 +7,6 @@ export default async function Layout( ) { const { children } = props; - const repoStats = await getReposStats(); - if (isServiceError(repoStats)) { - throw new ServiceErrorException(repoStats); - } - return (
diff --git a/packages/web/src/app/(app)/search/components/searchLandingPage.tsx b/packages/web/src/app/(app)/search/components/searchLandingPage.tsx index e9236607e..a6d072f29 100644 --- a/packages/web/src/app/(app)/search/components/searchLandingPage.tsx +++ b/packages/web/src/app/(app)/search/components/searchLandingPage.tsx @@ -5,7 +5,7 @@ import { SyntaxReferenceGuideHint } from "../../components/syntaxReferenceGuideH import Link from "next/link" import { SearchBar } from "../../components/searchBar" import { SearchModeSelector } from "../../components/searchModeSelector" -import { getRepos, getReposStats } from "@/actions" +import { getRepos } from "@/actions" import { ServiceErrorException } from "@/lib/serviceError" import { isServiceError } from "@/lib/utils" @@ -25,10 +25,7 @@ export const SearchLandingPage = async ({ take: 10, }); - const repoStats = await getReposStats(); - if (isServiceError(carouselRepos)) throw new ServiceErrorException(carouselRepos); - if (isServiceError(repoStats)) throw new ServiceErrorException(repoStats); return (
@@ -55,7 +52,8 @@ export const SearchLandingPage = async ({
diff --git a/yarn.lock b/yarn.lock index ece7d1c39..0982a01a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9194,7 +9194,6 @@ __metadata: p-limit: "npm:^7.2.0" posthog-node: "npm:^5.24.15" prom-client: "npm:^15.1.3" - redlock: "npm:5.0.0-beta.2" simple-git: "npm:^3.36.0" tsc-watch: "npm:^6.2.0" tsx: "npm:^4.21.0" @@ -18863,7 +18862,7 @@ __metadata: languageName: node linkType: hard -"node-abort-controller@npm:^3.0.1, node-abort-controller@npm:^3.1.1": +"node-abort-controller@npm:^3.1.1": version: 3.1.1 resolution: "node-abort-controller@npm:3.1.1" checksum: 10c0/f7ad0e7a8e33809d4f3a0d1d65036a711c39e9d23e0319d80ebe076b9a3b4432b4d6b86a7fab65521de3f6872ffed36fc35d1327487c48eb88c517803403eda3 @@ -20887,15 +20886,6 @@ __metadata: languageName: node linkType: hard -"redlock@npm:5.0.0-beta.2": - version: 5.0.0-beta.2 - resolution: "redlock@npm:5.0.0-beta.2" - dependencies: - node-abort-controller: "npm:^3.0.1" - checksum: 10c0/6664a1b7807ec0ceb223d8f50bf087ac9a6df6356329c47b4374c3a7e4cbcb24a3045d45735dbc74422fc399657baf533e24c0d6528c141c9dd14701880d4fe4 - languageName: node - linkType: hard - "reflect.getprototypeof@npm:^1.0.6, reflect.getprototypeof@npm:^1.0.9": version: 1.0.10 resolution: "reflect.getprototypeof@npm:1.0.10" From 7336c5e1297cdd3ccc5b190f03e5aa7f45250bf7 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Fri, 31 Jul 2026 15:53:55 -0700 Subject: [PATCH 06/15] further wip --- docs/snippets/schemas/v3/index.schema.mdx | 26 +- packages/backend/package.json | 7 +- packages/backend/src/api.ts | 15 +- packages/backend/src/index.ts | 3 +- packages/backend/src/jobManager.ts | 12 +- .../backend/src/reconciliationWorkload.ts | 2 +- packages/schemas/src/v3/index.schema.ts | 26 +- packages/schemas/src/v3/index.type.ts | 11 +- packages/shared/package.json | 4 +- packages/shared/src/constants.ts | 10 +- schemas/v3/index.json | 13 +- yarn.lock | 376 +++++++++++------- 12 files changed, 313 insertions(+), 192 deletions(-) diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index 0a25f6917..4f38c3429 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -42,19 +42,20 @@ }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", "minimum": 1, "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -96,12 +97,12 @@ }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 } }, @@ -228,19 +229,20 @@ }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", "minimum": 1, "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -282,12 +284,12 @@ }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 } }, diff --git a/packages/backend/package.json b/packages/backend/package.json index 8d9f61aaf..7788e5093 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -22,6 +22,9 @@ "vitest": "^4.1.4" }, "dependencies": { + "@bull-board/api": "6.11.2", + "@bull-board/express": "6.11.2", + "@bull-board/ui": "6.11.2", "@coderabbitai/bitbucket": "^1.1.3", "@gitbeaker/rest": "^40.5.1", "@octokit/app": "^16.1.1", @@ -35,7 +38,7 @@ "@types/express": "^5.0.0", "argparse": "^2.0.1", "azure-devops-node-api": "^15.1.1", - "bullmq": "^5.34.10", + "bullmq": "^5.81.3", "chokidar": "^4.0.3", "cross-fetch": "^4.0.0", "dotenv": "^16.4.5", @@ -46,7 +49,7 @@ "gitea-js": "^1.22.0", "glob": "^11.1.0", "http-status-codes": "^2.3.0", - "ioredis": "^5.4.2", + "ioredis": "^5.11.1", "lowdb": "^7.0.1", "micromatch": "^4.0.8", "p-limit": "^7.2.0", diff --git a/packages/backend/src/api.ts b/packages/backend/src/api.ts index fd1d91ba5..a04d7bd9f 100644 --- a/packages/backend/src/api.ts +++ b/packages/backend/src/api.ts @@ -1,4 +1,8 @@ import { createLogger, env } from '@sourcebot/shared'; +import { createBullBoard } from '@bull-board/api'; +import { BullMQAdapter } from '@bull-board/api/bullMQAdapter.js'; +import { ExpressAdapter } from '@bull-board/express'; +import { Queue } from 'bullmq'; import express, { Request, Response } from 'express'; import 'express-async-errors'; import * as http from "http"; @@ -12,11 +16,19 @@ const PORT = Number(workerApiUrl.port) || (workerApiUrl.protocol === "https:" ? export class Api { private server: http.Server; - constructor(promClient: PromClient) { + constructor(promClient: PromClient, queues: Queue[]) { const app = express(); app.use(express.json()); app.use(express.urlencoded({ extended: true })); + const bullBoardAdapter = new ExpressAdapter(); + bullBoardAdapter.setBasePath('/admin/queues'); + createBullBoard({ + queues: queues.map(queue => new BullMQAdapter(queue, { readOnlyMode: true })), + serverAdapter: bullBoardAdapter, + }); + app.use('/admin/queues', bullBoardAdapter.getRouter()); + // Prometheus metrics endpoint app.use('/metrics', async (_req: Request, res: Response) => { res.set('Content-Type', promClient.registry.contentType); @@ -26,6 +38,7 @@ export class Api { this.server = app.listen(PORT, () => { logger.debug(`API server is running on port ${PORT}`); + logger.debug(`Bull Board is available at ${workerApiUrl.origin}/admin/queues`); }); } diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 73db4fc3f..8a8527189 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -48,7 +48,6 @@ if (await hasEntitlement('github-app')) { } const promClient = new PromClient(); -const api = new Api(promClient); logger.info('Worker started.'); @@ -67,6 +66,8 @@ jobManager.register(reconciliationWorkload); jobManager.register(connectionWorkload); jobManager.register(repoIndexWorkload); +const api = new Api(promClient, jobManager.getQueues()); + await cleanupOrphanedRepoResources(prisma); await jobManager.start(); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index cd49b5c76..b05ab4264 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -1,6 +1,6 @@ import * as Sentry from "@sentry/node"; import { BullMQClient, createBullMQJobLogger, createLogger, DataOf, JobLifecycleContext, QueueName } from "@sourcebot/shared"; -import { Job, Worker } from "bullmq"; +import { Job, Queue, Worker } from "bullmq"; import { Redis } from "ioredis"; import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; import { JobDetail, JobManager, Schedule, Workload } from "./types.js"; @@ -63,6 +63,12 @@ export class BullMQJobManager implements JobManager { this.workloads.set(name, workload); } + getQueues(): Queue[] { + return [...this.workloads.values()].map((workload) => + this.bullmqClient.getQueue(workload.queueSpec), + ); + } + async start(): Promise { if (this.workloads.size === 0) { logger.debug('start() called with nothing registered; nothing to do'); @@ -117,7 +123,7 @@ export class BullMQJobManager implements JobManager { `${LOG_TAG}:${spec.name}:job:${job.id ?? 'unknown'}`, ); const lifecycleContext = this.jobLifecycleContext(job); - jobLogger.info(`Started workload "${spec.name}"`); + jobLogger.debug(`Started workload "${spec.name}"`); try { await workload.onStarted?.(lifecycleContext); @@ -128,7 +134,7 @@ export class BullMQJobManager implements JobManager { updateProgress: (progress) => job.updateProgress(progress), trigger: (target, data) => this.trigger(target, data), }); - jobLogger.info(`Completed workload "${spec.name}"`); + jobLogger.debug(`Completed workload "${spec.name}"`); return result; } catch (error) { jobLogger.error(`Workload "${spec.name}" attempt failed`, error); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts index e36e7c975..f34008b2a 100644 --- a/packages/backend/src/reconciliationWorkload.ts +++ b/packages/backend/src/reconciliationWorkload.ts @@ -12,7 +12,7 @@ export const createReconciliationWorkload = ({ settings, }: ReconciliationWorkloadDependencies): Workload<'reconciliation'> => ({ concurrency: 1, - schedule: { every: '15m' }, + schedule: { every: '10s' }, queueSpec: RECONCILIATION_QUEUE, process: async ({ logger, trigger }) => { const connectionThreshold = new Date(Date.now() - settings.resyncConnectionIntervalMs); diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index c55c72a05..228a7f9ce 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -41,19 +41,20 @@ const schema = { }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", "minimum": 1, "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -95,12 +96,12 @@ const schema = { }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 } }, @@ -227,19 +228,20 @@ const schema = { }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", "minimum": 1, "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -281,12 +283,12 @@ const schema = { }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 } }, diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index 3bede3def..833b5aa15 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -110,15 +110,16 @@ export interface Settings { reindexRepoPollingIntervalMs?: number; /** * @deprecated - * The number of connection sync jobs to run concurrently. Defaults to 8. + * The number of connection sync jobs to run concurrently. Defaults to 2. */ maxConnectionSyncJobConcurrency?: number; /** - * The number of repo indexing jobs to run concurrently. Defaults to 8. + * The number of repo indexing jobs to run concurrently. Defaults to 2. */ maxRepoIndexingJobConcurrency?: number; /** - * The number of repo GC jobs to run concurrently. Defaults to 8. + * @deprecated + * The number of repo GC jobs to run concurrently. Defaults to 2. */ maxRepoGarbageCollectionJobConcurrency?: number; /** @@ -153,11 +154,11 @@ export interface Settings { */ experiment_userDrivenPermissionSyncIntervalMs?: number; /** - * The number of account permission sync jobs to run concurrently. Defaults to 8. + * The number of account permission sync jobs to run concurrently. Defaults to 2. */ maxAccountPermissionSyncJobConcurrency?: number; /** - * The number of repo permission sync jobs to run concurrently. Defaults to 8. + * The number of repo permission sync jobs to run concurrently. Defaults to 2. */ maxRepoPermissionSyncJobConcurrency?: number; } diff --git a/packages/shared/package.json b/packages/shared/package.json index e865230c3..7967ad2a6 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -19,8 +19,8 @@ "@sourcebot/schemas": "workspace:*", "@t3-oss/env-core": "^0.13.10", "ajv": "^8.17.1", - "bullmq": "^5.34.10", - "ioredis": "^5.4.2", + "bullmq": "^5.81.3", + "ioredis": "^5.11.1", "micromatch": "^4.0.8", "strip-json-comments": "^5.0.1", "triple-beam": "^1.4.1", diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index c299ef1cc..fce116549 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -23,9 +23,9 @@ export const DEFAULT_CONFIG_SETTINGS: ConfigSettings = { resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, // 24 hours resyncConnectionPollingIntervalMs: 1000 * 1, // 1 second reindexRepoPollingIntervalMs: 1000 * 1, // 1 second - maxConnectionSyncJobConcurrency: 8, - maxRepoIndexingJobConcurrency: 8, - maxRepoGarbageCollectionJobConcurrency: 8, + maxConnectionSyncJobConcurrency: 2, + maxRepoIndexingJobConcurrency: 2, + maxRepoGarbageCollectionJobConcurrency: 2, repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours enablePublicAccess: false, // deprected, use FORCE_ENABLE_ANONYMOUS_ACCESS instead @@ -33,8 +33,8 @@ export const DEFAULT_CONFIG_SETTINGS: ConfigSettings = { userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours (deprecated) experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours (deprecated) - maxAccountPermissionSyncJobConcurrency: 8, - maxRepoPermissionSyncJobConcurrency: 8, + maxAccountPermissionSyncJobConcurrency: 2, + maxRepoPermissionSyncJobConcurrency: 2, } export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES: CodeHostType[] = [ diff --git a/schemas/v3/index.json b/schemas/v3/index.json index 6dc6255f1..887d40eb7 100644 --- a/schemas/v3/index.json +++ b/schemas/v3/index.json @@ -40,19 +40,20 @@ }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", "minimum": 1, "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -94,12 +95,12 @@ }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 } }, diff --git a/yarn.lock b/yarn.lock index 0982a01a2..99721e6bd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1340,6 +1340,38 @@ __metadata: languageName: node linkType: hard +"@bull-board/api@npm:6.11.2": + version: 6.11.2 + resolution: "@bull-board/api@npm:6.11.2" + dependencies: + redis-info: "npm:^3.1.0" + peerDependencies: + "@bull-board/ui": 6.11.2 + checksum: 10c0/d6a82bdd598d41c4e09dd4e8a001f49e87ed40aa535ca841c5430590f0df206ae49ac8835c2832b00cd6bdb413ef89ea7083a5ffc1b779a307744ea69d83c76d + languageName: node + linkType: hard + +"@bull-board/express@npm:6.11.2": + version: 6.11.2 + resolution: "@bull-board/express@npm:6.11.2" + dependencies: + "@bull-board/api": "npm:6.11.2" + "@bull-board/ui": "npm:6.11.2" + ejs: "npm:^3.1.10" + express: "npm:^4.21.1 || ^5.0.0" + checksum: 10c0/36cf6fb63f51f095934ba6167be8704f45d881c882d56e11e34f9b60e0dac16bc989553292942797af0f62beac2c10f116a852988dadb1bc07cd06d48e1b8ae3 + languageName: node + linkType: hard + +"@bull-board/ui@npm:6.11.2": + version: 6.11.2 + resolution: "@bull-board/ui@npm:6.11.2" + dependencies: + "@bull-board/api": "npm:6.11.2" + checksum: 10c0/7ddcb49222c09f32f63d9a55f000445cd59dd6c73620f5e30ffcf04cfa8727c91918485a7c3d17c709d2b721b4a8d32d2026c2a7f208953cd3facd602e595f03 + languageName: node + linkType: hard + "@cfworker/json-schema@npm:^4.0.2": version: 4.1.1 resolution: "@cfworker/json-schema@npm:4.1.1" @@ -3220,10 +3252,10 @@ __metadata: languageName: node linkType: hard -"@ioredis/commands@npm:^1.1.1": - version: 1.2.0 - resolution: "@ioredis/commands@npm:1.2.0" - checksum: 10c0/a5d3c29dd84d8a28b7c67a441ac1715cbd7337a7b88649c0f17c345d89aa218578d2b360760017c48149ef8a70f44b051af9ac0921a0622c2b479614c4f65b36 +"@ioredis/commands@npm:1.10.0": + version: 1.10.0 + resolution: "@ioredis/commands@npm:1.10.0" + checksum: 10c0/baf91e62d0e64ef2b5f7ca4413dc2456fe250e87483beac4a1c8ef1fe5ad0d2fcdeb9b89d4556d8ef6c7455c64a964359d729601fdb06b2f4c76c35dd59afa99 languageName: node linkType: hard @@ -3805,44 +3837,44 @@ __metadata: languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.4" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.4" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.4" conditions: os=linux & cpu=arm64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.4" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.4" conditions: os=linux & cpu=x64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.4" conditions: os=win32 & cpu=x64 languageName: node linkType: hard @@ -9159,6 +9191,9 @@ __metadata: version: 0.0.0-use.local resolution: "@sourcebot/backend@workspace:packages/backend" dependencies: + "@bull-board/api": "npm:6.11.2" + "@bull-board/express": "npm:6.11.2" + "@bull-board/ui": "npm:6.11.2" "@coderabbitai/bitbucket": "npm:^1.1.3" "@gitbeaker/rest": "npm:^40.5.1" "@octokit/app": "npm:^16.1.1" @@ -9175,7 +9210,7 @@ __metadata: "@types/node": "npm:^22.7.5" argparse: "npm:^2.0.1" azure-devops-node-api: "npm:^15.1.1" - bullmq: "npm:^5.34.10" + bullmq: "npm:^5.81.3" chokidar: "npm:^4.0.3" cross-env: "npm:^7.0.3" cross-fetch: "npm:^4.0.0" @@ -9187,7 +9222,7 @@ __metadata: gitea-js: "npm:^1.22.0" glob: "npm:^11.1.0" http-status-codes: "npm:^2.3.0" - ioredis: "npm:^5.4.2" + ioredis: "npm:^5.11.1" json-schema-to-typescript: "npm:^15.0.4" lowdb: "npm:^7.0.1" micromatch: "npm:^4.0.8" @@ -9272,9 +9307,9 @@ __metadata: "@types/micromatch": "npm:^4.0.9" "@types/node": "npm:^22.7.5" ajv: "npm:^8.17.1" - bullmq: "npm:^5.34.10" + bullmq: "npm:^5.81.3" cross-env: "npm:^7.0.3" - ioredis: "npm:^5.4.2" + ioredis: "npm:^5.11.1" micromatch: "npm:^4.0.8" strip-json-comments: "npm:^5.0.1" triple-beam: "npm:^1.4.1" @@ -11592,7 +11627,7 @@ __metadata: languageName: node linkType: hard -"async@npm:^3.2.3": +"async@npm:^3.2.3, async@npm:^3.2.6": version: 3.2.6 resolution: "async@npm:3.2.6" checksum: 10c0/36484bb15ceddf07078688d95e27076379cc2f87b10c03b6dd8a83e89475a3c8df5848859dd06a4c95af1e4c16fc973de0171a77f18ea00be899aca2a4f85e70 @@ -11848,6 +11883,15 @@ __metadata: languageName: node linkType: hard +"brace-expansion@npm:^2.0.1": + version: 2.1.4 + resolution: "brace-expansion@npm:2.1.4" + dependencies: + balanced-match: "npm:^1.0.0" + checksum: 10c0/6c0a0e2573eac1dc565b52b1e1bfbeba39bf1830d106ebbc61ff1eaefcf610e9111cd3baa091addd18e33292135c99e019d3184d966389d85dc958fdcdc1449f + languageName: node + linkType: hard + "brace-expansion@npm:^2.0.3": version: 2.1.0 resolution: "brace-expansion@npm:2.1.0" @@ -11896,18 +11940,22 @@ __metadata: languageName: node linkType: hard -"bullmq@npm:^5.34.10": - version: 5.44.3 - resolution: "bullmq@npm:5.44.3" +"bullmq@npm:^5.81.3": + version: 5.81.3 + resolution: "bullmq@npm:5.81.3" dependencies: - cron-parser: "npm:^4.9.0" - ioredis: "npm:^5.4.1" - msgpackr: "npm:^1.11.2" - node-abort-controller: "npm:^3.1.1" - semver: "npm:^7.5.4" - tslib: "npm:^2.0.0" - uuid: "npm:^9.0.0" - checksum: 10c0/2785929ef59645980e3981d6f5e3f7ef25ef0e8488281a7aa48e89c25a89ff5dde0431051f48b2edadd9675c5b780c02577724c33d5d31762009820b8d404cdb + cron-parser: "npm:4.9.0" + ioredis: "npm:5.11.1" + msgpackr: "npm:2.0.5" + node-abort-controller: "npm:3.1.1" + semver: "npm:7.8.5" + tslib: "npm:2.8.1" + peerDependencies: + redis: ">=5.0.0" + peerDependenciesMeta: + redis: + optional: true + checksum: 10c0/e28abf1f37191966d0204e8717040f99807749579ba34e6cd8ac4698f3daa0327be094e52fa52c6aab991f4cd4cdd338b6c950134ec083be166c19f82d96cde1 languageName: node linkType: hard @@ -12211,10 +12259,10 @@ __metadata: languageName: node linkType: hard -"cluster-key-slot@npm:^1.1.0": - version: 1.1.2 - resolution: "cluster-key-slot@npm:1.1.2" - checksum: 10c0/d7d39ca28a8786e9e801eeb8c770e3c3236a566625d7299a47bb71113fb2298ce1039596acb82590e598c52dbc9b1f088c8f587803e697cb58e1867a95ff94d3 +"cluster-key-slot@npm:1.1.1": + version: 1.1.1 + resolution: "cluster-key-slot@npm:1.1.1" + checksum: 10c0/079b1ae86b20e2d53308a877b08de5e830722a45c07810569d0dab4955bed569da33ac9f79998289d014adf02cca7223a0647cb0ee6548a12ab3c4f9beac1377 languageName: node linkType: hard @@ -12752,7 +12800,7 @@ __metadata: languageName: node linkType: hard -"cron-parser@npm:^4.9.0": +"cron-parser@npm:4.9.0": version: 4.9.0 resolution: "cron-parser@npm:4.9.0" dependencies: @@ -13370,6 +13418,18 @@ __metadata: languageName: node linkType: hard +"debug@npm:4.4.3, debug@npm:^4.4.3, debug@npm:~4.4.1": + version: 4.4.3 + resolution: "debug@npm:4.4.3" + dependencies: + ms: "npm:^2.1.3" + peerDependenciesMeta: + supports-color: + optional: true + checksum: 10c0/d79136ec6c83ecbefd0f6a5593da6a9c91ec4d7ddc4b54c883d6e71ec9accb5f67a1a5e96d00a328196b5b5c86d365e98d8a3a70856aaf16b4e7b1985e67f5a6 + languageName: node + linkType: hard + "debug@npm:^3.2.7": version: 3.2.7 resolution: "debug@npm:3.2.7" @@ -13391,18 +13451,6 @@ __metadata: languageName: node linkType: hard -"debug@npm:^4.4.3, debug@npm:~4.4.1": - version: 4.4.3 - resolution: "debug@npm:4.4.3" - dependencies: - ms: "npm:^2.1.3" - peerDependenciesMeta: - supports-color: - optional: true - checksum: 10c0/d79136ec6c83ecbefd0f6a5593da6a9c91ec4d7ddc4b54c883d6e71ec9accb5f67a1a5e96d00a328196b5b5c86d365e98d8a3a70856aaf16b4e7b1985e67f5a6 - languageName: node - linkType: hard - "debug@npm:~4.3.2": version: 4.3.7 resolution: "debug@npm:4.3.7" @@ -13509,7 +13557,7 @@ __metadata: languageName: node linkType: hard -"denque@npm:^2.1.0": +"denque@npm:2.1.0": version: 2.1.0 resolution: "denque@npm:2.1.0" checksum: 10c0/f9ef81aa0af9c6c614a727cb3bd13c5d7db2af1abf9e6352045b86e85873e629690f6222f4edd49d10e4ccf8f078bbeec0794fafaf61b659c0589d0c511ec363 @@ -13782,6 +13830,17 @@ __metadata: languageName: node linkType: hard +"ejs@npm:^3.1.10": + version: 3.1.10 + resolution: "ejs@npm:3.1.10" + dependencies: + jake: "npm:^10.8.5" + bin: + ejs: bin/cli.js + checksum: 10c0/52eade9e68416ed04f7f92c492183340582a36482836b11eab97b159fcdcfdedc62233a1bf0bf5e5e1851c501f2dca0e2e9afd111db2599e4e7f53ee29429ae1 + languageName: node + linkType: hard + "electron-to-chromium@npm:^1.5.73": version: 1.5.123 resolution: "electron-to-chromium@npm:1.5.123" @@ -14795,6 +14854,42 @@ __metadata: languageName: node linkType: hard +"express@npm:^4.21.1 || ^5.0.0, express@npm:^5.2.1": + version: 5.2.1 + resolution: "express@npm:5.2.1" + dependencies: + accepts: "npm:^2.0.0" + body-parser: "npm:^2.2.1" + content-disposition: "npm:^1.0.0" + content-type: "npm:^1.0.5" + cookie: "npm:^0.7.1" + cookie-signature: "npm:^1.2.1" + debug: "npm:^4.4.0" + depd: "npm:^2.0.0" + encodeurl: "npm:^2.0.0" + escape-html: "npm:^1.0.3" + etag: "npm:^1.8.1" + finalhandler: "npm:^2.1.0" + fresh: "npm:^2.0.0" + http-errors: "npm:^2.0.0" + merge-descriptors: "npm:^2.0.0" + mime-types: "npm:^3.0.0" + on-finished: "npm:^2.4.1" + once: "npm:^1.4.0" + parseurl: "npm:^1.3.3" + proxy-addr: "npm:^2.0.7" + qs: "npm:^6.14.0" + range-parser: "npm:^1.2.1" + router: "npm:^2.2.0" + send: "npm:^1.1.0" + serve-static: "npm:^2.2.0" + statuses: "npm:^2.0.1" + type-is: "npm:^2.0.1" + vary: "npm:^1.1.2" + checksum: 10c0/45e8c841ad188a41402ddcd1294901e861ee0819f632fb494f2ed344ef9c43315d294d443fb48d594e6586a3b779785120f43321417adaef8567316a55072949 + languageName: node + linkType: hard + "express@npm:^4.21.2": version: 4.21.2 resolution: "express@npm:4.21.2" @@ -14834,42 +14929,6 @@ __metadata: languageName: node linkType: hard -"express@npm:^5.2.1": - version: 5.2.1 - resolution: "express@npm:5.2.1" - dependencies: - accepts: "npm:^2.0.0" - body-parser: "npm:^2.2.1" - content-disposition: "npm:^1.0.0" - content-type: "npm:^1.0.5" - cookie: "npm:^0.7.1" - cookie-signature: "npm:^1.2.1" - debug: "npm:^4.4.0" - depd: "npm:^2.0.0" - encodeurl: "npm:^2.0.0" - escape-html: "npm:^1.0.3" - etag: "npm:^1.8.1" - finalhandler: "npm:^2.1.0" - fresh: "npm:^2.0.0" - http-errors: "npm:^2.0.0" - merge-descriptors: "npm:^2.0.0" - mime-types: "npm:^3.0.0" - on-finished: "npm:^2.4.1" - once: "npm:^1.4.0" - parseurl: "npm:^1.3.3" - proxy-addr: "npm:^2.0.7" - qs: "npm:^6.14.0" - range-parser: "npm:^1.2.1" - router: "npm:^2.2.0" - send: "npm:^1.1.0" - serve-static: "npm:^2.2.0" - statuses: "npm:^2.0.1" - type-is: "npm:^2.0.1" - vary: "npm:^1.1.2" - checksum: 10c0/45e8c841ad188a41402ddcd1294901e861ee0819f632fb494f2ed344ef9c43315d294d443fb48d594e6586a3b779785120f43321417adaef8567316a55072949 - languageName: node - linkType: hard - "extend@npm:^3.0.0, extend@npm:^3.0.2": version: 3.0.2 resolution: "extend@npm:3.0.2" @@ -15090,6 +15149,15 @@ __metadata: languageName: node linkType: hard +"filelist@npm:^1.0.4": + version: 1.0.6 + resolution: "filelist@npm:1.0.6" + dependencies: + minimatch: "npm:^5.0.1" + checksum: 10c0/6ee725bec3e1936d680a45f14439b224d9f7c71658c145addcf551dd82f03d608522eb6b191aa086b392bc3e52ed4ce0ed8d78e24b203e6c5e867560a05d1121 + languageName: node + linkType: hard + "fill-range@npm:^7.1.1": version: 7.1.1 resolution: "fill-range@npm:7.1.1" @@ -16291,20 +16359,18 @@ __metadata: languageName: node linkType: hard -"ioredis@npm:^5.4.1, ioredis@npm:^5.4.2": - version: 5.6.0 - resolution: "ioredis@npm:5.6.0" +"ioredis@npm:5.11.1, ioredis@npm:^5.11.1": + version: 5.11.1 + resolution: "ioredis@npm:5.11.1" dependencies: - "@ioredis/commands": "npm:^1.1.1" - cluster-key-slot: "npm:^1.1.0" - debug: "npm:^4.3.4" - denque: "npm:^2.1.0" - lodash.defaults: "npm:^4.2.0" - lodash.isarguments: "npm:^3.1.0" - redis-errors: "npm:^1.2.0" - redis-parser: "npm:^3.0.0" - standard-as-callback: "npm:^2.1.0" - checksum: 10c0/a885e5146640fc448706871290ef424ffa39af561f7ee3cf1590085209a509f85e99082bdaaf3cd32fa66758aea3fc2055d1109648ddca96fac4944bf2092c30 + "@ioredis/commands": "npm:1.10.0" + cluster-key-slot: "npm:1.1.1" + debug: "npm:4.4.3" + denque: "npm:2.1.0" + redis-errors: "npm:1.2.0" + redis-parser: "npm:3.0.0" + standard-as-callback: "npm:2.1.0" + checksum: 10c0/a8b27043cf2c045dfc93f40a32ce24cf9f8b57799a37f4234c4b925c365ccf131629590f94a512f546fda2ba8ed034009c94c4933ecd44c50bc166636d929fd6 languageName: node linkType: hard @@ -16776,6 +16842,19 @@ __metadata: languageName: node linkType: hard +"jake@npm:^10.8.5": + version: 10.9.4 + resolution: "jake@npm:10.9.4" + dependencies: + async: "npm:^3.2.6" + filelist: "npm:^1.0.4" + picocolors: "npm:^1.1.1" + bin: + jake: bin/cli.js + checksum: 10c0/bb52f000340d4a32f1a3893b9abe56ef2b77c25da4dbf2c0c874a8159d082dddda50a5ad10e26060198bd645b928ba8dba3b362710f46a247e335321188c5a9c + languageName: node + linkType: hard + "jiti@npm:2.4.2": version: 2.4.2 resolution: "jiti@npm:2.4.2" @@ -17445,20 +17524,6 @@ __metadata: languageName: node linkType: hard -"lodash.defaults@npm:^4.2.0": - version: 4.2.0 - resolution: "lodash.defaults@npm:4.2.0" - checksum: 10c0/d5b77aeb702caa69b17be1358faece33a84497bcca814897383c58b28a2f8dfc381b1d9edbec239f8b425126a3bbe4916223da2a576bb0411c2cefd67df80707 - languageName: node - linkType: hard - -"lodash.isarguments@npm:^3.1.0": - version: 3.1.0 - resolution: "lodash.isarguments@npm:3.1.0" - checksum: 10c0/5e8f95ba10975900a3920fb039a3f89a5a79359a1b5565e4e5b4310ed6ebe64011e31d402e34f577eca983a1fc01ff86c926e3cbe602e1ddfc858fdd353e62d8 - languageName: node - linkType: hard - "lodash.isplainobject@npm:^4.0.6": version: 4.0.6 resolution: "lodash.isplainobject@npm:4.0.6" @@ -17473,7 +17538,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:^4.17.21": +"lodash@npm:^4.17.11, lodash@npm:^4.17.21": version: 4.18.1 resolution: "lodash@npm:4.18.1" checksum: 10c0/757228fc68805c59789e82185135cf85f05d0b2d3d54631d680ca79ec21944ec8314d4533639a14b8bcfbd97a517e78960933041a5af17ecb693ec6eecb99a27 @@ -18430,6 +18495,15 @@ __metadata: languageName: node linkType: hard +"minimatch@npm:^5.0.1": + version: 5.1.9 + resolution: "minimatch@npm:5.1.9" + dependencies: + brace-expansion: "npm:^2.0.1" + checksum: 10c0/4202718683815a7288b13e470160a4f9560cf392adef4f453927505817e01ef6b3476ecde13cfcaed17e7326dd3b69ad44eb2daeb19a217c5500f9277893f1d6 + languageName: node + linkType: hard + "minimatch@npm:^9.0.4, minimatch@npm:^9.0.5": version: 9.0.9 resolution: "minimatch@npm:9.0.9" @@ -18611,16 +18685,16 @@ __metadata: languageName: node linkType: hard -"msgpackr-extract@npm:^3.0.2": - version: 3.0.3 - resolution: "msgpackr-extract@npm:3.0.3" - dependencies: - "@msgpackr-extract/msgpackr-extract-darwin-arm64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-darwin-x64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-linux-arm": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-linux-arm64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-linux-x64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-win32-x64": "npm:3.0.3" +"msgpackr-extract@npm:^3.0.4": + version: 3.0.4 + resolution: "msgpackr-extract@npm:3.0.4" + dependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-darwin-x64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-arm": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-arm64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-x64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-win32-x64": "npm:3.0.4" node-gyp: "npm:latest" node-gyp-build-optional-packages: "npm:5.2.2" dependenciesMeta: @@ -18638,19 +18712,19 @@ __metadata: optional: true bin: download-msgpackr-prebuilds: bin/download-prebuilds.js - checksum: 10c0/e504fd8bf86a29d7527c83776530ee6dc92dcb0273bb3679fd4a85173efead7f0ee32fb82c8410a13c33ef32828c45f81118ffc0fbed5d6842e72299894623b4 + checksum: 10c0/582a9d17abbf3019e600e948736695056280ce401fd0235ee2474e95f9952208b9f6cce4d0e355b03b7d3c5630e6c3d11fe5fc27fdedb2311cce48de464338d8 languageName: node linkType: hard -"msgpackr@npm:^1.11.2": - version: 1.11.2 - resolution: "msgpackr@npm:1.11.2" +"msgpackr@npm:2.0.5": + version: 2.0.5 + resolution: "msgpackr@npm:2.0.5" dependencies: - msgpackr-extract: "npm:^3.0.2" + msgpackr-extract: "npm:^3.0.4" dependenciesMeta: msgpackr-extract: optional: true - checksum: 10c0/7d2e81ca82c397b2352d470d6bc8f4a967fe4fe14f8fc1fc9906b23009fdfb543999b1ad29c700b8861581e0b6bf903d6f0fefb69a09375cbca6d4d802e6c906 + checksum: 10c0/7ac9820cecd44d24d2ef07994405a277509f004d4fb0a77d04e10e9071d0599e86ad53a9daf673e801518230774bdaca72b1fdcff2b516584ab58bd19362e6ad languageName: node linkType: hard @@ -18862,7 +18936,7 @@ __metadata: languageName: node linkType: hard -"node-abort-controller@npm:^3.1.1": +"node-abort-controller@npm:3.1.1": version: 3.1.1 resolution: "node-abort-controller@npm:3.1.1" checksum: 10c0/f7ad0e7a8e33809d4f3a0d1d65036a711c39e9d23e0319d80ebe076b9a3b4432b4d6b86a7fab65521de3f6872ffed36fc35d1327487c48eb88c517803403eda3 @@ -20870,14 +20944,23 @@ __metadata: languageName: node linkType: hard -"redis-errors@npm:^1.0.0, redis-errors@npm:^1.2.0": +"redis-errors@npm:1.2.0, redis-errors@npm:^1.0.0": version: 1.2.0 resolution: "redis-errors@npm:1.2.0" checksum: 10c0/5b316736e9f532d91a35bff631335137a4f974927bb2fb42bf8c2f18879173a211787db8ac4c3fde8f75ed6233eb0888e55d52510b5620e30d69d7d719c8b8a7 languageName: node linkType: hard -"redis-parser@npm:^3.0.0": +"redis-info@npm:^3.1.0": + version: 3.1.0 + resolution: "redis-info@npm:3.1.0" + dependencies: + lodash: "npm:^4.17.11" + checksum: 10c0/ec0f31d97893c5828cec7166486d74198c92160c60073b6f2fe805cdf575a10ddcccc7641737d44b8f451355f0ab5b6c7b0d79e8fc24742b75dd625f91ffee38 + languageName: node + linkType: hard + +"redis-parser@npm:3.0.0": version: 3.0.0 resolution: "redis-parser@npm:3.0.0" dependencies: @@ -21567,6 +21650,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:7.8.5": + version: 7.8.5 + resolution: "semver@npm:7.8.5" + bin: + semver: bin/semver.js + checksum: 10c0/b1f3127a5be8125a94f37188b361c212466c292c6910adce3ec106cff5dc211ccaedc4739c11bb70fda59d6fc1f040a9bca289f4e093451521a2372e5231fe0c + languageName: node + linkType: hard + "semver@npm:^6.3.1": version: 6.3.1 resolution: "semver@npm:6.3.1" @@ -21576,7 +21668,7 @@ __metadata: languageName: node linkType: hard -"semver@npm:^7.3.5, semver@npm:^7.5.3, semver@npm:^7.5.4, semver@npm:^7.6.0, semver@npm:^7.6.3": +"semver@npm:^7.3.5, semver@npm:^7.5.3, semver@npm:^7.6.0, semver@npm:^7.6.3": version: 7.7.1 resolution: "semver@npm:7.7.1" bin: @@ -22310,7 +22402,7 @@ __metadata: languageName: node linkType: hard -"standard-as-callback@npm:^2.1.0": +"standard-as-callback@npm:2.1.0": version: 2.1.0 resolution: "standard-as-callback@npm:2.1.0" checksum: 10c0/012677236e3d3fdc5689d29e64ea8a599331c4babe86956bf92fc5e127d53f85411c5536ee0079c52c43beb0026b5ce7aa1d834dd35dd026e82a15d1bcaead1f @@ -23241,7 +23333,7 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2.0.0, tslib@npm:^2.1.0, tslib@npm:^2.4.0, tslib@npm:^2.6.2, tslib@npm:^2.7.0, tslib@npm:^2.8.0, tslib@npm:^2.8.1": +"tslib@npm:2.8.1, tslib@npm:^2.0.0, tslib@npm:^2.1.0, tslib@npm:^2.4.0, tslib@npm:^2.6.2, tslib@npm:^2.7.0, tslib@npm:^2.8.0, tslib@npm:^2.8.1": version: 2.8.1 resolution: "tslib@npm:2.8.1" checksum: 10c0/9c4759110a19c53f992d9aae23aac5ced636e99887b51b9e61def52611732872ff7668757d4e4c61f19691e36f4da981cd9485e869b4a7408d689f6bf1f14e62 From 81c419067d75120a068c99d0f44d40595b53e6e3 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Fri, 31 Jul 2026 16:14:49 -0700 Subject: [PATCH 07/15] simplicity: remove much of the web changes --- packages/backend/src/jobManager.ts | 2 - .../migration.sql | 20 - .../migration.sql | 22 -- packages/db/prisma/schema.prisma | 65 +++- packages/db/tools/scripts/inject-repo-data.ts | 22 +- packages/shared/src/queue.ts | 28 -- packages/web/src/actions.ts | 38 +- .../components/defaultSidebar/index.tsx | 5 +- .../src/app/(app)/chat/chatLandingPage.tsx | 11 +- packages/web/src/app/(app)/repos/layout.tsx | 9 + .../search/components/searchLandingPage.tsx | 8 +- .../(app)/settings/connections/[id]/page.tsx | 211 +++++++++++ .../components/connectionJobsTable.tsx | 344 ++++++++++++++++++ .../components/connectionsTable.tsx | 294 +++++++++++++++ .../connections/connectionSyncLogsDialog.tsx | 203 ----------- .../settings/connections/connectionsList.tsx | 282 -------------- .../app/(app)/settings/connections/page.tsx | 97 ++--- .../web/src/app/(app)/settings/layout.tsx | 7 + .../web/src/features/workerApi/actions.ts | 75 +--- 19 files changed, 1069 insertions(+), 674 deletions(-) delete mode 100644 packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql delete mode 100644 packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql create mode 100644 packages/web/src/app/(app)/settings/connections/[id]/page.tsx create mode 100644 packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx create mode 100644 packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx delete mode 100644 packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx delete mode 100644 packages/web/src/app/(app)/settings/connections/connectionsList.tsx diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index b05ab4264..97c44da09 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -123,7 +123,6 @@ export class BullMQJobManager implements JobManager { `${LOG_TAG}:${spec.name}:job:${job.id ?? 'unknown'}`, ); const lifecycleContext = this.jobLifecycleContext(job); - jobLogger.debug(`Started workload "${spec.name}"`); try { await workload.onStarted?.(lifecycleContext); @@ -134,7 +133,6 @@ export class BullMQJobManager implements JobManager { updateProgress: (progress) => job.updateProgress(progress), trigger: (target, data) => this.trigger(target, data), }); - jobLogger.debug(`Completed workload "${spec.name}"`); return result; } catch (error) { jobLogger.error(`Workload "${spec.name}" attempt failed`, error); diff --git a/packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql b/packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql deleted file mode 100644 index c4315cee2..000000000 --- a/packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql +++ /dev/null @@ -1,20 +0,0 @@ -/* - Warnings: - - - You are about to drop the `ConnectionSyncJob` table. If the table is not empty, all the data it contains will be lost. - -*/ --- DropForeignKey -ALTER TABLE "ConnectionSyncJob" DROP CONSTRAINT "ConnectionSyncJob_connectionId_fkey"; - --- AlterTable -ALTER TABLE "Connection" ADD COLUMN "latestSyncJobId" TEXT; - --- DropTable -DROP TABLE "ConnectionSyncJob"; - --- DropEnum -DROP TYPE "ConnectionSyncJobStatus"; - --- DropEnum -DROP TYPE "ConnectionSyncStatus"; diff --git a/packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql b/packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql deleted file mode 100644 index ab019a779..000000000 --- a/packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql +++ /dev/null @@ -1,22 +0,0 @@ -/* - Warnings: - - - You are about to drop the column `latestIndexingJobStatus` on the `Repo` table. All the data in the column will be lost. - - You are about to drop the `RepoIndexingJob` table. If the table is not empty, all the data it contains will be lost. - -*/ --- DropForeignKey -ALTER TABLE "RepoIndexingJob" DROP CONSTRAINT "RepoIndexingJob_repoId_fkey"; - --- AlterTable -ALTER TABLE "Repo" DROP COLUMN "latestIndexingJobStatus", -ADD COLUMN "latestIndexingJobId" TEXT; - --- DropTable -DROP TABLE "RepoIndexingJob"; - --- DropEnum -DROP TYPE "RepoIndexingJobStatus"; - --- DropEnum -DROP TYPE "RepoIndexingJobType"; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 198b13a81..e43f2887b 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -10,6 +10,15 @@ datasource db { url = env("DATABASE_URL") } +enum ConnectionSyncStatus { + SYNC_NEEDED + IN_SYNC_QUEUE + SYNCING + SYNCED + SYNCED_WITH_WARNINGS + FAILED +} + enum ChatVisibility { PRIVATE PUBLIC @@ -65,10 +74,10 @@ model Repo { permissionSyncJobs RepoPermissionSyncJob[] permissionSyncedAt DateTime? /// When the permissions were last synced successfully. + jobs RepoIndexingJob[] indexedAt DateTime? /// When the repo was last indexed successfully. - latestIndexingJobId String? - indexedCommitHash String? /// The commit hash of the last indexed commit (on HEAD). + latestIndexingJobStatus RepoIndexingJobStatus? /// The status of the latest indexing job. pushedAt DateTime? /// The timestamp of the most recent commit across all branches. external_id String /// The id of the repo in the external service @@ -86,6 +95,35 @@ model Repo { @@index([indexedAt]) } +enum RepoIndexingJobStatus { + PENDING + IN_PROGRESS + COMPLETED + FAILED +} + +enum RepoIndexingJobType { + INDEX + CLEANUP +} + +model RepoIndexingJob { + id String @id @default(cuid()) + type RepoIndexingJobType + status RepoIndexingJobStatus @default(PENDING) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + completedAt DateTime? + metadata Json? /// For schema see repoIndexingJobMetadataSchema in packages/shared/src/types.ts + + errorMessage String? + + repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade) + repoId Int + + @@index([repoId, type, status]) +} + enum RepoPermissionSyncJobStatus { PENDING IN_PROGRESS @@ -143,9 +181,9 @@ model Connection { // The type of connection (e.g., github, gitlab, etc.) connectionType ConnectionType + syncJobs ConnectionSyncJob[] /// When the connection was last synced successfully. syncedAt DateTime? - latestSyncJobId String? /// Controls whether repository permissions are enforced for this connection. /// When `PERMISSION_SYNC_ENABLED` is false, this setting has no effect. @@ -170,6 +208,27 @@ model Connection { @@unique([name, orgId]) } +enum ConnectionSyncJobStatus { + PENDING + IN_PROGRESS + COMPLETED + FAILED +} + +model ConnectionSyncJob { + id String @id @default(cuid()) + status ConnectionSyncJobStatus @default(PENDING) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + completedAt DateTime? + + warningMessages String[] + errorMessage String? + + connection Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade) + connectionId Int +} + model RepoToConnection { addedAt DateTime @default(now()) diff --git a/packages/db/tools/scripts/inject-repo-data.ts b/packages/db/tools/scripts/inject-repo-data.ts index 5540f97b4..609bcdcc7 100644 --- a/packages/db/tools/scripts/inject-repo-data.ts +++ b/packages/db/tools/scripts/inject-repo-data.ts @@ -2,6 +2,7 @@ import { Script } from "../scriptRunner"; import { PrismaClient } from "../../dist"; const NUM_REPOS = 1000; +const NUM_INDEXING_JOBS_PER_REPO = 10000; const NUM_PERMISSION_JOBS_PER_REPO = 10000; export const injectRepoData: Script = { @@ -36,6 +37,8 @@ export const injectRepoData: Script = { console.log(`Creating ${NUM_REPOS} repos...`); const statuses = ['PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'] as const; + const indexingJobTypes = ['INDEX', 'CLEANUP'] as const; + for (let i = 0; i < NUM_REPOS; i++) { const repo = await prisma.repo.create({ data: { @@ -68,8 +71,23 @@ export const injectRepoData: Script = { } }); } + + for (let j = 0; j < NUM_INDEXING_JOBS_PER_REPO; j++) { + const status = statuses[Math.floor(Math.random() * statuses.length)]; + const type = indexingJobTypes[Math.floor(Math.random() * indexingJobTypes.length)]; + await prisma.repoIndexingJob.create({ + data: { + repoId: repo.id, + type, + status, + completedAt: status === 'COMPLETED' || status === 'FAILED' ? new Date() : null, + errorMessage: status === 'FAILED' ? 'Mock indexing error' : null, + metadata: {} + } + }); + } } - console.log(`Created ${NUM_REPOS} repos with associated permission jobs.`); + console.log(`Created ${NUM_REPOS} repos with associated jobs.`); } -}; +}; \ No newline at end of file diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 585d1e457..81e20c21f 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -26,20 +26,6 @@ export const CONNECTION_QUEUE: QueueSpec<'connection'> = { keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, - onEnqueued: async ({ - prisma, - data: { connectionId }, - jobId - }) => { - await prisma.connection.update({ - where: { - id: connectionId - }, - data: { - latestSyncJobId: jobId - } - }); - }, dedupKey: (data) => `connection:${data.connectionId}`, } @@ -62,20 +48,6 @@ export const REPO_INDEX_QUEUE: QueueSpec<'repo-index'> = { keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, - onEnqueued: async ({ - prisma, - data: { repoId }, - jobId, - }) => { - await prisma.repo.update({ - where: { - id: repoId, - }, - data: { - latestIndexingJobId: jobId, - }, - }); - }, dedupKey: (data) => `repo:${data.repoId}`, }; diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 5db1dbdfc..c64979173 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -4,7 +4,7 @@ import { createAudit } from "@/ee/features/audit/audit"; import { ErrorCode } from "@/lib/errorCodes"; import { notFound, ServiceError } from "@/lib/serviceError"; import { sew } from "@/middleware/sew"; -import { OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; +import { ConnectionSyncJobStatus, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; @@ -274,6 +274,42 @@ export const getReposStats = async () => sew(() => }) ) +export const getConnectionStats = async () => sew(() => + withAuth(async ({ org, prisma }) => { + const [ + numberOfConnections, + numberOfConnectionsWithFirstTimeSyncJobsInProgress, + ] = await Promise.all([ + prisma.connection.count({ + where: { + orgId: org.id, + } + }), + prisma.connection.count({ + where: { + orgId: org.id, + syncedAt: null, + syncJobs: { + some: { + status: { + in: [ + ConnectionSyncJobStatus.PENDING, + ConnectionSyncJobStatus.IN_PROGRESS, + ] + } + } + } + } + }) + ]); + + return { + numberOfConnections, + numberOfConnectionsWithFirstTimeSyncJobsInProgress, + }; + }) +); + export const getRepoInfoByName = async (repoName: string) => sew(() => withOptionalAuth(async ({ org, prisma }) => { // @note: repo names are represented by their remote url diff --git a/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx b/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx index c09bad644..a0ccd6269 100644 --- a/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx +++ b/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx @@ -2,6 +2,7 @@ import { cookies } from "next/headers"; import { auth } from "@/auth"; import { HOME_VIEW_COOKIE_NAME } from "@/lib/constants"; import { HomeView } from "@/hooks/useHomeView"; +import { getConnectionStats } from "@/actions"; import { getOrgAccountRequests } from "@/features/membership/actions"; import { isServiceError } from "@/lib/utils"; import { ServiceErrorException } from "@/lib/serviceError"; @@ -45,9 +46,11 @@ export async function DefaultSidebar() { if (!isOwner) { return false; } + const connectionStats = await getConnectionStats(); const joinRequests = await getOrgAccountRequests(); + const hasConnectionNotification = !isServiceError(connectionStats) && connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0; const hasJoinRequestNotification = !isServiceError(joinRequests) && joinRequests.length > 0; - return hasJoinRequestNotification; + return hasConnectionNotification || hasJoinRequestNotification; })(); return ( diff --git a/packages/web/src/app/(app)/chat/chatLandingPage.tsx b/packages/web/src/app/(app)/chat/chatLandingPage.tsx index fdebccf77..b78e28e10 100644 --- a/packages/web/src/app/(app)/chat/chatLandingPage.tsx +++ b/packages/web/src/app/(app)/chat/chatLandingPage.tsx @@ -1,4 +1,4 @@ -import { getRepos, getSearchContexts } from "@/actions"; +import { getRepos, getReposStats, getSearchContexts } from "@/actions"; import { SourcebotLogo } from "@/app/components/sourcebotLogo"; import { getConfiguredLanguageModelsInfo } from "@/features/chat/utils.server"; import { CustomSlateEditor } from "@/features/chat/customSlateEditor"; @@ -35,6 +35,8 @@ export async function ChatLandingPage() { take: 10, }); + const repoStats = await getReposStats(); + if (isServiceError(allRepos)) { throw new ServiceErrorException(allRepos); } @@ -47,6 +49,10 @@ export async function ChatLandingPage() { throw new ServiceErrorException(carouselRepos); } + if (isServiceError(repoStats)) { + throw new ServiceErrorException(repoStats); + } + const demoExamples = env.SOURCEBOT_DEMO_EXAMPLES_PATH ? await (async () => { try { return (await measure(() => loadJsonFile(env.SOURCEBOT_DEMO_EXAMPLES_PATH!, demoExamplesSchema), 'loadExamplesJsonFile')).data; @@ -78,8 +84,7 @@ export async function ChatLandingPage() {
diff --git a/packages/web/src/app/(app)/repos/layout.tsx b/packages/web/src/app/(app)/repos/layout.tsx index 61f3cd34e..88738d22e 100644 --- a/packages/web/src/app/(app)/repos/layout.tsx +++ b/packages/web/src/app/(app)/repos/layout.tsx @@ -1,3 +1,7 @@ +import { getReposStats } from "@/actions"; +import { ServiceErrorException } from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; + interface LayoutProps { children: React.ReactNode; } @@ -7,6 +11,11 @@ export default async function Layout( ) { const { children } = props; + const repoStats = await getReposStats(); + if (isServiceError(repoStats)) { + throw new ServiceErrorException(repoStats); + } + return (
diff --git a/packages/web/src/app/(app)/search/components/searchLandingPage.tsx b/packages/web/src/app/(app)/search/components/searchLandingPage.tsx index a6d072f29..e9236607e 100644 --- a/packages/web/src/app/(app)/search/components/searchLandingPage.tsx +++ b/packages/web/src/app/(app)/search/components/searchLandingPage.tsx @@ -5,7 +5,7 @@ import { SyntaxReferenceGuideHint } from "../../components/syntaxReferenceGuideH import Link from "next/link" import { SearchBar } from "../../components/searchBar" import { SearchModeSelector } from "../../components/searchModeSelector" -import { getRepos } from "@/actions" +import { getRepos, getReposStats } from "@/actions" import { ServiceErrorException } from "@/lib/serviceError" import { isServiceError } from "@/lib/utils" @@ -25,7 +25,10 @@ export const SearchLandingPage = async ({ take: 10, }); + const repoStats = await getReposStats(); + if (isServiceError(carouselRepos)) throw new ServiceErrorException(carouselRepos); + if (isServiceError(repoStats)) throw new ServiceErrorException(repoStats); return (
@@ -52,8 +55,7 @@ export const SearchLandingPage = async ({
diff --git a/packages/web/src/app/(app)/settings/connections/[id]/page.tsx b/packages/web/src/app/(app)/settings/connections/[id]/page.tsx new file mode 100644 index 000000000..edcc61069 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/[id]/page.tsx @@ -0,0 +1,211 @@ +import { sew } from "@/middleware/sew"; +import { BackButton } from "@/app/(app)/components/backButton"; +import { DisplayDate } from "@/app/(app)/components/DisplayDate"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { notFound as notFoundServiceError, ServiceErrorException } from "@/lib/serviceError"; +import { notFound } from "next/navigation"; +import { isServiceError } from "@/lib/utils"; +import { withAuth } from "@/middleware/withAuth"; +import { AzureDevOpsConnectionConfig, BitbucketConnectionConfig, GenericGitHostConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GithubConnectionConfig, GitlabConnectionConfig } from "@sourcebot/schemas/v3/index.type"; +import { env, getConfigSettings } from "@sourcebot/shared"; +import { Info } from "lucide-react"; +import Link from "next/link"; +import { Suspense } from "react"; +import { ConnectionJobsTable } from "../components/connectionJobsTable"; + +interface ConnectionDetailPageProps { + params: Promise<{ + id: string + }> +} + +export default async function ConnectionDetailPage(props: ConnectionDetailPageProps) { + const params = await props.params; + const { id } = params; + + const connectionId = Number.parseInt(id); + if (isNaN(connectionId)) { + return notFound(); + } + + const connection = await getConnectionWithJobs(connectionId); + if (isServiceError(connection)) { + throw new ServiceErrorException(connection); + } + + const configSettings = await getConfigSettings(env.CONFIG_PATH); + + const nextSyncAttempt = (() => { + const latestJob = connection.syncJobs.length > 0 ? connection.syncJobs[0] : null; + if (!latestJob) { + return undefined; + } + + if (latestJob.completedAt) { + return new Date(latestJob.completedAt.getTime() + configSettings.resyncConnectionIntervalMs); + } + + return undefined; + })(); + + // Extracts the code host URL from the connection config. + const codeHostUrl: string = (() => { + const connectionType = connection.connectionType; + switch (connectionType) { + case 'github': { + const config = connection.config as unknown as GithubConnectionConfig; + return config.url ?? 'https://github.com'; + } + case 'gitlab': { + const config = connection.config as unknown as GitlabConnectionConfig; + return config.url ?? 'https://gitlab.com'; + } + case 'gitea': { + const config = connection.config as unknown as GiteaConnectionConfig; + return config.url ?? 'https://gitea.com'; + } + case 'gerrit': { + const config = connection.config as unknown as GerritConnectionConfig; + return config.url; + } + case 'bitbucket': { + const config = connection.config as unknown as BitbucketConnectionConfig; + if (config.deploymentType === 'cloud') { + return config.url ?? 'https://bitbucket.org'; + } else { + return config.url!; + } + } + case 'azuredevops': { + const config = connection.config as unknown as AzureDevOpsConnectionConfig; + return config.url ?? 'https://dev.azure.com'; + } + case 'git': { + const config = connection.config as unknown as GenericGitHostConnectionConfig; + return config.url; + } + } + })(); + + return ( +
+ +
+

{connection.name}

+ + + {codeHostUrl} + +
+ +
+ + + + Created + + + + + +

When this connection was first added to Sourcebot

+
+
+
+
+ + + +
+ + + + + Last synced + + + + + +

The last time this connection was successfully synced

+
+
+
+
+ + {connection.syncedAt ? : "Never"} + +
+ + + + + Scheduled + + + + + +

When the connection will be resynced next. Modifying the config will also trigger a resync.

+
+
+
+
+ + {nextSyncAttempt ? : "-"} + +
+
+ + + + Sync History + History of all sync jobs for this connection. + + + }> + + + + +
+ ) +} + +const getConnectionWithJobs = async (id: number) => sew(() => + withAuth(async ({ prisma, org }) => { + const connection = await prisma.connection.findUnique({ + where: { + id, + orgId: org.id, + }, + include: { + syncJobs: { + orderBy: { + createdAt: 'desc', + }, + }, + }, + }); + + if (!connection) { + return notFoundServiceError(); + } + + return connection; + }) +) \ No newline at end of file diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx new file mode 100644 index 000000000..9277d81b7 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx @@ -0,0 +1,344 @@ +"use client" + +import { Badge } from "@/components/ui/badge" +import { Button } from "@/components/ui/button" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" +import { + type ColumnDef, + type ColumnFiltersState, + type SortingState, + type VisibilityState, + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, +} from "@tanstack/react-table" +import { cva } from "class-variance-authority" +import { AlertCircle, AlertTriangle, ArrowUpDown, PlusCircleIcon, RefreshCwIcon } from "lucide-react" +import * as React from "react" +import { CopyIconButton } from "@/app/(app)/components/copyIconButton" +import { useMemo } from "react" +import { LightweightCodeHighlighter } from "@/app/(app)/components/lightweightCodeHighlighter" +import { useRouter } from "next/navigation" +import { useToast } from "@/components/hooks/use-toast" +import { DisplayDate } from "@/app/(app)/components/DisplayDate" +import { LoadingButton } from "@/components/ui/loading-button" +import { syncConnection } from "@/features/workerApi/actions" +import { isServiceError } from "@/lib/utils" + + +export type ConnectionSyncJob = { + id: string + status: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" + createdAt: Date + updatedAt: Date + completedAt: Date | null + errorMessage: string | null + warningMessages: string[] +} + +const statusBadgeVariants = cva("", { + variants: { + status: { + PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", + IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", + COMPLETED: "bg-green-600 text-white hover:bg-green-700", + FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", + }, + }, +}) + +const getStatusBadge = (status: ConnectionSyncJob["status"]) => { + const labels = { + PENDING: "Pending", + IN_PROGRESS: "In Progress", + COMPLETED: "Completed", + FAILED: "Failed", + } + + return {labels[status]} +} + +const getDuration = (start: Date, end: Date | null) => { + if (!end) return "-" + const diff = end.getTime() - start.getTime() + const minutes = Math.floor(diff / 60000) + const seconds = Math.floor((diff % 60000) / 1000) + return `${minutes}m ${seconds}s` +} + +export const columns: ColumnDef[] = [ + { + accessorKey: "status", + header: "Status", + cell: ({ row }) => { + const job = row.original + return ( +
+ {getStatusBadge(row.getValue("status"))} + {job.errorMessage ? ( + + + + + + + + {job.errorMessage} + + + + + ) : job.warningMessages.length > 0 ? ( + + + + + + +

{job.warningMessages.length} warning(s) while syncing:

+
+ {job.warningMessages.map((warning, index) => ( +
+ {index + 1}. + {warning} +
+ ))} +
+
+
+
+ ) : null} +
+ ) + }, + filterFn: (row, id, value) => { + return value.includes(row.getValue(id)) + }, + }, + { + accessorKey: "createdAt", + header: ({ column }) => { + return ( + + ) + }, + cell: ({ row }) => , + }, + { + accessorKey: "completedAt", + header: ({ column }) => { + return ( + + ) + }, + cell: ({ row }) => { + const completedAt = row.getValue("completedAt") as Date | null; + if (!completedAt) { + return "-"; + } + + return + }, + }, + { + id: "duration", + header: "Duration", + cell: ({ row }) => { + const job = row.original + return getDuration(job.createdAt, job.completedAt) + }, + }, + { + accessorKey: "id", + header: "Job ID", + cell: ({ row }) => { + const id = row.getValue("id") as string + return ( +
+ {id} + { + navigator.clipboard.writeText(id); + return true; + }} /> +
+ ) + }, + }, +] + +export const ConnectionJobsTable = ({ data, connectionId }: { data: ConnectionSyncJob[], connectionId: number }) => { + const [sorting, setSorting] = React.useState([{ id: "createdAt", desc: true }]) + const [columnFilters, setColumnFilters] = React.useState([]) + const [columnVisibility, setColumnVisibility] = React.useState({}) + const router = useRouter(); + const { toast } = useToast(); + + const [isSyncSubmitting, setIsSyncSubmitting] = React.useState(false); + const onSyncButtonClick = React.useCallback(async () => { + setIsSyncSubmitting(true); + const response = await syncConnection(connectionId); + + if (!isServiceError(response)) { + const { jobId } = response; + toast({ + description: `✅ Connection synced successfully. Job ID: ${jobId}`, + }) + router.refresh(); + } else { + toast({ + description: `❌ Failed to sync connection. ${response.message}`, + }); + } + + setIsSyncSubmitting(false); + }, [connectionId, router, toast]); + + const table = useReactTable({ + data, + columns, + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onColumnVisibilityChange: setColumnVisibility, + state: { + sorting, + columnFilters, + columnVisibility, + }, + }) + + const { + numCompleted, + numInProgress, + numPending, + numFailed, + } = useMemo(() => { + return { + numCompleted: data.filter((job) => job.status === "COMPLETED").length, + numInProgress: data.filter((job) => job.status === "IN_PROGRESS").length, + numPending: data.filter((job) => job.status === "PENDING").length, + numFailed: data.filter((job) => job.status === "FAILED").length, + }; + }, [data]); + + return ( +
+
+ + +
+ + + + + Trigger sync + +
+
+ +
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => { + return ( + + {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} + + ) + })} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + {flexRender(cell.column.columnDef.cell, cell.getContext())} + ))} + + )) + ) : ( + + + No sync jobs found. + + + )} + +
+
+ +
+
+ {table.getFilteredRowModel().rows.length} job(s) total +
+
+ + +
+
+
+ ) +} diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx new file mode 100644 index 000000000..e52e7ef1e --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx @@ -0,0 +1,294 @@ +"use client" + +import { DisplayDate } from "@/app/(app)/components/DisplayDate" +import { NotificationDot } from "@/app/(app)/components/notificationDot" +import { useToast } from "@/components/hooks/use-toast" +import { Badge } from "@/components/ui/badge" +import { Button } from "@/components/ui/button" +import { Input } from "@/components/ui/input" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" +import { getCodeHostIcon } from "@/lib/utils" +import { ConnectionType } from "@sourcebot/db" +import { + type ColumnDef, + type ColumnFiltersState, + type SortingState, + type VisibilityState, + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, +} from "@tanstack/react-table" +import { cva } from "class-variance-authority" +import { ArrowUpDown, RefreshCwIcon } from "lucide-react" +import Image from "next/image" +import Link from "next/link" +import { useRouter } from "next/navigation" +import { useMemo, useState } from "react" + + +export type Connection = { + id: number + name: string + syncedAt: Date | null + connectionType: ConnectionType + latestJobStatus: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" | null + isFirstTimeSync: boolean +} + +const statusBadgeVariants = cva("", { + variants: { + status: { + PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", + IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", + COMPLETED: "bg-green-600 text-white hover:bg-green-700", + FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", + }, + }, +}) + +const getStatusBadge = (status: Connection["latestJobStatus"]) => { + if (!status) { + return "-"; + } + + const labels = { + PENDING: "Pending", + IN_PROGRESS: "In Progress", + COMPLETED: "Completed", + FAILED: "Failed", + } + + return {labels[status]} +} + +export const columns: ColumnDef[] = [ + { + accessorKey: "name", + size: 400, + header: ({ column }) => { + return ( + + ) + }, + cell: ({ row }) => { + const connection = row.original; + const codeHostIcon = getCodeHostIcon(connection.connectionType); + + return ( +
+ {`${connection.connectionType} + + {connection.name} + + {connection.isFirstTimeSync && ( + + + + + + + + This is the first time Sourcebot is syncing this connection. It may take a few minutes to complete. + + + )} +
+ ) + }, + }, + { + accessorKey: "latestJobStatus", + size: 150, + header: "Lastest status", + cell: ({ row }) => getStatusBadge(row.getValue("latestJobStatus")), + }, + { + accessorKey: "syncedAt", + size: 200, + header: ({ column }) => { + return ( + + ) + }, + cell: ({ row }) => { + const syncedAt = row.getValue("syncedAt") as Date | null; + if (!syncedAt) { + return "-"; + } + + return ( + + ) + } + }, +] + +export const ConnectionsTable = ({ data }: { data: Connection[] }) => { + const [sorting, setSorting] = useState([]) + const [columnFilters, setColumnFilters] = useState([]) + const [columnVisibility, setColumnVisibility] = useState({}) + const [rowSelection, setRowSelection] = useState({}) + const router = useRouter(); + const { toast } = useToast(); + + const { + numCompleted, + numInProgress, + numPending, + numFailed, + numNoJobs, + } = useMemo(() => { + return { + numCompleted: data.filter((connection) => connection.latestJobStatus === "COMPLETED").length, + numInProgress: data.filter((connection) => connection.latestJobStatus === "IN_PROGRESS").length, + numPending: data.filter((connection) => connection.latestJobStatus === "PENDING").length, + numFailed: data.filter((connection) => connection.latestJobStatus === "FAILED").length, + numNoJobs: data.filter((connection) => connection.latestJobStatus === null).length, + } + }, [data]); + + const table = useReactTable({ + data, + columns, + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onColumnVisibilityChange: setColumnVisibility, + onRowSelectionChange: setRowSelection, + columnResizeMode: 'onChange', + enableColumnResizing: false, + state: { + sorting, + columnFilters, + columnVisibility, + rowSelection, + }, + }) + + return ( +
+
+ table.getColumn("name")?.setFilterValue(event.target.value)} + className="max-w-sm" + /> + + +
+
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => { + return ( + + {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} + + ) + })} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender(cell.column.columnDef.cell, cell.getContext())} + + ))} + + )) + ) : ( + + + No results. + + + )} + +
+
+
+
+ {table.getFilteredRowModel().rows.length} {data.length > 1 ? 'connections' : 'connection'} total +
+
+ + +
+
+
+ ) +} diff --git a/packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx b/packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx deleted file mode 100644 index 0c41c6b38..000000000 --- a/packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx +++ /dev/null @@ -1,203 +0,0 @@ -"use client"; - -import { Button } from "@/components/ui/button"; -import { - Dialog, - DialogContent, - DialogDescription, - DialogHeader, - DialogTitle, -} from "@/components/ui/dialog"; -import { getConnectionSyncJobLogs } from "@/features/workerApi/actions"; -import { cn, isServiceError } from "@/lib/utils"; -import type { JobLogEntry, JobLogLevel } from "@sourcebot/shared"; -import { Loader2, RefreshCw } from "lucide-react"; -import { useEffect, useRef, useState } from "react"; - -const LOG_POLL_INTERVAL_MS = 1000; - -const JOB_LOG_LEVEL_CLASS_NAMES: Record = { - debug: "text-muted-foreground", - info: "text-blue-600 dark:text-blue-400", - warn: "text-amber-600 dark:text-amber-400", - error: "text-destructive", -}; - -const formatLogTimestamp = (timestamp: string | null) => { - if (!timestamp) { - return "—"; - } - return new Date(timestamp).toLocaleTimeString(); -}; - -type ConnectionSyncLogsDialogProps = { - connectionId: number; - connectionName: string; - jobId: string | null; - open: boolean; - onOpenChange: (open: boolean) => void; -}; - -export const ConnectionSyncLogsDialog = ({ - connectionId, - connectionName, - jobId, - open, - onOpenChange, -}: ConnectionSyncLogsDialogProps) => { - const [logs, setLogs] = useState([]); - const [hasLoaded, setHasLoaded] = useState(false); - const [isRefreshing, setIsRefreshing] = useState(false); - const [logsError, setLogsError] = useState(null); - const pollNowRef = useRef<() => void>(() => undefined); - - useEffect(() => { - if (!open || !jobId) { - return; - } - - let cursor = 0; - let isCancelled = false; - let isRequestInFlight = false; - - setLogs([]); - setHasLoaded(false); - setLogsError(null); - - const poll = async () => { - if (isRequestInFlight) { - return; - } - - isRequestInFlight = true; - setIsRefreshing(true); - - try { - const result = await getConnectionSyncJobLogs( - connectionId, - jobId, - cursor, - ); - if (isCancelled) { - return; - } - if (isServiceError(result)) { - setLogsError(result.message); - return; - } - - setLogsError(null); - setLogs((currentLogs) => ( - result.count < cursor - ? result.logs - : [...currentLogs, ...result.logs] - )); - cursor = result.count; - } catch { - if (!isCancelled) { - setLogsError("Failed to load logs for this sync."); - } - } finally { - if (!isCancelled) { - setHasLoaded(true); - setIsRefreshing(false); - } - isRequestInFlight = false; - } - }; - - const pollNow = () => { - void poll(); - }; - pollNowRef.current = pollNow; - - pollNow(); - const interval = window.setInterval(pollNow, LOG_POLL_INTERVAL_MS); - - return () => { - isCancelled = true; - window.clearInterval(interval); - if (pollNowRef.current === pollNow) { - pollNowRef.current = () => undefined; - } - }; - }, [connectionId, jobId, open]); - - const isInitialLoading = !hasLoaded && isRefreshing; - - return ( - - -
- - {connectionName} sync logs - - Latest sync job {jobId}. Logs update automatically while this dialog is open. - - - -
- -
- {isInitialLoading ? ( -
- - Loading logs… -
- ) : logs.length > 0 ? ( - <> - {logsError && ( -
- {logsError} -
- )} -
- {logs.map((entry, index) => ( -
- - {formatLogTimestamp(entry.timestamp)} - - - {entry.level} - -

{entry.message}

-
- ))} -
- - ) : logsError ? ( -
- {logsError} -
- ) : ( -
- No logs were recorded for this job. -
- )} -
-
-
- ); -}; diff --git a/packages/web/src/app/(app)/settings/connections/connectionsList.tsx b/packages/web/src/app/(app)/settings/connections/connectionsList.tsx deleted file mode 100644 index 49acc679b..000000000 --- a/packages/web/src/app/(app)/settings/connections/connectionsList.tsx +++ /dev/null @@ -1,282 +0,0 @@ -"use client"; - -import { DisplayDate } from "@/app/(app)/components/DisplayDate"; -import { useToast } from "@/components/hooks/use-toast"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuTrigger, -} from "@/components/ui/dropdown-menu"; -import { Input } from "@/components/ui/input"; -import { syncConnection } from "@/features/workerApi/actions"; -import { cn, getCodeHostIcon, isServiceError } from "@/lib/utils"; -import { ConnectionType } from "@sourcebot/db"; -import { AlertCircle, CheckCircle2, CircleDashed, FileText, MoreHorizontal, RefreshCw, Search } from "lucide-react"; -import Image from "next/image"; -import { useRouter } from "next/navigation"; -import { useEffect, useMemo, useState } from "react"; -import { ConnectionSyncLogsDialog } from "./connectionSyncLogsDialog"; -import { WorkloadJob } from "@sourcebot/shared"; - -const SYNC_STATUS_POLL_INTERVAL_MS = 1000; - -export type ConnectionV2 = { - id: number; - name: string; - connectionType: ConnectionType; - syncedAt: Date | null; - currentJob: WorkloadJob<'connection'> | null; -}; - -const getConnectionStatus = (connection: ConnectionV2) => { - switch (connection.currentJob?.status) { - case "PENDING": - return { - key: "syncing", - label: "Queued", - icon: , - className: "border-blue-200 bg-blue-50 text-blue-700 dark:border-blue-900 dark:bg-blue-950 dark:text-blue-300", - }; - case "IN_PROGRESS": - return { - key: "syncing", - label: "Syncing", - icon: , - className: "border-blue-200 bg-blue-50 text-blue-700 dark:border-blue-900 dark:bg-blue-950 dark:text-blue-300", - }; - case "FAILED": - return { - key: "failed", - label: "Sync failed", - icon: , - className: "border-destructive/30 bg-destructive/10 text-destructive", - }; - case "COMPLETED": - return { - key: "healthy", - label: "Healthy", - icon: , - className: "border-green-200 bg-green-50 text-green-700 dark:border-green-900 dark:bg-green-950 dark:text-green-300", - }; - default: - return connection.syncedAt - ? { - key: "healthy", - label: "Healthy", - icon: , - className: "border-green-200 bg-green-50 text-green-700 dark:border-green-900 dark:bg-green-950 dark:text-green-300", - } - : { - key: "not-synced", - label: "Never synced", - icon: , - className: "border-border bg-muted text-muted-foreground", - }; - } -}; - -const ConnectionRow = ({ connection }: { connection: ConnectionV2 }) => { - const [isSubmitting, setIsSubmitting] = useState(false); - const [submittedJobId, setSubmittedJobId] = useState(null); - const [isLogsOpen, setIsLogsOpen] = useState(false); - const router = useRouter(); - const { toast } = useToast(); - const codeHostIcon = getCodeHostIcon(connection.connectionType); - const isRunning = connection.currentJob?.status === "PENDING" || - connection.currentJob?.status === "IN_PROGRESS"; - const isSubmittedJobSettled = connection.currentJob?.id === submittedJobId && - (connection.currentJob.status === "COMPLETED" || connection.currentJob.status === "FAILED"); - const isSyncRunning = isSubmitting || isRunning || (submittedJobId !== null && !isSubmittedJobSettled); - const status = isSyncRunning && !isRunning - ? { - key: "syncing", - label: "Syncing", - icon: , - className: "border-blue-200 bg-blue-50 text-blue-700 dark:border-blue-900 dark:bg-blue-950 dark:text-blue-300", - } - : getConnectionStatus(connection); - - useEffect(() => { - if (!isSyncRunning) { - return; - } - - const interval = window.setInterval(() => { - router.refresh(); - }, SYNC_STATUS_POLL_INTERVAL_MS); - - return () => { - window.clearInterval(interval); - }; - }, [isSyncRunning, router]); - - const onSync = async () => { - setIsSubmitting(true); - try { - const result = await syncConnection(connection.id); - if (isServiceError(result)) { - toast({ - description: `❌ Failed to sync connection. ${result.message}`, - }); - return; - } - - setSubmittedJobId(result.jobId); - toast({ - description: `✅ Connection sync triggered. Job ID: ${result.jobId}`, - }); - router.refresh(); - } catch { - toast({ - description: "❌ Failed to sync connection.", - }); - } finally { - setIsSubmitting(false); - } - }; - - return ( - <> -
-
-
- {`${connection.connectionType} -
-
-

{connection.name}

-

{connection.connectionType}

-
-
- -
-

Current status

- - {status.icon} - {status.label} - - {connection.currentJob?.status === "FAILED" && connection.currentJob.errorMessage && ( -

{connection.currentJob.errorMessage}

- )} -
- -
-

Last successful sync

-

- {connection.syncedAt - ? - : Not yet synced} -

-
- - - - - - - { - setIsLogsOpen(true); - }} - > - - View logs - - { - void onSync(); - }} - > - - {isSyncRunning ? "Sync in progress" : "Trigger sync"} - - - -
- - - - ); -}; - -export const ConnectionsList = ({ data }: { data: ConnectionV2[] }) => { - const [query, setQuery] = useState(""); - const healthyCount = useMemo( - () => data.filter((connection) => getConnectionStatus(connection).key === "healthy").length, - [data], - ); - const filteredConnections = useMemo(() => { - const normalizedQuery = query.trim().toLowerCase(); - - return data.filter((connection) => ( - normalizedQuery.length === 0 || - connection.name.toLowerCase().includes(normalizedQuery) || - connection.connectionType.toLowerCase().includes(normalizedQuery) - )); - }, [data, query]); - - return ( -
-
-
- - setQuery(event.target.value)} - placeholder="Search connections..." - className="pl-9" - /> -
-

- {healthyCount} healthy · {data.length} total -

-
- -
- {filteredConnections.length > 0 ? filteredConnections.map((connection, index) => ( -
0 ? "border-t" : undefined}> - -
- )) : ( -
-

{data.length === 0 ? "No code host connections" : "No matching connections"}

-

- {data.length === 0 - ? "Add a connection to begin syncing repositories." - : "Try changing your search."} -

-
- )} -
- - {filteredConnections.length > 0 && ( -

- Showing {filteredConnections.length} of {data.length} {data.length === 1 ? "connection" : "connections"} -

- )} -
- ); -}; diff --git a/packages/web/src/app/(app)/settings/connections/page.tsx b/packages/web/src/app/(app)/settings/connections/page.tsx index a1d99cd38..fdce7b08f 100644 --- a/packages/web/src/app/(app)/settings/connections/page.tsx +++ b/packages/web/src/app/(app)/settings/connections/page.tsx @@ -1,72 +1,77 @@ +import { sew } from "@/middleware/sew"; import { ServiceErrorException } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; -import { sew } from "@/middleware/sew"; import { withAuth } from "@/middleware/withAuth"; -import { getBullMQClient } from "@/lib/bullmqClient"; import Link from "next/link"; -import { ConnectionsList } from "./connectionsList"; -import { CONNECTION_QUEUE } from "@sourcebot/shared"; +import { ConnectionsTable } from "./components/connectionsTable"; +import { ConnectionSyncJobStatus } from "@prisma/client"; const DOCS_URL = "https://docs.sourcebot.dev/docs/connections/indexing-your-code"; -export default async function ConnectionsV2Page() { - const connections = await getConnectionsWithCurrentStatus(); - if (isServiceError(connections)) { - throw new ServiceErrorException(connections); +export default async function ConnectionsPage() { + const _connections = await getConnectionsWithLatestJob(); + if (isServiceError(_connections)) { + throw new ServiceErrorException(_connections); } + // Sort connections so that first time syncs are at the top. + const connections = _connections + .map((connection) => ({ + ...connection, + isFirstTimeSync: connection.syncedAt === null && connection.syncJobs.filter((job) => job.status === ConnectionSyncJobStatus.PENDING || job.status === ConnectionSyncJobStatus.IN_PROGRESS).length > 0, + latestJobStatus: connection.syncJobs.length > 0 ? connection.syncJobs[0].status : null, + })) + .sort((a, b) => { + if (a.isFirstTimeSync && !b.isFirstTimeSync) { + return -1; + } + if (!a.isFirstTimeSync && b.isFirstTimeSync) { + return 1; + } + return a.name.localeCompare(b.name); + }); + return (
-
-

Code Host Connections

- - Prototype - -
-

- Monitor and sync your external code hosts.{" "} - - Learn more - -

+

Code Host Connections

+

Manage your connections to external code hosts. Learn more

- - + ({ + id: connection.id, + name: connection.name, + connectionType: connection.connectionType, + syncedAt: connection.syncedAt, + latestJobStatus: connection.latestJobStatus, + isFirstTimeSync: connection.isFirstTimeSync, + }))} />
- ); + ) } -const getConnectionsWithCurrentStatus = async () => sew(() => +const getConnectionsWithLatestJob = async () => sew(() => withAuth(async ({ prisma, org }) => { const connections = await prisma.connection.findMany({ where: { orgId: org.id, }, - select: { - id: true, - name: true, - connectionType: true, - syncedAt: true, - latestSyncJobId: true, + include: { + _count: { + select: { + syncJobs: true, + } + }, + syncJobs: { + orderBy: { + createdAt: 'desc' + }, + take: 1 + }, }, orderBy: { - name: 'asc', + name: 'asc' }, }); - return Promise.all(connections.map(async ({ - latestSyncJobId, - ...connection - }) => { - const job = latestSyncJobId - ? await getBullMQClient().getJob(CONNECTION_QUEUE, latestSyncJobId) - : null; - - return { - ...connection, - currentJob: job - }; - })); - }) -); + return connections; + })); \ No newline at end of file diff --git a/packages/web/src/app/(app)/settings/layout.tsx b/packages/web/src/app/(app)/settings/layout.tsx index e57e4180e..655ca28f5 100644 --- a/packages/web/src/app/(app)/settings/layout.tsx +++ b/packages/web/src/app/(app)/settings/layout.tsx @@ -3,6 +3,7 @@ import { Metadata } from "next" import { redirect } from "next/navigation"; import { auth } from "@/auth"; import { isServiceError } from "@/lib/utils"; +import { getConnectionStats } from "@/actions"; import { getOrgAccountRequests } from "@/features/membership/actions"; import { ServiceErrorException } from "@/lib/serviceError"; import { OrgRole } from "@prisma/client"; @@ -47,6 +48,10 @@ export const getSidebarNavGroups = async () => numJoinRequests = requests.length; } + const connectionStats = await getConnectionStats(); + if (isServiceError(connectionStats)) { + throw new ServiceErrorException(connectionStats); + } const hasAskEntitlement = await hasEntitlement("ask"); const groups: NavGroup[] = [ @@ -114,6 +119,8 @@ export const getSidebarNavGroups = async () => { title: "Connections", href: `/settings/connections`, + hrefRegex: `/settings/connections(/[^/]+)?$`, + isNotificationDotVisible: connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0, icon: "plug" as const, }, { diff --git a/packages/web/src/features/workerApi/actions.ts b/packages/web/src/features/workerApi/actions.ts index 906e28ac6..8492a7b7e 100644 --- a/packages/web/src/features/workerApi/actions.ts +++ b/packages/web/src/features/workerApi/actions.ts @@ -1,79 +1,38 @@ 'use server'; import { sew } from "@/middleware/sew"; -import { notFound, repositoryNotFound, unexpectedError } from "@/lib/serviceError"; +import { repositoryNotFound, unexpectedError } from "@/lib/serviceError"; import { withAuth, withOptionalAuth } from "@/middleware/withAuth"; import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; import { OrgRole } from "@sourcebot/db"; -import { CONNECTION_QUEUE, env } from "@sourcebot/shared"; +import { env } from "@sourcebot/shared"; import z from "zod"; -import { getBullMQClient } from "@/lib/bullmqClient"; +import { requestAccountPermissionSync } from "./client.server"; const WORKER_API_URL = env.WORKER_API_URL; export const syncConnection = async (connectionId: number) => sew(() => - withAuth(({ org, prisma, role }) => - withMinimumOrgRole(role, OrgRole.OWNER, async () => { - const connection = await prisma.connection.findUnique({ - where: { - id: connectionId, - orgId: org.id, - }, - select: { - id: true, - orgId: true, - }, - }); - - if (!connection) { - return notFound('Connection not found'); - } - - const jobId = await getBullMQClient().enqueue(CONNECTION_QUEUE, { - connectionId: connection.id, - orgId: connection.orgId, - }); - - return { jobId }; - }) - ) -); - -export const getConnectionSyncJobLogs = async ( - connectionId: number, - jobId: string, - start = 0, -) => sew(() => - withAuth(({ org, prisma, role }) => + withAuth(({ role }) => withMinimumOrgRole(role, OrgRole.OWNER, async () => { - const connection = await prisma.connection.findUnique({ - where: { - id: connectionId, - orgId: org.id, - }, - select: { - id: true, + const response = await fetch(`${WORKER_API_URL}/api/sync-connection`, { + method: 'POST', + body: JSON.stringify({ + connectionId + }), + headers: { + 'Content-Type': 'application/json', }, }); - if (!connection) { - return notFound('Connection not found'); - } - - const client = getBullMQClient(); - const job = await client.getJob(CONNECTION_QUEUE, jobId); - if ( - !job || - job.data.connectionId !== connection.id || - job.data.orgId !== org.id - ) { - return notFound('Connection sync job not found'); + if (!response.ok) { + return unexpectedError('Failed to sync connection'); } - return client.getJobLogs(CONNECTION_QUEUE, jobId, { - start: Number.isInteger(start) && start >= 0 ? start : 0, - ascending: true, + const data = await response.json(); + const schema = z.object({ + jobId: z.string(), }); + return schema.parse(data); }) ) ); From f6ad4b38d0444f10cdf331b0dfcefd8fc1f154fb Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Fri, 31 Jul 2026 16:20:24 -0700 Subject: [PATCH 08/15] wip --- docs/snippets/schemas/v3/index.schema.mdx | 6 ++---- packages/backend/src/connectionWorkload.ts | 24 ++++++++++++++-------- packages/backend/src/index.ts | 6 +++++- packages/backend/src/repoIndexWorkload.ts | 4 ++-- packages/schemas/src/v3/index.schema.ts | 6 ++---- packages/schemas/src/v3/index.type.ts | 1 - schemas/v3/index.json | 3 +-- 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index 43367fe87..36e908e30 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -43,8 +43,7 @@ "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", - "minimum": 1, - "deprecated": true + "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", @@ -230,8 +229,7 @@ "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", - "minimum": 1, - "deprecated": true + "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts index b05b9537e..9c35951cc 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionWorkload.ts @@ -1,14 +1,22 @@ -import { Workload } from "./types.js"; -import { prisma } from "./prisma.js"; +import { Settings, Workload } from "./types.js"; import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; import { CONNECTION_QUEUE, env, loadConfig } from "@sourcebot/shared"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import * as Sentry from "@sentry/node"; +import { PrismaClient } from "@sourcebot/db"; -export const connectionWorkload: Workload<'connection'> = { +interface Props { + db: PrismaClient, + settings: Settings; +} + +export const createConnectionWorkload = ({ + db, + settings +}: Props): Workload<'connection'> => ({ queueSpec: CONNECTION_QUEUE, - concurrency: 2, + concurrency: settings.maxConnectionSyncJobConcurrency, process: async ({ data: { connectionId, @@ -21,7 +29,7 @@ export const connectionWorkload: Workload<'connection'> = { connectionId, orgId, }); - const connection = await prisma.connection.findUniqueOrThrow({ + const connection = await db.connection.findUniqueOrThrow({ where: { id: connectionId } @@ -55,7 +63,7 @@ export const connectionWorkload: Workload<'connection'> = { // captured by the connection's config (e.g., it was deleted, marked archived, etc.), it won't // appear in the repoData array above, and so the RepoToConnection record won't be re-created. // Repos that have no RepoToConnection records are considered orphaned and can be deleted. - await prisma.$transaction(async (tx) => { + await db.$transaction(async (tx) => { const deleteStart = performance.now(); await tx.connection.update({ where: { @@ -104,7 +112,7 @@ export const connectionWorkload: Workload<'connection'> = { }); }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); - await prisma.connection.update({ + await db.connection.update({ where: { id: connectionId, }, @@ -131,7 +139,7 @@ export const connectionWorkload: Workload<'connection'> = { connectionId, }); } -} +}) const discoverConnectionRepositories = async ({ config, diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 5d6d196cf..8e3fbce99 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -13,7 +13,7 @@ import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; import { createReconciliationWorkload } from "./reconciliationWorkload.js"; import { redis } from "./redis.js"; -import { connectionWorkload } from "./connectionWorkload.js"; +import { connectionWorkload, createConnectionWorkload } from "./connectionWorkload.js"; import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; import { Api } from "./api.js"; @@ -51,6 +51,10 @@ const reconciliationWorkload = createReconciliationWorkload({ db: prisma, settings, }); +const connectionWorkload = createConnectionWorkload({ + db: prisma, + settings, +}); const repoIndexWorkload = createRepoIndexWorkload({ db: prisma, settings, diff --git a/packages/backend/src/repoIndexWorkload.ts b/packages/backend/src/repoIndexWorkload.ts index a2a66e77d..3377d1c7e 100644 --- a/packages/backend/src/repoIndexWorkload.ts +++ b/packages/backend/src/repoIndexWorkload.ts @@ -13,7 +13,7 @@ import { cleanupTempShards, indexGitRepository } from './zoekt.js'; const LOG_TAG = 'repo-index-workload'; const logger = createLogger(LOG_TAG); -interface RepoIndexWorkloadDependencies { +interface Props { db: PrismaClient; settings: Settings; } @@ -21,7 +21,7 @@ interface RepoIndexWorkloadDependencies { export const createRepoIndexWorkload = ({ db, settings, -}: RepoIndexWorkloadDependencies): Workload<'repo-index'> => ({ +}: Props): Workload<'repo-index'> => ({ queueSpec: REPO_INDEX_QUEUE, concurrency: settings.maxRepoIndexingJobConcurrency, process: async ({ data, logger: jobLogger, signal }) => { diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index ef5c461dd..15296f33a 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -42,8 +42,7 @@ const schema = { "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", - "minimum": 1, - "deprecated": true + "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", @@ -229,8 +228,7 @@ const schema = { "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", - "minimum": 1, - "deprecated": true + "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index b3b03b34d..901e7762b 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -110,7 +110,6 @@ export interface Settings { */ reindexRepoPollingIntervalMs?: number; /** - * @deprecated * The number of connection sync jobs to run concurrently. Defaults to 2. */ maxConnectionSyncJobConcurrency?: number; diff --git a/schemas/v3/index.json b/schemas/v3/index.json index 887d40eb7..6b5b4fbb1 100644 --- a/schemas/v3/index.json +++ b/schemas/v3/index.json @@ -41,8 +41,7 @@ "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", - "minimum": 1, - "deprecated": true + "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", From d367d9faf8b7c1002d6aa9877b439f8b24b75bd7 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Fri, 31 Jul 2026 16:44:39 -0700 Subject: [PATCH 09/15] add necessary lifecyle hooks --- packages/backend/src/configManager.ts | 2 +- .../backend/src/connectionWorkload.test.ts | 125 +++++++++++++--- packages/backend/src/connectionWorkload.ts | 65 +++++++- packages/backend/src/index.ts | 2 +- packages/backend/src/jobManager.test.ts | 10 +- .../src/reconciliationWorkload.test.ts | 8 +- .../backend/src/reconciliationWorkload.ts | 2 +- .../backend/src/repoIndexWorkload.test.ts | 140 ++++++++++++++++++ packages/backend/src/repoIndexWorkload.ts | 80 +++++++++- packages/shared/src/bullmqClient.test.ts | 52 +++++-- packages/shared/src/queue.ts | 47 ++++-- 11 files changed, 468 insertions(+), 65 deletions(-) create mode 100644 packages/backend/src/repoIndexWorkload.test.ts diff --git a/packages/backend/src/configManager.ts b/packages/backend/src/configManager.ts index 669d3d43e..8d04778a5 100644 --- a/packages/backend/src/configManager.ts +++ b/packages/backend/src/configManager.ts @@ -101,7 +101,7 @@ export class ConfigManager { if (connectionNeedsSyncing) { logger.debug(`Change detected for connection '${key}' (id: ${connection.id}). Creating sync job.`); - await this.jobManager.trigger('connection', { + await this.jobManager.trigger('connection-sync', { connectionId: connection.id, orgId: SINGLE_TENANT_ORG_ID, }) diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionWorkload.test.ts index d21d4a78a..679bd891b 100644 --- a/packages/backend/src/connectionWorkload.test.ts +++ b/packages/backend/src/connectionWorkload.test.ts @@ -1,8 +1,11 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; +import type { PrismaClient } from '@sourcebot/db'; const mocks = vi.hoisted(() => ({ connectionFindUniqueOrThrow: vi.fn(), connectionUpdate: vi.fn(), + connectionSyncJobUpsert: vi.fn(), + connectionSyncJobUpdate: vi.fn(), transactionConnectionUpdate: vi.fn(), transactionRepoUpsert: vi.fn(), compileGithubConfig: vi.fn(), @@ -16,7 +19,7 @@ vi.mock('@sentry/node', () => ({ vi.mock('@sourcebot/shared', () => ({ CONNECTION_QUEUE: { - name: 'connection', + name: 'connection-sync', dedupKey: ({ connectionId }: { connectionId: number }) => `connection:${connectionId}`, jobOptions: { attempts: 2, @@ -36,23 +39,6 @@ vi.mock('@sourcebot/shared', () => ({ loadConfig: mocks.loadConfig, })); -vi.mock('./prisma.js', () => ({ - prisma: { - connection: { - findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, - update: mocks.connectionUpdate, - }, - $transaction: vi.fn(async (callback) => callback({ - connection: { - update: mocks.transactionConnectionUpdate, - }, - repo: { - upsert: mocks.transactionRepoUpsert, - }, - })), - }, -})); - vi.mock('./repoCompileUtils.js', () => ({ compileAzureDevOpsConfig: vi.fn(), compileBitbucketConfig: vi.fn(), @@ -67,7 +53,35 @@ vi.mock('./ee/syncSearchContexts.js', () => ({ syncSearchContexts: mocks.syncSearchContexts, })); -import { connectionWorkload } from './connectionWorkload.js'; +import { createConnectionWorkload } from './connectionWorkload.js'; + +const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => callback({ + connection: { + update: mocks.transactionConnectionUpdate, + }, + repo: { + upsert: mocks.transactionRepoUpsert, + }, +})); + +const db = { + connection: { + findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, + update: mocks.connectionUpdate, + }, + connectionSyncJob: { + upsert: mocks.connectionSyncJobUpsert, + update: mocks.connectionSyncJobUpdate, + }, + $transaction: transaction, +} as unknown as PrismaClient; + +const connectionWorkload = createConnectionWorkload({ + db, + settings: { + maxConnectionSyncJobConcurrency: 2, + } as never, +}); const data = { connectionId: 42, @@ -79,6 +93,7 @@ const lifecycleContext = { jobId: 'job-1', attemptsMade: 0, maxAttempts: 2, + prisma: db, }; describe('connectionWorkload', () => { @@ -86,11 +101,65 @@ describe('connectionWorkload', () => { vi.clearAllMocks(); }); - test('does not declare database-backed lifecycle hooks', () => { - expect(connectionWorkload.queueSpec.onEnqueued).toBeUndefined(); - expect(connectionWorkload.onStarted).toBeUndefined(); - expect(connectionWorkload.onCompleted).toBeUndefined(); - expect(connectionWorkload.onTerminalFailure).toBeUndefined(); + test('declares database-backed lifecycle hooks', () => { + expect(connectionWorkload.onStarted).toBeTypeOf('function'); + expect(connectionWorkload.onCompleted).toBeTypeOf('function'); + expect(connectionWorkload.onTerminalFailure).toBeTypeOf('function'); + }); + + test('marks the connection sync job as in progress when started', async () => { + await connectionWorkload.onStarted?.(lifecycleContext); + + expect(mocks.connectionSyncJobUpsert).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + update: { + status: 'IN_PROGRESS', + completedAt: null, + errorMessage: null, + warningMessages: [], + }, + create: { + id: 'job-1', + connectionId: 42, + status: 'IN_PROGRESS', + warningMessages: [], + }, + }); + }); + + test('marks the connection sync job as completed', async () => { + await connectionWorkload.onCompleted?.(lifecycleContext, undefined); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'COMPLETED', + completedAt: expect.any(Date), + errorMessage: null, + }, + }); + }); + + test('marks the connection sync job as failed after terminal failure', async () => { + await connectionWorkload.onTerminalFailure?.( + lifecycleContext, + new Error('Connection credentials expired'), + ); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'FAILED', + completedAt: expect.any(Date), + errorMessage: 'Connection credentials expired', + }, + }); }); test('discovers repositories using the connection provider', async () => { @@ -136,6 +205,14 @@ describe('connectionWorkload', () => { }, ); expect(updateProgress).not.toHaveBeenCalled(); + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + warningMessages: ['Repository was archived'], + }, + }); expect(result).toBeUndefined(); }); }); diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts index 9c35951cc..a6f87353b 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionWorkload.ts @@ -4,7 +4,7 @@ import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHost import { CONNECTION_QUEUE, env, loadConfig } from "@sourcebot/shared"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import * as Sentry from "@sentry/node"; -import { PrismaClient } from "@sourcebot/db"; +import { ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db"; interface Props { db: PrismaClient, @@ -14,7 +14,7 @@ interface Props { export const createConnectionWorkload = ({ db, settings -}: Props): Workload<'connection'> => ({ +}: Props): Workload<'connection-sync'> => ({ queueSpec: CONNECTION_QUEUE, concurrency: settings.maxConnectionSyncJobConcurrency, process: async ({ @@ -24,6 +24,7 @@ export const createConnectionWorkload = ({ }, logger, signal, + jobId, }) => { logger.info(`Syncing connection ${connectionId}`, { connectionId, @@ -43,7 +44,16 @@ export const createConnectionWorkload = ({ signal, }); - let { repoData } = result; + let { repoData, warnings } = result; + + await db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + warningMessages: warnings, + }, + }); logger.info(`Discovered ${repoData.length} repositories`, { connectionId, @@ -138,8 +148,51 @@ export const createConnectionWorkload = ({ logger.info(`Connection ${connectionId} sync finished`, { connectionId, }); - } -}) + }, + onStarted: async ({ data: { connectionId }, jobId }) => { + await db.connectionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: ConnectionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + warningMessages: [], + }, + create: { + id: jobId, + connectionId, + status: ConnectionSyncJobStatus.IN_PROGRESS, + warningMessages: [], + }, + }); + }, + onCompleted: async ({ jobId }) => { + await db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + }, + }); + }, + onTerminalFailure: async ({ jobId }, error) => { + await db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + }); + }, +}); const discoverConnectionRepositories = async ({ config, @@ -173,4 +226,4 @@ const discoverConnectionRepositories = async ({ return compileGenericGitHostConfig(config, connectionId); } } -}; \ No newline at end of file +}; diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 8e3fbce99..d45f24936 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -13,7 +13,7 @@ import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; import { createReconciliationWorkload } from "./reconciliationWorkload.js"; import { redis } from "./redis.js"; -import { connectionWorkload, createConnectionWorkload } from "./connectionWorkload.js"; +import { createConnectionWorkload } from "./connectionWorkload.js"; import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; import { Api } from "./api.js"; diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts index 894efdade..2a6f1649f 100644 --- a/packages/backend/src/jobManager.test.ts +++ b/packages/backend/src/jobManager.test.ts @@ -111,10 +111,10 @@ describe('normalizeJobState', () => { }); const createWorkload = ( - overrides: Partial> = {}, -): Workload<'connection', { repoCount: number }> => ({ + overrides: Partial> = {}, +): Workload<'connection-sync', { repoCount: number }> => ({ queueSpec: { - name: 'connection', + name: 'connection-sync', dedupKey: ({ connectionId }) => `connection:${connectionId}`, jobOptions: { attempts: 2, @@ -150,7 +150,7 @@ describe('BullMQJobManager lifecycle', () => { const workload = createWorkload(); manager.register(workload); - const result = await manager.trigger('connection', data); + const result = await manager.trigger('connection-sync', data); expect(result).toBe('job-1'); expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data); @@ -183,7 +183,7 @@ describe('BullMQJobManager lifecycle', () => { }); test('provides the structured job logger to the workload processor', async () => { - const process = vi.fn(async (context: ProcessContext<'connection'>) => { + const process = vi.fn(async (context: ProcessContext<'connection-sync'>) => { context.logger.info('Processing connection'); return { repoCount: 3 }; }); diff --git a/packages/backend/src/reconciliationWorkload.test.ts b/packages/backend/src/reconciliationWorkload.test.ts index 949af1c50..eadb37b88 100644 --- a/packages/backend/src/reconciliationWorkload.test.ts +++ b/packages/backend/src/reconciliationWorkload.test.ts @@ -53,14 +53,14 @@ describe('reconciliationWorkload', () => { vi.useRealTimers(); }); - test('runs every 15 minutes on the reconciliation queue', () => { + test('runs every 10 seconds on the reconciliation queue', () => { const workload = createReconciliationWorkload({ db, settings, }); expect(workload.queueSpec.name).toBe('reconciliation'); - expect(workload.schedule).toEqual({ every: '15m' }); + expect(workload.schedule).toEqual({ every: '10s' }); expect(workload.concurrency).toBe(1); }); @@ -100,11 +100,11 @@ describe('reconciliationWorkload', () => { }, }); expect(trigger).toHaveBeenCalledTimes(2); - expect(trigger).toHaveBeenCalledWith('connection', { + expect(trigger).toHaveBeenCalledWith('connection-sync', { connectionId: 42, orgId: 1, }); - expect(trigger).toHaveBeenCalledWith('connection', { + expect(trigger).toHaveBeenCalledWith('connection-sync', { connectionId: 84, orgId: 2, }); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts index f34008b2a..90a14f14e 100644 --- a/packages/backend/src/reconciliationWorkload.ts +++ b/packages/backend/src/reconciliationWorkload.ts @@ -31,7 +31,7 @@ export const createReconciliationWorkload = ({ await Promise.all(connections.map(async (connection) => { logger.debug(`Scheduling connection sync for connection ${connection.id}`); - await trigger('connection', { + await trigger('connection-sync', { connectionId: connection.id, orgId: connection.orgId, }); diff --git a/packages/backend/src/repoIndexWorkload.test.ts b/packages/backend/src/repoIndexWorkload.test.ts new file mode 100644 index 000000000..076814b3d --- /dev/null +++ b/packages/backend/src/repoIndexWorkload.test.ts @@ -0,0 +1,140 @@ +import type { PrismaClient } from '@sourcebot/db'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { createRepoIndexWorkload } from './repoIndexWorkload.js'; + +const repoIndexingJobUpsert = vi.fn(); +const repoIndexingJobUpdate = vi.fn(); +const repoUpdate = vi.fn(); +const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => callback({ + repoIndexingJob: { + upsert: repoIndexingJobUpsert, + update: repoIndexingJobUpdate, + }, + repo: { + update: repoUpdate, + }, +})); + +const db = { + $transaction: transaction, +} as unknown as PrismaClient; + +const workload = createRepoIndexWorkload({ + db, + settings: { + maxRepoIndexingJobConcurrency: 2, + } as never, +}); + +const lifecycleContext = { + data: { + repoId: 42, + type: 'INDEX' as const, + }, + jobId: 'job-1', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, +}; + +describe('repoIndexWorkload lifecycle', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + test('declares database-backed lifecycle hooks', () => { + expect(workload.onStarted).toBeTypeOf('function'); + expect(workload.onCompleted).toBeTypeOf('function'); + expect(workload.onTerminalFailure).toBeTypeOf('function'); + }); + + test('marks the repo indexing job and repo as in progress when started', async () => { + await workload.onStarted?.(lifecycleContext); + + expect(repoIndexingJobUpsert).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + update: { + status: 'IN_PROGRESS', + completedAt: null, + errorMessage: null, + }, + create: { + id: 'job-1', + repoId: 42, + type: 'INDEX', + status: 'IN_PROGRESS', + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestIndexingJobStatus: 'IN_PROGRESS', + }, + }); + }); + + test('marks the repo indexing job and repo as completed', async () => { + await workload.onCompleted?.(lifecycleContext, undefined); + + expect(repoIndexingJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'COMPLETED', + completedAt: expect.any(Date), + errorMessage: null, + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestIndexingJobStatus: 'COMPLETED', + }, + }); + }); + + test('does not update a completed cleanup job after its repo cascades the job row', async () => { + await workload.onCompleted?.({ + ...lifecycleContext, + data: { + repoId: 42, + type: 'CLEANUP', + }, + }, undefined); + + expect(transaction).not.toHaveBeenCalled(); + }); + + test('marks the repo indexing job and repo as failed after terminal failure', async () => { + await workload.onTerminalFailure?.( + lifecycleContext, + new Error('Unable to clone repository'), + ); + + expect(repoIndexingJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'FAILED', + completedAt: expect.any(Date), + errorMessage: 'Unable to clone repository', + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestIndexingJobStatus: 'FAILED', + }, + }); + }); +}); diff --git a/packages/backend/src/repoIndexWorkload.ts b/packages/backend/src/repoIndexWorkload.ts index 3377d1c7e..7e21d9493 100644 --- a/packages/backend/src/repoIndexWorkload.ts +++ b/packages/backend/src/repoIndexWorkload.ts @@ -1,4 +1,4 @@ -import { PrismaClient, Repo } from "@sourcebot/db"; +import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import { createLogger, getRepoPath, JobLogSink, getRepoIdFromPath, RepoMetadata, repoMetadataSchema, REPO_INDEX_QUEUE } from "@sourcebot/shared"; import { existsSync } from 'fs'; import { readdir, rm } from 'fs/promises'; @@ -81,6 +81,84 @@ export const createRepoIndexWorkload = ({ } } }, + onStarted: async ({ data: { repoId, type }, jobId }) => { + await db.$transaction(async (tx) => { + await tx.repoIndexingJob.upsert({ + where: { + id: jobId, + }, + update: { + status: RepoIndexingJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + repoId, + type: RepoIndexingJobType[type], + status: RepoIndexingJobStatus.IN_PROGRESS, + }, + }); + await tx.repo.update({ + where: { + id: repoId, + }, + data: { + latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, + }, + }); + }); + }, + onCompleted: async ({ data: { repoId, type }, jobId }) => { + // A successful cleanup deletes the Repo in `process`, which cascades to its + // RepoIndexingJob records. There is no row left to mark as completed. + if (type === RepoIndexingJobType.CLEANUP) { + return; + } + + await db.$transaction(async (tx) => { + await tx.repoIndexingJob.update({ + where: { + id: jobId, + }, + data: { + status: RepoIndexingJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + }, + }); + await tx.repo.update({ + where: { + id: repoId, + }, + data: { + latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, + }, + }); + }); + }, + onTerminalFailure: async ({ data: { repoId }, jobId }, error) => { + await db.$transaction(async (tx) => { + await tx.repoIndexingJob.update({ + where: { + id: jobId, + }, + data: { + status: RepoIndexingJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + }); + await tx.repo.update({ + where: { + id: repoId, + }, + data: { + latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, + }, + }); + }); + }, }); const indexRepository = async ( diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts index c73b1fb5d..34955363c 100644 --- a/packages/shared/src/bullmqClient.test.ts +++ b/packages/shared/src/bullmqClient.test.ts @@ -2,7 +2,7 @@ import type { Redis } from 'ioredis'; import { beforeEach, describe, expect, test, vi } from 'vitest'; import type { QueueSpec } from './queue.js'; import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from './jobLogger.js'; -import { REPO_INDEX_QUEUE } from './queue.js'; +import { CONNECTION_QUEUE, REPO_INDEX_QUEUE } from './queue.js'; const queueMocks = vi.hoisted(() => ({ add: vi.fn(), @@ -32,8 +32,8 @@ vi.mock('bullmq', () => ({ import { BullMQClient } from './bullmqClient.js'; import { PrismaClient } from '@sourcebot/db'; -const connectionSpec: QueueSpec<'connection'> = { - name: 'connection', +const connectionSpec: QueueSpec<'connection-sync'> = { + name: 'connection-sync', dedupKey: ({ connectionId }) => `connection:${connectionId}`, jobOptions: { attempts: 2, @@ -61,7 +61,7 @@ describe('BullMQClient', () => { expect(result).toEqual(expect.any(String)); expect(queueMocks.add).toHaveBeenCalledWith( - 'connection', + 'connection-sync', data, expect.objectContaining({ jobId: result, @@ -97,11 +97,35 @@ describe('BullMQClient', () => { expect(onEnqueued).not.toHaveBeenCalled(); }); - test('stores the latest repo indexing job id when enqueueing a repo job', async () => { - const repoUpdate = vi.fn(); + test('upserts a pending connection sync job when enqueueing a connection sync', async () => { + const connectionSyncJobUpsert = vi.fn(); const client = new BullMQClient(redis, { - repo: { - update: repoUpdate, + connectionSyncJob: { + upsert: connectionSyncJobUpsert, + }, + } as unknown as PrismaClient); + + const result = await client.enqueue(CONNECTION_QUEUE, data); + + expect(connectionSyncJobUpsert).toHaveBeenCalledWith({ + where: { + id: result, + }, + update: {}, + create: { + id: result, + connectionId: 42, + status: 'PENDING', + warningMessages: [], + }, + }); + }); + + test('upserts a pending repo indexing job when enqueueing with repo-level deduplication', async () => { + const repoIndexingJobUpsert = vi.fn(); + const client = new BullMQClient(redis, { + repoIndexingJob: { + upsert: repoIndexingJobUpsert, }, } as unknown as PrismaClient); @@ -120,12 +144,16 @@ describe('BullMQClient', () => { deduplication: { id: 'repo:42' }, }), ); - expect(repoUpdate).toHaveBeenCalledWith({ + expect(repoIndexingJobUpsert).toHaveBeenCalledWith({ where: { - id: 42, + id: result, }, - data: { - latestIndexingJobId: result, + update: {}, + create: { + id: result, + repoId: 42, + type: 'INDEX', + status: 'PENDING', }, }); }); diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 81e20c21f..7c65ac5ef 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -1,5 +1,5 @@ -import { PrismaClient } from "@sourcebot/db"; +import { ConnectionSyncJobStatus, PrismaClient, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from "./jobLogger.js"; export type QueueName = keyof QueueRegistry; @@ -7,37 +7,50 @@ export type DataOf = QueueRegistry[TName]; type EmptyJobData = Record; interface QueueRegistry { - 'connection': { + 'reconciliation': EmptyJobData, + 'connection-sync': { connectionId: number, orgId: number }, - 'reconciliation': EmptyJobData, 'repo-index': { repoId: number, type: 'INDEX' | 'CLEANUP', }, } -export const CONNECTION_QUEUE: QueueSpec<'connection'> = { - name: 'connection', +export const RECONCILIATION_QUEUE: QueueSpec<'reconciliation'> = { + name: 'reconciliation', jobOptions: { attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, - dedupKey: (data) => `connection:${data.connectionId}`, -} - +}; -export const RECONCILIATION_QUEUE: QueueSpec<'reconciliation'> = { - name: 'reconciliation', +export const CONNECTION_QUEUE: QueueSpec<'connection-sync'> = { + name: 'connection-sync', jobOptions: { attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, + dedupKey: (data) => `connection:${data.connectionId}`, + onEnqueued: async ({ prisma, data: { connectionId }, jobId }) => { + await prisma.connectionSyncJob.upsert({ + where: { + id: jobId, + }, + update: {}, + create: { + id: jobId, + connectionId, + status: ConnectionSyncJobStatus.PENDING, + warningMessages: [], + }, + }); + } }; export const REPO_INDEX_QUEUE: QueueSpec<'repo-index'> = { @@ -49,6 +62,20 @@ export const REPO_INDEX_QUEUE: QueueSpec<'repo-index'> = { keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, dedupKey: (data) => `repo:${data.repoId}`, + onEnqueued: async ({ prisma, data: { repoId, type }, jobId }) => { + await prisma.repoIndexingJob.upsert({ + where: { + id: jobId, + }, + update: {}, + create: { + id: jobId, + repoId, + type: RepoIndexingJobType[type], + status: RepoIndexingJobStatus.PENDING, + }, + }); + }, }; export interface QueueSpec { From c7d9744b00bbc53dd21ead168c50fb49cac24557 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 3 Aug 2026 15:55:57 -0700 Subject: [PATCH 10/15] further wip --- docs/snippets/schemas/v3/index.schema.mdx | 20 +- packages/backend/src/bitbucket.ts | 2 +- .../backend/src/connectionWorkload.test.ts | 322 +++++++++-- packages/backend/src/connectionWorkload.ts | 157 +++-- .../ee/accountPermissionSyncWorkload.test.ts | 423 ++++++++++++++ .../src/ee/accountPermissionSyncWorkload.ts | 525 +++++++++++++++++ .../src/ee/accountPermissionSyncer.test.ts | 205 ------- .../backend/src/ee/accountPermissionSyncer.ts | 535 ------------------ .../src/ee/repoPermissionSyncWorkload.test.ts | 361 ++++++++++++ .../src/ee/repoPermissionSyncWorkload.ts | 407 +++++++++++++ .../backend/src/ee/repoPermissionSyncer.ts | 436 -------------- packages/backend/src/index.ts | 12 + packages/backend/src/jobManager.test.ts | 229 +++++--- packages/backend/src/jobManager.ts | 184 ++++-- .../src/reconciliationWorkload.test.ts | 199 +++++++ .../backend/src/reconciliationWorkload.ts | 200 +++++-- .../backend/src/repoIndexWorkload.test.ts | 99 ++-- packages/backend/src/types.ts | 60 +- packages/schemas/src/v3/index.schema.ts | 20 +- packages/schemas/src/v3/index.type.ts | 10 +- packages/shared/src/bullmqClient.test.ts | 251 -------- packages/shared/src/bullmqClient.ts | 22 - packages/shared/src/constants.ts | 8 +- packages/shared/src/env.server.ts | 1 - packages/shared/src/index.server.ts | 3 +- packages/shared/src/jobLogger.test.ts | 19 + packages/shared/src/jobLogger.ts | 33 +- packages/shared/src/queue.ts | 104 ++-- packages/web/src/lib/bullmqClient.ts | 3 +- schemas/v3/index.json | 10 +- 30 files changed, 2948 insertions(+), 1912 deletions(-) create mode 100644 packages/backend/src/ee/accountPermissionSyncWorkload.test.ts create mode 100644 packages/backend/src/ee/accountPermissionSyncWorkload.ts delete mode 100644 packages/backend/src/ee/accountPermissionSyncer.test.ts delete mode 100644 packages/backend/src/ee/accountPermissionSyncer.ts create mode 100644 packages/backend/src/ee/repoPermissionSyncWorkload.test.ts create mode 100644 packages/backend/src/ee/repoPermissionSyncWorkload.ts delete mode 100644 packages/backend/src/ee/repoPermissionSyncer.ts delete mode 100644 packages/shared/src/bullmqClient.test.ts diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index 36e908e30..1cff30599 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -42,17 +42,17 @@ }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", "minimum": 1, "deprecated": true }, @@ -96,12 +96,12 @@ }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 } }, @@ -228,17 +228,17 @@ }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", "minimum": 1, "deprecated": true }, @@ -282,12 +282,12 @@ }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 } }, diff --git a/packages/backend/src/bitbucket.ts b/packages/backend/src/bitbucket.ts index 9051164a7..84f85d023 100644 --- a/packages/backend/src/bitbucket.ts +++ b/packages/backend/src/bitbucket.ts @@ -755,7 +755,7 @@ export const getReposForAuthenticatedBitbucketServerUser = async ( * @note This only covers direct user-to-repo grants. It does NOT include users who have access via: * - Project-level permissions (inherited by all repos in the project) * - Group membership - * These users will still gain access through account-driven syncing (accountPermissionSyncer). + * These users will still gain access through account-driven permission syncing. * * @see https://developer.atlassian.com/server/bitbucket/rest/v906/api-group-repository/#api-rest-api-latest-projects-projectkey-repos-reposlug-permissions-users-get */ diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionWorkload.test.ts index 679bd891b..d55deb752 100644 --- a/packages/backend/src/connectionWorkload.test.ts +++ b/packages/backend/src/connectionWorkload.test.ts @@ -1,29 +1,31 @@ -import { beforeEach, describe, expect, test, vi } from 'vitest'; -import type { PrismaClient } from '@sourcebot/db'; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import type { PrismaClient } from "@sourcebot/db"; const mocks = vi.hoisted(() => ({ connectionFindUniqueOrThrow: vi.fn(), connectionUpdate: vi.fn(), connectionSyncJobUpsert: vi.fn(), connectionSyncJobUpdate: vi.fn(), - transactionConnectionUpdate: vi.fn(), - transactionRepoUpsert: vi.fn(), + repoFindMany: vi.fn(), + repoUpsert: vi.fn(), + repoToConnectionDeleteMany: vi.fn(), compileGithubConfig: vi.fn(), loadConfig: vi.fn(), syncSearchContexts: vi.fn(), })); -vi.mock('@sentry/node', () => ({ +vi.mock("@sentry/node", () => ({ captureException: vi.fn(), })); -vi.mock('@sourcebot/shared', () => ({ +vi.mock("@sourcebot/shared", () => ({ CONNECTION_QUEUE: { - name: 'connection-sync', - dedupKey: ({ connectionId }: { connectionId: number }) => `connection:${connectionId}`, + name: "connection-sync", + dedupKey: ({ connectionId }: { connectionId: number }) => + `connection:${connectionId}`, jobOptions: { attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, + backoff: { type: "exponential", delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: 500, }, @@ -33,13 +35,13 @@ vi.mock('@sourcebot/shared', () => ({ error: vi.fn(), })), env: { - CONFIG_PATH: '/config.json', + CONFIG_PATH: "/config.json", CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: 60_000, }, loadConfig: mocks.loadConfig, })); -vi.mock('./repoCompileUtils.js', () => ({ +vi.mock("./repoCompileUtils.js", () => ({ compileAzureDevOpsConfig: vi.fn(), compileBitbucketConfig: vi.fn(), compileGenericGitHostConfig: vi.fn(), @@ -49,31 +51,28 @@ vi.mock('./repoCompileUtils.js', () => ({ compileGitlabConfig: vi.fn(), })); -vi.mock('./ee/syncSearchContexts.js', () => ({ +vi.mock("./ee/syncSearchContexts.js", () => ({ syncSearchContexts: mocks.syncSearchContexts, })); -import { createConnectionWorkload } from './connectionWorkload.js'; - -const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => callback({ - connection: { - update: mocks.transactionConnectionUpdate, - }, - repo: { - upsert: mocks.transactionRepoUpsert, - }, -})); +import { createConnectionWorkload } from "./connectionWorkload.js"; const db = { connection: { findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, update: mocks.connectionUpdate, }, + repo: { + findMany: mocks.repoFindMany, + upsert: mocks.repoUpsert, + }, + repoToConnection: { + deleteMany: mocks.repoToConnectionDeleteMany, + }, connectionSyncJob: { upsert: mocks.connectionSyncJobUpsert, update: mocks.connectionSyncJobUpdate, }, - $transaction: transaction, } as unknown as PrismaClient; const connectionWorkload = createConnectionWorkload({ @@ -88,94 +87,106 @@ const data = { orgId: 7, }; +const lifecycleLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + const lifecycleContext = { data, - jobId: 'job-1', + jobId: "job-1", attemptsMade: 0, maxAttempts: 2, prisma: db, + logger: lifecycleLogger, }; -describe('connectionWorkload', () => { +describe("connectionWorkload", () => { beforeEach(() => { vi.clearAllMocks(); }); - test('declares database-backed lifecycle hooks', () => { - expect(connectionWorkload.onStarted).toBeTypeOf('function'); - expect(connectionWorkload.onCompleted).toBeTypeOf('function'); - expect(connectionWorkload.onTerminalFailure).toBeTypeOf('function'); + test("declares database-backed lifecycle hooks", () => { + expect(connectionWorkload.onStarted).toBeTypeOf("function"); + expect(connectionWorkload.onCompleted).toBeTypeOf("function"); + expect(connectionWorkload.onTerminalFailure).toBeTypeOf("function"); }); - test('marks the connection sync job as in progress when started', async () => { + test("marks the connection sync job as in progress when started", async () => { await connectionWorkload.onStarted?.(lifecycleContext); expect(mocks.connectionSyncJobUpsert).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, update: { - status: 'IN_PROGRESS', + status: "IN_PROGRESS", completedAt: null, errorMessage: null, warningMessages: [], }, create: { - id: 'job-1', + id: "job-1", connectionId: 42, - status: 'IN_PROGRESS', + status: "IN_PROGRESS", warningMessages: [], }, }); }); - test('marks the connection sync job as completed', async () => { - await connectionWorkload.onCompleted?.(lifecycleContext, undefined); + test("marks the connection sync job as completed", async () => { + await connectionWorkload.onCompleted?.(lifecycleContext, { + reposToCleanup: [], + reposToIndex: [], + }); expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, data: { - status: 'COMPLETED', + status: "COMPLETED", completedAt: expect.any(Date), errorMessage: null, }, }); }); - test('marks the connection sync job as failed after terminal failure', async () => { + test("marks the connection sync job as failed after terminal failure", async () => { await connectionWorkload.onTerminalFailure?.( lifecycleContext, - new Error('Connection credentials expired'), + new Error("Connection credentials expired"), ); expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, data: { - status: 'FAILED', + status: "FAILED", completedAt: expect.any(Date), - errorMessage: 'Connection credentials expired', + errorMessage: "Connection credentials expired", }, }); }); - test('discovers repositories using the connection provider', async () => { + test("discovers repositories using the connection provider", async () => { const config = { - type: 'github' as const, + type: "github" as const, }; mocks.connectionFindUniqueOrThrow.mockResolvedValue({ id: 42, - name: 'github', + name: "github", config, }); mocks.compileGithubConfig.mockResolvedValue({ repoData: [], - warnings: ['Repository was archived'], + warnings: ["Repository was archived"], }); mocks.connectionUpdate.mockResolvedValue({}); + mocks.repoFindMany.mockResolvedValue([]); mocks.loadConfig.mockResolvedValue({ contexts: undefined }); mocks.syncSearchContexts.mockResolvedValue(undefined); const updateProgress = vi.fn(); @@ -196,23 +207,220 @@ describe('connectionWorkload', () => { trigger: vi.fn(), }); - expect(mocks.compileGithubConfig).toHaveBeenCalledWith(config, 42, signal); - expect(logger.info).toHaveBeenCalledWith( - 'Discovered 0 repositories', - { - connectionId: 42, - repositoryCount: 0, - }, + expect(mocks.compileGithubConfig).toHaveBeenCalledWith( + config, + 42, + signal, ); + expect(logger.info).toHaveBeenCalledWith("Discovered 0 repositories", { + connectionId: 42, + repositoryCount: 0, + }); expect(updateProgress).not.toHaveBeenCalled(); expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, data: { - warningMessages: ['Repository was archived'], + warningMessages: ["Repository was archived"], + }, + }); + expect(result).toEqual({ + reposToCleanup: [], + reposToIndex: [], + }); + }); + + test("finds orphaned repositories and repositories needing a first index", async () => { + const config = { + type: "github" as const, + }; + const existingIndexedAt = new Date("2026-07-30T12:00:00.000Z"); + const existingRepo = { + external_id: "repo-1", + external_codeHostUrl: "https://github.com", + displayName: "sourcebot/repo-1", + connections: { + create: { + connectionId: 42, + }, + }, + }; + const newRepo = { + external_id: "repo-4", + external_codeHostUrl: "https://github.com", + displayName: "sourcebot/repo-4", + connections: { + create: { + connectionId: 42, + }, + }, + }; + + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + config, + }); + mocks.compileGithubConfig.mockResolvedValue({ + repoData: [existingRepo, newRepo], + warnings: [], + }); + mocks.repoFindMany + .mockResolvedValueOnce([{ id: 1 }, { id: 2 }, { id: 3 }]) + .mockResolvedValueOnce([{ id: 2, name: "github.com/sourcebot/repo-2" }]); + mocks.repoUpsert + .mockResolvedValueOnce({ + id: 1, + name: "github.com/sourcebot/repo-1", + indexedAt: existingIndexedAt, + }) + .mockResolvedValueOnce({ + id: 4, + name: "github.com/sourcebot/repo-4", + indexedAt: null, + }); + mocks.repoToConnectionDeleteMany.mockResolvedValue({ count: 2 }); + mocks.connectionUpdate.mockResolvedValue({}); + mocks.loadConfig.mockResolvedValue({ contexts: undefined }); + mocks.syncSearchContexts.mockResolvedValue(undefined); + const trigger = vi.fn(); + + const result = await connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger, + }); + + expect(mocks.repoFindMany).toHaveBeenNthCalledWith(1, { + where: { + connections: { + some: { + connectionId: 42, + }, + }, }, + select: { + id: true, + }, + }); + expect(mocks.repoUpsert).toHaveBeenNthCalledWith(1, { + where: { + external_id_external_codeHostUrl_orgId: { + external_id: "repo-1", + external_codeHostUrl: "https://github.com", + orgId: 7, + }, + }, + update: { + ...existingRepo, + connections: { + createMany: { + data: { + connectionId: 42, + }, + skipDuplicates: true, + }, + }, + }, + create: existingRepo, + select: { + id: true, + name: true, + indexedAt: true, + }, + }); + expect(mocks.repoToConnectionDeleteMany).toHaveBeenCalledWith({ + where: { + connectionId: 42, + repoId: { + in: [2, 3], + }, + }, + }); + expect(mocks.repoFindMany).toHaveBeenNthCalledWith(2, { + where: { + id: { + in: [2, 3], + }, + connections: { + none: {}, + }, + }, + select: { + id: true, + name: true, + }, + }); + expect(result).toEqual({ + reposToCleanup: [ + { id: 2, name: "github.com/sourcebot/repo-2" }, + ], + reposToIndex: [ + { id: 4, name: "github.com/sourcebot/repo-4" }, + ], + }); + expect(trigger).toHaveBeenNthCalledWith(1, "repo-index", { + repoId: 2, + type: "CLEANUP", + }); + expect(trigger).toHaveBeenNthCalledWith(2, "repo-index", { + repoId: 4, + type: "INDEX", + }); + expect(mocks.repoUpsert.mock.invocationCallOrder[1]).toBeLessThan( + mocks.repoToConnectionDeleteMany.mock.invocationCallOrder[0], + ); + expect(mocks.repoFindMany.mock.invocationCallOrder[1]).toBeLessThan( + trigger.mock.invocationCallOrder[0], + ); + }); + + test("does not mark the connection synced when scheduling fails", async () => { + const config = { + type: "github" as const, + }; + const newRepo = { + external_id: "repo-4", + external_codeHostUrl: "https://github.com", + displayName: "sourcebot/repo-4", + connections: { + create: { + connectionId: 42, + }, + }, + }; + + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + config, + }); + mocks.compileGithubConfig.mockResolvedValue({ + repoData: [newRepo], + warnings: [], + }); + mocks.repoFindMany.mockResolvedValueOnce([]); + mocks.repoUpsert.mockResolvedValueOnce({ + id: 4, + name: "github.com/sourcebot/repo-4", + indexedAt: null, + }); + const trigger = vi.fn().mockRejectedValue(new Error("Redis unavailable")); + + await expect(connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger, + })).rejects.toThrow("Redis unavailable"); + + expect(trigger).toHaveBeenCalledWith("repo-index", { + repoId: 4, + type: "INDEX", }); - expect(result).toBeUndefined(); + expect(mocks.connectionUpdate).not.toHaveBeenCalled(); + expect(mocks.syncSearchContexts).not.toHaveBeenCalled(); }); }); diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts index a6f87353b..b6e429b50 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionWorkload.ts @@ -11,10 +11,15 @@ interface Props { settings: Settings; } +interface ConnectionSyncResult { + reposToCleanup: { id: number; name: string }[]; + reposToIndex: { id: number; name: string }[]; +} + export const createConnectionWorkload = ({ db, settings -}: Props): Workload<'connection-sync'> => ({ +}: Props): Workload<'connection-sync', ConnectionSyncResult> => ({ queueSpec: CONNECTION_QUEUE, concurrency: settings.maxConnectionSyncJobConcurrency, process: async ({ @@ -25,6 +30,7 @@ export const createConnectionWorkload = ({ logger, signal, jobId, + trigger, }) => { logger.info(`Syncing connection ${connectionId}`, { connectionId, @@ -68,59 +74,107 @@ export const createConnectionWorkload = ({ ); }) - // @note: to handle orphaned Repos we delete all RepoToConnection records for this connection, - // and then recreate them when we upsert the repos. For example, if a repo is no-longer - // captured by the connection's config (e.g., it was deleted, marked archived, etc.), it won't - // appear in the repoData array above, and so the RepoToConnection record won't be re-created. - // Repos that have no RepoToConnection records are considered orphaned and can be deleted. - await db.$transaction(async (tx) => { - const deleteStart = performance.now(); - await tx.connection.update({ - where: { - id: connectionId, + const previouslyAssociatedRepos = await db.repo.findMany({ + where: { + connections: { + some: { + connectionId, + }, }, - data: { - repos: { - deleteMany: {} - } - } - }); - const deleteDuration = performance.now() - deleteStart; - logger.debug(`Deleted existing repository associations`, { - connectionId, - connectionName: connection.name, - durationMs: deleteDuration, - }); + }, + select: { + id: true, + }, + }); + + const upsertedRepos: { id: number; name: string; indexedAt: Date | null }[] = []; - const totalUpsertStart = performance.now(); - for (const repo of repoData) { - const upsertStart = performance.now(); - await tx.repo.upsert({ - where: { - external_id_external_codeHostUrl_orgId: { - external_id: repo.external_id, - external_codeHostUrl: repo.external_codeHostUrl, - orgId: orgId, - } + for (const repo of repoData) { + const upsertedRepo = await db.repo.upsert({ + where: { + external_id_external_codeHostUrl_orgId: { + external_id: repo.external_id, + external_codeHostUrl: repo.external_codeHostUrl, + orgId: orgId, + } + }, + update: { + ...repo, + connections: { + createMany: { + data: { + connectionId, + }, + skipDuplicates: true, + }, }, - update: repo, - create: repo, - }) - const upsertDuration = performance.now() - upsertStart; - logger.debug(`Upserted repository ${repo.displayName}`, { + }, + create: repo, + select: { + id: true, + name: true, + indexedAt: true, + }, + }) + upsertedRepos.push(upsertedRepo); + } + + const currentRepoIds = new Set(upsertedRepos.map(({ id }) => id)); + const staleRepoIds = previouslyAssociatedRepos + .map(({ id }) => id) + .filter((id) => !currentRepoIds.has(id)); + + if (staleRepoIds.length > 0) { + await db.repoToConnection.deleteMany({ + where: { connectionId, - externalId: repo.external_id, - durationMs: upsertDuration, - }); - } - const totalUpsertDuration = performance.now() - totalUpsertStart; - logger.info(`Stored ${repoData.length} repositories`, { - connectionId, - connectionName: connection.name, - repositoryCount: repoData.length, - durationMs: totalUpsertDuration, + repoId: { + in: staleRepoIds, + }, + }, }); - }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); + } + + const reposToCleanup = staleRepoIds.length > 0 + ? await db.repo.findMany({ + where: { + id: { + in: staleRepoIds, + }, + connections: { + none: {}, + }, + }, + select: { + id: true, + name: true, + }, + }) + : []; + + const reposToIndex = upsertedRepos + .filter(({ indexedAt }) => indexedAt === null) + .map(({ id, name }) => ({ id, name })); + + await Promise.all(reposToCleanup.map(({ id }) => + trigger('repo-index', { + repoId: id, + type: 'CLEANUP', + }) + )); + + await Promise.all(reposToIndex.map(({ id }) => + trigger('repo-index', { + repoId: id, + type: 'INDEX', + }) + )); + + logger.info(`Stored ${repoData.length} repositories`, { + connectionId, + connectionName: connection.name, + repositoryCount: repoData.length, + }); await db.connection.update({ where: { @@ -148,6 +202,11 @@ export const createConnectionWorkload = ({ logger.info(`Connection ${connectionId} sync finished`, { connectionId, }); + + return { + reposToCleanup, + reposToIndex, + }; }, onStarted: async ({ data: { connectionId }, jobId }) => { await db.connectionSyncJob.upsert({ diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts new file mode 100644 index 000000000..7326524a6 --- /dev/null +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts @@ -0,0 +1,423 @@ +import type { PrismaClient } from "@sourcebot/db"; +import type { JobLogger } from "@sourcebot/shared"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + captureException: vi.fn(), + createBitbucketCloudClient: vi.fn(), + createBitbucketServerClient: vi.fn(), + ensureFreshAccountToken: vi.fn(), + getIdentityProviderConfig: vi.fn(), + getReposForAuthenticatedBitbucketCloudUser: vi.fn(), + getReposForAuthenticatedBitbucketServerUser: vi.fn(), + hasEntitlement: vi.fn(), +})); + +vi.mock("@sentry/node", () => ({ + captureException: mocks.captureException, +})); + +vi.mock("@sourcebot/shared", async (importOriginal) => ({ + ...(await importOriginal()), + ACCOUNT_PERMISSION_SYNC_QUEUE: { + name: "account-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: 500, + }, + }, + getIdentityProviderConfig: mocks.getIdentityProviderConfig, +})); + +vi.mock("../entitlements.js", () => ({ + hasEntitlement: mocks.hasEntitlement, +})); + +vi.mock("../bitbucket.js", () => ({ + createBitbucketCloudClient: mocks.createBitbucketCloudClient, + createBitbucketServerClient: mocks.createBitbucketServerClient, + getReposForAuthenticatedBitbucketCloudUser: + mocks.getReposForAuthenticatedBitbucketCloudUser, + getReposForAuthenticatedBitbucketServerUser: + mocks.getReposForAuthenticatedBitbucketServerUser, +})); + +vi.mock("./tokenRefresh.js", async (importOriginal) => ({ + ...(await importOriginal()), + ensureFreshAccountToken: mocks.ensureFreshAccountToken, +})); + +import { + classifyPermissionSyncFailure, + createAccountPermissionSyncWorkload, +} from "./accountPermissionSyncWorkload.js"; +import { + PermissionSyncUpstreamError, + type PermissionSyncUpstreamErrorKind, +} from "./permissionSyncError.js"; +import { + TokenRefreshError, + type TokenRefreshErrorKind, +} from "./tokenRefresh.js"; + +const tokenRefreshError = ( + kind: TokenRefreshErrorKind, + status?: number, +): TokenRefreshError => + new TokenRefreshError(`Token refresh failed: ${kind}`, { + kind, + status, + }); + +const upstreamError = ( + kind: PermissionSyncUpstreamErrorKind, +): PermissionSyncUpstreamError => + new PermissionSyncUpstreamError(`Permission sync failed: ${kind}`, { + kind, + provider: "github", + operation: "list_accessible_repositories", + }); + +const account = { + id: "account_1", + providerId: "bitbucket-server", + issuerUrl: "https://bitbucket.example.com", + user: { email: "user@example.com" }, +}; +const accountFindUniqueOrThrow = vi.fn().mockResolvedValue(account); +const accountUpdate = vi.fn().mockResolvedValue(account); +const repoFindMany = vi.fn().mockResolvedValue([]); +const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); +const permissionDeleteMany = vi.fn().mockResolvedValue({ count: 95 }); +const permissionSyncJobUpsert = vi.fn(); +const permissionSyncJobUpdate = vi.fn().mockResolvedValue({ account }); +const transaction = vi.fn((queries: Array>) => + Promise.all(queries), +); + +const db = { + account: { + findUniqueOrThrow: accountFindUniqueOrThrow, + update: accountUpdate, + }, + accountToRepoPermission: { + createMany: permissionCreateMany, + deleteMany: permissionDeleteMany, + }, + repo: { + findMany: repoFindMany, + }, + accountPermissionSyncJob: { + upsert: permissionSyncJobUpsert, + update: permissionSyncJobUpdate, + }, + $transaction: transaction, +} as unknown as PrismaClient; + +const jobLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), +} satisfies JobLogger; + +const createWorkload = () => + createAccountPermissionSyncWorkload({ + db, + settings: { + maxAccountPermissionSyncJobConcurrency: 2, + } as never, + }); + +const lifecycleContext = { + data: { + accountId: "account_1", + }, + jobId: "job_1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger: jobLogger, +}; + +const processContext = { + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), +}; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.hasEntitlement.mockResolvedValue(true); + mocks.ensureFreshAccountToken.mockReset().mockResolvedValue("access-token"); + mocks.getIdentityProviderConfig.mockReset().mockResolvedValue({ + provider: "bitbucket-server", + baseUrl: "https://bitbucket.example.com", + }); + mocks.createBitbucketServerClient.mockReset().mockReturnValue({}); + mocks.getReposForAuthenticatedBitbucketServerUser + .mockReset() + .mockResolvedValue([]); + accountFindUniqueOrThrow.mockResolvedValue(account); + repoFindMany.mockResolvedValue([]); + permissionCreateMany.mockResolvedValue({ count: 0 }); + permissionDeleteMany.mockResolvedValue({ count: 95 }); + permissionSyncJobUpdate.mockResolvedValue({ account }); +}); + +describe("classifyPermissionSyncFailure", () => { + test("fails closed when the refresh token is rejected", () => { + expect( + classifyPermissionSyncFailure( + tokenRefreshError("refresh_token_rejected", 400), + ), + ).toEqual({ + action: "clear_permissions", + reason: "oauth_refresh_token_rejected", + }); + }); + + test.each([ + ["transient", 500], + ["configuration", 400], + ["invalid_response", undefined], + ["local_credential", undefined], + ] satisfies Array<[TokenRefreshErrorKind, number | undefined]>)( + "keeps permissions for a %s token refresh failure", + (kind, status) => { + expect( + classifyPermissionSyncFailure(tokenRefreshError(kind, status)), + ).toEqual({ + action: "preserve_permissions", + }); + }, + ); + + test("does not treat a token refresh configuration error with HTTP 401 as an API authorization failure", () => { + expect( + classifyPermissionSyncFailure( + tokenRefreshError("configuration", 401), + ), + ).toEqual({ + action: "preserve_permissions", + }); + }); + + test.each([ + ["credential_rejected", "upstream_credential_rejected"], + ["insufficient_scope", "upstream_insufficient_scope"], + ] as const)( + "fails closed for a classified %s upstream failure", + (kind, reason) => { + expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ + action: "clear_permissions", + reason, + }); + }, + ); + + test.each([ + "rate_limited", + "upstream_unavailable", + "forbidden", + "unknown", + ] satisfies PermissionSyncUpstreamErrorKind[])( + "keeps permissions for a classified %s upstream failure", + (kind) => { + expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ + action: "preserve_permissions", + }); + }, + ); + + test.each([401, 403, 410])( + "does not fail closed on an unclassified HTTP %s error", + (status) => { + const error = Object.assign(new Error(`HTTP ${status}`), { + status, + }); + expect(classifyPermissionSyncFailure(error)).toEqual({ + action: "preserve_permissions", + }); + }, + ); +}); + +describe("accountPermissionSyncWorkload", () => { + test("uses the configured concurrency and database-backed lifecycle hooks", () => { + const workload = createWorkload(); + + expect(workload.queueSpec.name).toBe("account-permission-sync"); + expect(workload.concurrency).toBe(2); + expect(workload.onStarted).toBeTypeOf("function"); + expect(workload.onCompleted).toBeTypeOf("function"); + expect(workload.onTerminalFailure).toBeTypeOf("function"); + }); + + test("syncs the requested account", async () => { + const workload = createWorkload(); + + await workload.process(processContext); + + expect(accountFindUniqueOrThrow).toHaveBeenCalledWith({ + where: { id: "account_1" }, + include: { user: true }, + }); + expect(mocks.ensureFreshAccountToken).toHaveBeenCalledWith(account, db); + expect(mocks.createBitbucketServerClient).toHaveBeenCalledWith( + "https://bitbucket.example.com", + undefined, + "access-token", + ); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("does not run without the permission syncing entitlement", async () => { + mocks.hasEntitlement.mockResolvedValue(false); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toThrow( + "Permission syncing entitlement is not currently available.", + ); + + expect(accountFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + + test("atomically records a reauthentication issue when the refresh token is rejected", async () => { + const error = tokenRefreshError("refresh_token_rejected", 400); + mocks.ensureFreshAccountToken.mockRejectedValue(error); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(permissionDeleteMany).toHaveBeenCalledWith({ + where: { accountId: "account_1" }, + }); + expect(accountUpdate).toHaveBeenCalledWith({ + where: { id: "account_1" }, + data: { + permissionSyncIssue: "REAUTHENTICATION_REQUIRED", + permissionSyncIssueAt: expect.any(Date), + }, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("records an issue even when permissions were cleared by an earlier attempt", async () => { + const error = tokenRefreshError("refresh_token_rejected", 400); + permissionDeleteMany.mockResolvedValue({ count: 0 }); + mocks.ensureFreshAccountToken.mockRejectedValue(error); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(accountUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + permissionSyncIssue: "REAUTHENTICATION_REQUIRED", + }), + }), + ); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("records an insufficient-scope issue for scope failures", async () => { + const error = upstreamError("insufficient_scope"); + mocks.getReposForAuthenticatedBitbucketServerUser.mockRejectedValue( + error, + ); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(accountUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + permissionSyncIssue: "INSUFFICIENT_SCOPE", + }), + }), + ); + }); + + test("preserves permissions for a transient refresh failure", async () => { + const error = tokenRefreshError("transient", 500); + mocks.ensureFreshAccountToken.mockRejectedValue(error); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(permissionDeleteMany).not.toHaveBeenCalled(); + expect(accountUpdate).not.toHaveBeenCalled(); + expect(transaction).not.toHaveBeenCalled(); + }); + + test("marks a job as in progress when started", async () => { + await createWorkload().onStarted?.(lifecycleContext); + + expect(permissionSyncJobUpsert).toHaveBeenCalledWith({ + where: { id: "job_1" }, + update: { + status: "IN_PROGRESS", + completedAt: null, + errorMessage: null, + }, + create: { + id: "job_1", + accountId: "account_1", + status: "IN_PROGRESS", + }, + }); + }); + + test("marks a job completed and clears the account issue", async () => { + await createWorkload().onCompleted?.(lifecycleContext, undefined); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith({ + where: { id: "job_1" }, + data: { + status: "COMPLETED", + completedAt: expect.any(Date), + errorMessage: null, + account: { + update: { + permissionSyncedAt: expect.any(Date), + permissionSyncIssue: null, + permissionSyncIssueAt: null, + }, + }, + }, + select: { + account: { + include: { user: true }, + }, + }, + }); + }); + + test("marks a job failed after terminal failure", async () => { + const error = new Error("Upstream unavailable"); + + await createWorkload().onTerminalFailure?.(lifecycleContext, error); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: "job_1" }, + data: { + status: "FAILED", + completedAt: expect.any(Date), + errorMessage: "Upstream unavailable", + }, + }), + ); + expect(mocks.captureException).toHaveBeenCalledWith(error, { + tags: { + jobId: "job_1", + queue: "account-permission-sync", + }, + }); + }); +}); diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.ts new file mode 100644 index 000000000..af5a3b721 --- /dev/null +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.ts @@ -0,0 +1,525 @@ +import * as Sentry from "@sentry/node"; +import { + Account, + AccountPermissionSyncIssue, + AccountPermissionSyncJobStatus, + PermissionSyncSource, + PrismaClient, +} from "@sourcebot/db"; +import { + ACCOUNT_PERMISSION_SYNC_QUEUE, + getIdentityProviderConfig, + JobLogSink, +} from "@sourcebot/shared"; +import { + createBitbucketCloudClient, + createBitbucketServerClient, + getReposForAuthenticatedBitbucketCloudUser, + getReposForAuthenticatedBitbucketServerUser, +} from "../bitbucket.js"; +import { hasEntitlement } from "../entitlements.js"; +import { + createOctokitFromToken, + getOAuthScopesForAuthenticatedUser as getGitHubOAuthScopesForAuthenticatedUser, + getReposForAuthenticatedUser, +} from "../github.js"; +import { + createGitLabFromOAuthToken, + getOAuthScopesForAuthenticatedUser as getGitLabOAuthScopesForAuthenticatedUser, + getProjectsForAuthenticatedUser, +} from "../gitlab.js"; +import { + PermissionSyncUpstreamError, + withPermissionSyncUpstreamError, +} from "./permissionSyncError.js"; +import { ensureFreshAccountToken, TokenRefreshError } from "./tokenRefresh.js"; +import { Settings, Workload } from "../types.js"; +import { IdentityProviderConfig } from "@sourcebot/schemas/v3/index.type"; + +type AccountWithUser = Account & { user: { email: string | null } }; + +type SupportedProvider = + | "github" + | "gitlab" + | "bitbucket-cloud" + | "bitbucket-server"; +type ProviderConfig = Extract< + IdentityProviderConfig, + { provider: TProvider } +>; + +interface ProviderPermissionSyncProps { + db: PrismaClient; + account: AccountWithUser; + accessToken: string; + config: ProviderConfig; +} + +export type PermissionCleanupReason = + | "oauth_refresh_token_rejected" + | "upstream_credential_rejected" + | "upstream_insufficient_scope"; + +export type PermissionCleanupDecision = + | { + action: "clear_permissions"; + reason: PermissionCleanupReason; + } + | { + action: "preserve_permissions"; + }; + +export const classifyPermissionSyncFailure = ( + error: unknown, +): PermissionCleanupDecision => { + // Token refresh failures have their own classification. Do not fall through + // to the generic HTTP checks because another token endpoint failure may + // also carry a 401 or 403 status. + if (error instanceof TokenRefreshError) { + return error.kind === "refresh_token_rejected" + ? { + action: "clear_permissions", + reason: "oauth_refresh_token_rejected", + } + : { action: "preserve_permissions" }; + } + + if (error instanceof PermissionSyncUpstreamError) { + if (error.kind === "credential_rejected") { + return { + action: "clear_permissions", + reason: "upstream_credential_rejected", + }; + } + if (error.kind === "insufficient_scope") { + return { + action: "clear_permissions", + reason: "upstream_insufficient_scope", + }; + } + } + + return { action: "preserve_permissions" }; +}; + +const PERMISSION_CLEANUP_DETAILS: Record< + PermissionCleanupReason, + { + message: string; + issue: AccountPermissionSyncIssue; + } +> = { + oauth_refresh_token_rejected: { + message: "OAuth refresh token rejection", + issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, + }, + upstream_credential_rejected: { + message: "upstream credential rejection", + issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, + }, + upstream_insufficient_scope: { + message: "insufficient OAuth scope", + issue: AccountPermissionSyncIssue.INSUFFICIENT_SCOPE, + }, +}; + +interface AccountPermissionSyncWorkloadDependencies { + db: PrismaClient; + settings: Settings; +} + +export const createAccountPermissionSyncWorkload = ({ + db, + settings, +}: AccountPermissionSyncWorkloadDependencies): Workload<"account-permission-sync"> => { + return { + queueSpec: ACCOUNT_PERMISSION_SYNC_QUEUE, + concurrency: settings.maxAccountPermissionSyncJobConcurrency, + process: async ({ data: { accountId }, logger: jobLogger }) => { + if (!(await hasEntitlement("permission-syncing"))) { + throw new Error( + "Permission syncing entitlement is not currently available.", + ); + } + + const account = await db.account.findUniqueOrThrow({ + where: { + id: accountId, + }, + include: { + user: true, + }, + }); + + jobLogger.debug( + `Syncing permissions for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}...`, + ); + + try { + // Ensure the OAuth token is fresh, refreshing it if it is expired or near expiry. + const accessToken = await ensureFreshAccountToken(account, db); + + const idpConfig = await getIdentityProviderConfig( + account.providerId, + ); + if (!idpConfig) { + throw new Error( + "Unable to find IDP config in config.json.", + ); + } + + const repoIds = await getAccessibleRepoIds({ + db, + account, + accessToken, + config: idpConfig, + }); + + await db.$transaction([ + db.account.update({ + where: { + id: account.id, + }, + data: { + accessibleRepos: { + deleteMany: {}, + }, + }, + }), + db.accountToRepoPermission.createMany({ + data: repoIds.map((repoId) => ({ + accountId: account.id, + repoId, + source: PermissionSyncSource.ACCOUNT_DRIVEN, + })), + skipDuplicates: true, + }), + ]); + } catch (error) { + // Clear cached permissions only for classified permanent failures. + // Ambiguous HTTP errors and transient upstream failures preserve the + // last successful permission state. + const cleanupDecision = classifyPermissionSyncFailure(error); + + if (cleanupDecision.action === "clear_permissions") { + const details = + PERMISSION_CLEANUP_DETAILS[cleanupDecision.reason]; + const [{ count }] = await db.$transaction([ + db.accountToRepoPermission.deleteMany({ + where: { accountId: account.id }, + }), + db.account.update({ + where: { id: account.id }, + data: { + permissionSyncIssue: details.issue, + permissionSyncIssueAt: new Date(), + }, + }), + ]); + const message = + error instanceof Error ? error.message : String(error); + jobLogger.warn( + `Cleared ${count} permission row(s) for account ${account.id} (user ${account.user.email ?? "unknown"}) — fail-closed cleanup triggered by ${details.message}: ${message}`, + ); + } + throw error; + } + }, + onStarted: async ({ data: { accountId }, jobId }) => { + await db.accountPermissionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: AccountPermissionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + accountId, + status: AccountPermissionSyncJobStatus.IN_PROGRESS, + }, + }); + }, + onCompleted: async ({ jobId, logger: jobLogger }) => { + const { account } = await db.accountPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: AccountPermissionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + account: { + update: { + permissionSyncedAt: new Date(), + permissionSyncIssue: null, + permissionSyncIssueAt: null, + }, + }, + }, + select: { + account: { + include: { + user: true, + }, + }, + }, + }); + + jobLogger.debug( + `Permissions synced for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}`, + ); + }, + onTerminalFailure: async ( + { data: { accountId }, jobId, logger: jobLogger }, + error, + ) => { + Sentry.captureException(error, { + tags: { + jobId, + queue: ACCOUNT_PERMISSION_SYNC_QUEUE.name, + }, + }); + + const { account } = await db.accountPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: AccountPermissionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + select: { + account: { + include: { + user: true, + }, + }, + }, + }); + + jobLogger.error( + `Account permission sync job failed for account (id: ${accountId}) for user ${account.user.email ?? "unknown user (email not found)"}: ${error.message}`, + ); + }, + }; +}; + +const getAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: { + db: PrismaClient; + account: AccountWithUser; + accessToken: string; + config: IdentityProviderConfig; +}): Promise => { + switch (config.provider) { + case "github": + return getGitHubAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + case "gitlab": + return getGitLabAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + case "bitbucket-cloud": + return getBitbucketCloudAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + case "bitbucket-server": + return getBitbucketServerAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + default: + throw new Error(`Unsupported provider type: ${config.provider}`); + } +}; + +const getGitHubAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: ProviderPermissionSyncProps<"github">): Promise => { + const { octokit } = await createOctokitFromToken({ + token: accessToken, + url: config.baseUrl, + }); + + const scopes = await withPermissionSyncUpstreamError( + "github", + "inspect_token_scopes", + () => getGitHubOAuthScopesForAuthenticatedUser(octokit, accessToken), + ); + + // Token supports scope introspection (classic PAT or OAuth app token). + if (scopes !== null && !scopes.includes("repo")) { + throw new PermissionSyncUpstreamError( + `OAuth token with scopes [${scopes.join(", ")}] is missing the 'repo' scope required for permission syncing. Please re-authorize with GitHub to grant the required scope.`, + { + kind: "insufficient_scope", + provider: "github", + operation: "inspect_token_scopes", + }, + ); + } + + // Public repos do not need an explicit permission mapping. + const githubRepos = await withPermissionSyncUpstreamError( + "github", + "list_accessible_repositories", + () => getReposForAuthenticatedUser("private", octokit), + ); + const gitHubRepoIds = githubRepos.map((repo) => repo.id.toString()); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "github", + external_id: { + in: gitHubRepoIds, + }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; + +const getGitLabAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: ProviderPermissionSyncProps<"gitlab">): Promise => { + const api = await createGitLabFromOAuthToken({ + oauthToken: accessToken, + url: config.baseUrl, + }); + + const scopes = await withPermissionSyncUpstreamError( + "gitlab", + "inspect_token_scopes", + () => getGitLabOAuthScopesForAuthenticatedUser(api), + ); + if (!scopes.includes("read_api")) { + throw new PermissionSyncUpstreamError( + `OAuth token with scopes [${scopes.join(", ")}] is missing the 'read_api' scope required for permission syncing.`, + { + kind: "insufficient_scope", + provider: "gitlab", + operation: "inspect_token_scopes", + }, + ); + } + + // Public and internal repos do not need an explicit permission mapping. + const gitLabProjectIds = ( + await withPermissionSyncUpstreamError( + "gitlab", + "list_accessible_repositories", + () => getProjectsForAuthenticatedUser("private", api), + ) + ).map((project) => project.id.toString()); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "gitlab", + external_id: { + in: gitLabProjectIds, + }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; + +const getBitbucketCloudAccessibleRepoIds = async ({ + db, + account, + accessToken, +}: ProviderPermissionSyncProps<"bitbucket-cloud">): Promise => { + // Use a bearer token by omitting the user. + const client = createBitbucketCloudClient(undefined, accessToken); + const bitbucketRepos = await withPermissionSyncUpstreamError( + "bitbucket-cloud", + "list_accessible_repositories", + () => getReposForAuthenticatedBitbucketCloudUser(client), + ); + const bitbucketRepoUuids = bitbucketRepos.map((repo) => repo.uuid); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "bitbucketCloud", + external_id: { + in: bitbucketRepoUuids, + }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; + +const getBitbucketServerAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: ProviderPermissionSyncProps<"bitbucket-server">): Promise => { + const client = createBitbucketServerClient( + config.baseUrl, + undefined, + accessToken, + ); + const serverRepos = await withPermissionSyncUpstreamError( + "bitbucket-server", + "list_accessible_repositories", + () => getReposForAuthenticatedBitbucketServerUser(client), + ); + const serverRepoIds = serverRepos.map((repo) => repo.id); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "bitbucketServer", + external_id: { in: serverRepoIds }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; diff --git a/packages/backend/src/ee/accountPermissionSyncer.test.ts b/packages/backend/src/ee/accountPermissionSyncer.test.ts deleted file mode 100644 index 09b35d594..000000000 --- a/packages/backend/src/ee/accountPermissionSyncer.test.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { beforeEach, describe, expect, test, vi } from 'vitest'; - -const mocks = vi.hoisted(() => ({ - hasEntitlement: vi.fn(), -})); - -vi.mock('../entitlements.js', () => ({ - hasEntitlement: mocks.hasEntitlement, -})); - -import { AccountPermissionSyncer, classifyPermissionSyncFailure } from './accountPermissionSyncer.js'; -import { - PermissionSyncUpstreamError, - type PermissionSyncUpstreamErrorKind, -} from './permissionSyncError.js'; -import { TokenRefreshError, type TokenRefreshErrorKind } from './tokenRefresh.js'; - -const tokenRefreshError = ( - kind: TokenRefreshErrorKind, - status?: number, -): TokenRefreshError => new TokenRefreshError(`Token refresh failed: ${kind}`, { - kind, - status, -}); - -const upstreamError = ( - kind: PermissionSyncUpstreamErrorKind, -): PermissionSyncUpstreamError => new PermissionSyncUpstreamError(`Permission sync failed: ${kind}`, { - kind, - provider: 'github', - operation: 'list_accessible_repositories', -}); - -const createSyncerHarness = (syncError?: Error, permissionCount = 95) => { - const account = { - id: 'account_1', - providerId: 'bitbucket-server', - user: { email: 'user@example.com' }, - }; - const db = { - accountPermissionSyncJob: { - update: vi.fn().mockResolvedValue({ account }), - }, - accountToRepoPermission: { - deleteMany: vi.fn().mockResolvedValue({ count: permissionCount }), - }, - account: { - update: vi.fn().mockResolvedValue(account), - }, - $transaction: vi.fn((queries: Array>) => Promise.all(queries)), - }; - const syncAccountPermissions = syncError - ? vi.fn().mockRejectedValue(syncError) - : vi.fn().mockResolvedValue(undefined); - const syncer = Object.create(AccountPermissionSyncer.prototype) as { - db: typeof db; - syncAccountPermissions: typeof syncAccountPermissions; - runJob(job: { data: { jobId: string } }): Promise; - onJobCompleted(job: { data: { jobId: string } }): Promise; - }; - syncer.db = db; - syncer.syncAccountPermissions = syncAccountPermissions; - - return { - account, - db, - job: { data: { jobId: 'job_1' } }, - syncer, - }; -}; - -beforeEach(() => { - vi.clearAllMocks(); - mocks.hasEntitlement.mockResolvedValue(true); -}); - -describe('classifyPermissionSyncFailure', () => { - test('fails closed when the refresh token is rejected', () => { - expect(classifyPermissionSyncFailure(tokenRefreshError('refresh_token_rejected', 400))).toEqual({ - action: 'clear_permissions', - reason: 'oauth_refresh_token_rejected', - }); - }); - - test.each([ - ['transient', 500], - ['configuration', 400], - ['invalid_response', undefined], - ['local_credential', undefined], - ] satisfies Array<[TokenRefreshErrorKind, number | undefined]>)('keeps permissions for a %s token refresh failure', (kind, status) => { - expect(classifyPermissionSyncFailure(tokenRefreshError(kind, status))).toEqual({ - action: 'preserve_permissions', - }); - }); - - test('does not treat a token refresh configuration error with HTTP 401 as an API authorization failure', () => { - expect(classifyPermissionSyncFailure(tokenRefreshError('configuration', 401))).toEqual({ - action: 'preserve_permissions', - }); - }); - - test.each([ - ['credential_rejected', 'upstream_credential_rejected'], - ['insufficient_scope', 'upstream_insufficient_scope'], - ] as const)('fails closed for a classified %s upstream failure', (kind, reason) => { - expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ - action: 'clear_permissions', - reason, - }); - }); - - test.each([ - 'rate_limited', - 'upstream_unavailable', - 'forbidden', - 'unknown', - ] satisfies PermissionSyncUpstreamErrorKind[])('keeps permissions for a classified %s upstream failure', (kind) => { - expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ - action: 'preserve_permissions', - }); - }); - - test.each([401, 403, 410])('does not fail closed on an unclassified HTTP %s error', (status) => { - const error = Object.assign(new Error(`HTTP ${status}`), { status }); - expect(classifyPermissionSyncFailure(error)).toEqual({ - action: 'preserve_permissions', - }); - }); -}); - -describe('permission sync issue lifecycle', () => { - test('atomically records a reauthentication issue when the refresh token is rejected', async () => { - const error = tokenRefreshError('refresh_token_rejected', 400); - const { db, job, syncer } = createSyncerHarness(error); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.accountToRepoPermission.deleteMany).toHaveBeenCalledWith({ - where: { accountId: 'account_1' }, - }); - expect(db.account.update).toHaveBeenCalledWith({ - where: { id: 'account_1' }, - data: { - permissionSyncIssue: 'REAUTHENTICATION_REQUIRED', - permissionSyncIssueAt: expect.any(Date), - }, - }); - expect(db.$transaction).toHaveBeenCalledOnce(); - }); - - test('records an issue when permissions were already cleared by an earlier attempt', async () => { - const error = tokenRefreshError('refresh_token_rejected', 400); - const { db, job, syncer } = createSyncerHarness(error, 0); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.account.update).toHaveBeenCalledWith(expect.objectContaining({ - data: expect.objectContaining({ - permissionSyncIssue: 'REAUTHENTICATION_REQUIRED', - }), - })); - expect(db.$transaction).toHaveBeenCalledOnce(); - }); - - test('records an insufficient-scope issue for scope failures', async () => { - const error = upstreamError('insufficient_scope'); - const { db, job, syncer } = createSyncerHarness(error); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.account.update).toHaveBeenCalledWith(expect.objectContaining({ - data: expect.objectContaining({ - permissionSyncIssue: 'INSUFFICIENT_SCOPE', - }), - })); - }); - - test('does not record an issue or clear permissions for a transient refresh failure', async () => { - const error = tokenRefreshError('transient', 500); - const { db, job, syncer } = createSyncerHarness(error); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.accountToRepoPermission.deleteMany).not.toHaveBeenCalled(); - expect(db.account.update).not.toHaveBeenCalled(); - expect(db.$transaction).not.toHaveBeenCalled(); - }); - - test('clears the action-required issue after a successful permission sync', async () => { - const { db, job, syncer } = createSyncerHarness(); - - await syncer.onJobCompleted(job); - - expect(db.accountPermissionSyncJob.update).toHaveBeenCalledWith(expect.objectContaining({ - data: expect.objectContaining({ - account: { - update: expect.objectContaining({ - permissionSyncIssue: null, - permissionSyncIssueAt: null, - }), - }, - }), - })); - }); -}); diff --git a/packages/backend/src/ee/accountPermissionSyncer.ts b/packages/backend/src/ee/accountPermissionSyncer.ts deleted file mode 100644 index 7184a191e..000000000 --- a/packages/backend/src/ee/accountPermissionSyncer.ts +++ /dev/null @@ -1,535 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { - PrismaClient, - AccountPermissionSyncIssue, - AccountPermissionSyncJobStatus, - Account, - PermissionSyncSource, -} from "@sourcebot/db"; -import { env, createLogger, getIdentityProviderConfig, PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS } from "@sourcebot/shared"; -import { hasEntitlement } from "../entitlements.js"; -import { ensureFreshAccountToken, TokenRefreshError } from "./tokenRefresh.js"; -import { DelayedError, Job, Queue, Worker } from "bullmq"; -import { Redis } from "ioredis"; -import { - createOctokitFromToken, - getOAuthScopesForAuthenticatedUser as getGitHubOAuthScopesForAuthenticatedUser, - getReposForAuthenticatedUser, -} from "../github.js"; -import { - createGitLabFromOAuthToken, - getOAuthScopesForAuthenticatedUser as getGitLabOAuthScopesForAuthenticatedUser, - getProjectsForAuthenticatedUser, -} from "../gitlab.js"; -import { createBitbucketCloudClient, createBitbucketServerClient, getReposForAuthenticatedBitbucketCloudUser, getReposForAuthenticatedBitbucketServerUser } from "../bitbucket.js"; -import { Settings } from "../types.js"; -import { setIntervalAsync } from "../utils.js"; -import { PermissionSyncUpstreamError, withPermissionSyncUpstreamError } from "./permissionSyncError.js"; - -const LOG_TAG = 'user-permission-syncer'; -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); - -const QUEUE_NAME = 'accountPermissionSyncQueue'; -const POLLING_INTERVAL_MS = 1000; -const ENTITLEMENT_RETRY_DELAY_MS = 30 * 1000; - -type AccountPermissionSyncJob = { - jobId: string; -} - -export type PermissionCleanupReason = - | 'oauth_refresh_token_rejected' - | 'upstream_credential_rejected' - | 'upstream_insufficient_scope'; - -export type PermissionCleanupDecision = - | { - action: 'clear_permissions'; - reason: PermissionCleanupReason; - } - | { - action: 'preserve_permissions'; - }; - -const PERMISSION_CLEANUP_DETAILS: Record = { - oauth_refresh_token_rejected: { - message: 'OAuth refresh token rejection', - issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, - }, - upstream_credential_rejected: { - message: 'upstream credential rejection', - issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, - }, - upstream_insufficient_scope: { - message: 'insufficient OAuth scope', - issue: AccountPermissionSyncIssue.INSUFFICIENT_SCOPE, - }, -}; - -export const classifyPermissionSyncFailure = (error: unknown): PermissionCleanupDecision => { - // Token refresh failures have their own classification. Do not fall through - // to the generic HTTP checks because another token endpoint failure may - // also carry a 401 or 403 status. - if (error instanceof TokenRefreshError) { - return error.kind === 'refresh_token_rejected' - ? { action: 'clear_permissions', reason: 'oauth_refresh_token_rejected' } - : { action: 'preserve_permissions' }; - } - - if (error instanceof PermissionSyncUpstreamError) { - if (error.kind === 'credential_rejected') { - return { action: 'clear_permissions', reason: 'upstream_credential_rejected' }; - } - if (error.kind === 'insufficient_scope') { - return { action: 'clear_permissions', reason: 'upstream_insufficient_scope' }; - } - } - - return { action: 'preserve_permissions' }; -}; - -export class AccountPermissionSyncer { - private queue: Queue; - private worker: Worker; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - ) { - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - }); - this.worker = new Worker(QUEUE_NAME, this.runJob.bind(this), { - connection: redis, - concurrency: this.settings.maxAccountPermissionSyncJobConcurrency, - }); - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobFailed.bind(this)); - } - - public async startScheduler() { - logger.debug('Starting scheduler'); - - this.interval = setIntervalAsync(async () => { - if (!await hasEntitlement('permission-syncing')) { - return; - } - - const thresholdDate = new Date(Date.now() - this.settings.userDrivenPermissionSyncIntervalMs); - - const accounts = await this.db.account.findMany({ - where: { - AND: [ - { - providerType: { - in: PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS - } - }, - { - OR: [ - { permissionSyncedAt: null }, - { permissionSyncedAt: { lt: thresholdDate } }, - ] - }, - { - NOT: { - permissionSyncJobs: { - some: { - OR: [ - // Don't schedule if there are active jobs - { - status: { - in: [ - AccountPermissionSyncJobStatus.PENDING, - AccountPermissionSyncJobStatus.IN_PROGRESS, - ], - } - }, - // Don't schedule if there are recent failed jobs (within the threshold date). Note `gt` is used here since this is a inverse condition. - { - AND: [ - { status: AccountPermissionSyncJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - } - } - }, - ] - } - }); - - await this.schedulePermissionSync(accounts); - }, POLLING_INTERVAL_MS); - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - await this.worker.close(/* force = */ true); - await this.queue.close(); - } - - public async schedulePermissionSyncForAccount(account: Account) { - const [job] = await this.db.accountPermissionSyncJob.createManyAndReturn({ - data: [{ accountId: account.id }], - }); - - await this.queue.add('accountPermissionSyncJob', { - jobId: job.id, - }, { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - priority: 1, - }); - - return job.id; - } - - private async schedulePermissionSync(accounts: Account[]) { - // @note: we don't perform this in a transaction because - // we want to avoid the situation where a job is created and run - // prior to the transaction being committed. - const jobs = await this.db.accountPermissionSyncJob.createManyAndReturn({ - data: accounts.map(account => ({ - accountId: account.id, - })), - include: { - account: true, - } - }); - - await this.queue.addBulk(jobs.map((job) => ({ - name: 'accountPermissionSyncJob', - data: { - jobId: job.id, - }, - opts: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - // Priority 1 (high) for never-synced, Priority 2 (normal) for re-sync - priority: job.account.permissionSyncedAt === null ? 1 : 2, - } - }))) - } - - private async runJob(job: Job) { - if (!await hasEntitlement('permission-syncing')) { - await job.moveToDelayed(Date.now() + ENTITLEMENT_RETRY_DELAY_MS, job.token); - throw new DelayedError('Permission syncing entitlement is not currently available.'); - } - - const id = job.data.jobId; - const logger = createJobLogger(id); - - const { account } = await this.db.accountPermissionSyncJob.update({ - where: { - id, - }, - data: { - status: AccountPermissionSyncJobStatus.IN_PROGRESS, - }, - select: { - account: { - include: { - user: true, - } - } - } - }); - - try { - await this.syncAccountPermissions(account, logger); - } catch (error) { - // Clear cached permissions only for classified permanent failures. - // Ambiguous HTTP errors and transient upstream failures preserve the - // last successful permission state. - const cleanupDecision = classifyPermissionSyncFailure(error); - - if (cleanupDecision.action === 'clear_permissions') { - const details = PERMISSION_CLEANUP_DETAILS[cleanupDecision.reason]; - const [{ count }] = await this.db.$transaction([ - this.db.accountToRepoPermission.deleteMany({ - where: { accountId: account.id }, - }), - this.db.account.update({ - where: { id: account.id }, - data: { - permissionSyncIssue: details.issue, - permissionSyncIssueAt: new Date(), - }, - }), - ]); - const message = error instanceof Error ? error.message : String(error); - logger.warn(`Cleared ${count} permission row(s) for account ${account.id} (user ${account.user.email ?? 'unknown'}) — fail-closed cleanup triggered by ${details.message}: ${message}`); - } - throw error; - } - } - - private async syncAccountPermissions( - account: Account & { user: { email: string | null } }, - logger: ReturnType, - ) { - logger.debug(`Syncing permissions for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}...`); - - // Ensure the OAuth token is fresh, refreshing it if it is expired or near expiry. - const accessToken = await ensureFreshAccountToken(account, this.db); - - // Get a list of all repos that the user has access to from all connected accounts. - const repoIds = await (async () => { - const aggregatedRepoIds: Set = new Set(); - - const idpConfig = await getIdentityProviderConfig(account.providerId); - - if (!idpConfig) { - throw new Error(`Unable to find IDP config in config.json.`); - } - - if (idpConfig.provider === 'github') { - const { octokit } = await createOctokitFromToken({ - token: accessToken, - url: idpConfig.baseUrl, - }); - - const scopes = await withPermissionSyncUpstreamError( - 'github', - 'inspect_token_scopes', - () => getGitHubOAuthScopesForAuthenticatedUser(octokit, accessToken), - ); - - // Token supports scope introspection (classic PAT or OAuth app token) - if (scopes !== null) { - if (!scopes.includes('repo')) { - throw new PermissionSyncUpstreamError( - `OAuth token with scopes [${scopes.join(', ')}] is missing the 'repo' scope required for permission syncing. Please re-authorize with GitHub to grant the required scope.`, - { - kind: 'insufficient_scope', - provider: 'github', - operation: 'inspect_token_scopes', - }, - ); - } - } - - // @note: we only care about the private repos since we don't need to build a mapping - // for public repos. - // @see: packages/web/src/prisma.ts - const githubRepos = await withPermissionSyncUpstreamError( - 'github', - 'list_accessible_repositories', - () => getReposForAuthenticatedUser(/* visibility = */ 'private', octokit), - ); - const gitHubRepoIds = githubRepos.map(repo => repo.id.toString()); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'github', - external_id: { - in: gitHubRepoIds, - }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else if (idpConfig.provider === 'gitlab') { - const api = await createGitLabFromOAuthToken({ - oauthToken: accessToken, - url: idpConfig.baseUrl, - }); - - const scopes = await withPermissionSyncUpstreamError( - 'gitlab', - 'inspect_token_scopes', - () => getGitLabOAuthScopesForAuthenticatedUser(api), - ); - if (!scopes.includes('read_api')) { - throw new PermissionSyncUpstreamError( - `OAuth token with scopes [${scopes.join(', ')}] is missing the 'read_api' scope required for permission syncing.`, - { - kind: 'insufficient_scope', - provider: 'gitlab', - operation: 'inspect_token_scopes', - }, - ); - } - - // @note: we only care about the private repos since we don't need to build a - // mapping for public or internal repos. Note that internal repos are _not_ - // enforced by permission syncing and therefore we don't need to fetch them - // here. - // - // @see: packages/web/src/prisma.ts - const gitLabProjectIds = ( - await withPermissionSyncUpstreamError( - 'gitlab', - 'list_accessible_repositories', - () => getProjectsForAuthenticatedUser('private', api), - ) - ).map(project => project.id.toString()); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'gitlab', - external_id: { - in: gitLabProjectIds, - }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else if (idpConfig.provider === 'bitbucket-cloud') { - // @note: we don't pass a user here since we want to use a bearer token - // for authentication. - const client = createBitbucketCloudClient(/* user = */ undefined, accessToken) - const bitbucketRepos = await withPermissionSyncUpstreamError( - 'bitbucket-cloud', - 'list_accessible_repositories', - () => getReposForAuthenticatedBitbucketCloudUser(client), - ); - const bitbucketRepoUuids = bitbucketRepos.map(repo => repo.uuid); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'bitbucketCloud', - external_id: { - in: bitbucketRepoUuids, - }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else if (idpConfig.provider === 'bitbucket-server') { - const client = createBitbucketServerClient(idpConfig.baseUrl, /* user = */ undefined, accessToken); - const serverRepos = await withPermissionSyncUpstreamError( - 'bitbucket-server', - 'list_accessible_repositories', - () => getReposForAuthenticatedBitbucketServerUser(client), - ); - const serverRepoIds = serverRepos.map(r => r.id); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'bitbucketServer', - external_id: { in: serverRepoIds }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else { - throw new Error(`Unsupported provider type: ${idpConfig.provider}`); - } - - return Array.from(aggregatedRepoIds); - })(); - - await this.db.$transaction([ - this.db.account.update({ - where: { - id: account.id, - }, - data: { - accessibleRepos: { - deleteMany: {}, - } - } - }), - this.db.accountToRepoPermission.createMany({ - data: repoIds.map(repoId => ({ - accountId: account.id, - repoId, - source: PermissionSyncSource.ACCOUNT_DRIVEN, - })), - skipDuplicates: true, - }) - ]); - } - - private async onJobCompleted(job: Job) { - const logger = createJobLogger(job.data.jobId); - - const { account } = await this.db.accountPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: AccountPermissionSyncJobStatus.COMPLETED, - account: { - update: { - permissionSyncedAt: new Date(), - permissionSyncIssue: null, - permissionSyncIssueAt: null, - }, - }, - completedAt: new Date(), - }, - select: { - account: { - include: { - user: true, - } - } - } - }); - - logger.debug(`Permissions synced for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}`); - } - - private async onJobFailed(job: Job | undefined, err: Error) { - const logger = createJobLogger(job?.data.jobId ?? 'unknown'); - - Sentry.captureException(err, { - tags: { - jobId: job?.data.jobId, - queue: QUEUE_NAME, - } - }); - - const errorMessage = (accountId: string, email: string) => `Account permission sync job failed for account (id: ${accountId}) for user ${email}: ${err.message}`; - - if (job) { - const { account } = await this.db.accountPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: AccountPermissionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: err.message, - }, - select: { - account: { - include: { - user: true, - } - } - } - }); - - logger.error(errorMessage(account.id, account.user.email ?? 'unknown user (email not found)')); - } else { - logger.error(errorMessage('unknown account (id not found)', 'unknown user (id not found)')); - } - } -} diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts new file mode 100644 index 000000000..648c5b88d --- /dev/null +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts @@ -0,0 +1,361 @@ +import type { PrismaClient } from "@sourcebot/db"; +import type { JobLogger } from "@sourcebot/shared"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + captureException: vi.fn(), + createBitbucketCloudClient: vi.fn(), + createBitbucketServerClient: vi.fn(), + createGitLabFromPersonalAccessToken: vi.fn(), + createOctokitFromToken: vi.fn(), + getAuthCredentialsForRepo: vi.fn(), + getExplicitUserPermissionsForCloudRepo: vi.fn(), + getProjectMembers: vi.fn(), + getRepoCollaborators: vi.fn(), + getUserPermissionsForServerRepo: vi.fn(), + hasEntitlement: vi.fn(), +})); + +vi.mock("@sentry/node", () => ({ + captureException: mocks.captureException, +})); + +vi.mock("@sourcebot/shared", async (importOriginal) => ({ + ...(await importOriginal()), + REPO_PERMISSION_SYNC_QUEUE: { + name: "repo-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: 500, + }, + }, +})); + +vi.mock("../entitlements.js", () => ({ + hasEntitlement: mocks.hasEntitlement, +})); + +vi.mock("../utils.js", () => ({ + getAuthCredentialsForRepo: mocks.getAuthCredentialsForRepo, +})); + +vi.mock("../github.js", () => ({ + createOctokitFromToken: mocks.createOctokitFromToken, + getRepoCollaborators: mocks.getRepoCollaborators, + GITHUB_CLOUD_HOSTNAME: "github.com", +})); + +vi.mock("../gitlab.js", () => ({ + createGitLabFromPersonalAccessToken: + mocks.createGitLabFromPersonalAccessToken, + getProjectMembers: mocks.getProjectMembers, +})); + +vi.mock("../bitbucket.js", () => ({ + createBitbucketCloudClient: mocks.createBitbucketCloudClient, + createBitbucketServerClient: mocks.createBitbucketServerClient, + getExplicitUserPermissionsForCloudRepo: + mocks.getExplicitUserPermissionsForCloudRepo, + getUserPermissionsForServerRepo: mocks.getUserPermissionsForServerRepo, +})); + +import { createRepoPermissionSyncWorkload } from "./repoPermissionSyncWorkload.js"; + +const repo = { + id: 42, + name: "github.com/sourcebot-dev/sourcebot", + displayName: "sourcebot-dev/sourcebot", + external_codeHostType: "github", + external_id: "123", + metadata: {}, + connections: [], +}; +const repoFindUniqueOrThrow = vi.fn().mockResolvedValue(repo); +const repoUpdate = vi.fn().mockResolvedValue(repo); +const accountFindMany = vi.fn().mockResolvedValue([]); +const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); +const permissionSyncJobUpsert = vi.fn(); +const permissionSyncJobUpdate = vi.fn().mockResolvedValue({ repo }); +const transaction = vi.fn((queries: Array>) => + Promise.all(queries), +); + +const db = { + repo: { + findUniqueOrThrow: repoFindUniqueOrThrow, + update: repoUpdate, + }, + account: { + findMany: accountFindMany, + }, + accountToRepoPermission: { + createMany: permissionCreateMany, + }, + repoPermissionSyncJob: { + upsert: permissionSyncJobUpsert, + update: permissionSyncJobUpdate, + }, + $transaction: transaction, +} as unknown as PrismaClient; + +const jobLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), +} satisfies JobLogger; + +const createWorkload = () => + createRepoPermissionSyncWorkload({ + db, + settings: { + maxRepoPermissionSyncJobConcurrency: 3, + } as never, + }); + +const lifecycleContext = { + data: { + repoId: 42, + }, + jobId: "job_1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger: jobLogger, +}; + +const processContext = { + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), +}; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.hasEntitlement.mockResolvedValue(true); + mocks.getAuthCredentialsForRepo.mockReset().mockResolvedValue({ + hostUrl: "https://github.com", + token: "token", + }); + mocks.createOctokitFromToken.mockReset().mockResolvedValue({ octokit: {} }); + mocks.getRepoCollaborators.mockReset().mockResolvedValue([]); + repoFindUniqueOrThrow.mockResolvedValue(repo); + permissionSyncJobUpdate.mockResolvedValue({ repo }); + accountFindMany.mockResolvedValue([]); + repoUpdate.mockResolvedValue(repo); + permissionCreateMany.mockResolvedValue({ count: 0 }); +}); + +describe("repoPermissionSyncWorkload", () => { + test("uses the configured concurrency and database-backed lifecycle hooks", () => { + const workload = createWorkload(); + + expect(workload.queueSpec.name).toBe("repo-permission-sync"); + expect(workload.concurrency).toBe(3); + expect(workload.onStarted).toBeTypeOf("function"); + expect(workload.onCompleted).toBeTypeOf("function"); + expect(workload.onTerminalFailure).toBeTypeOf("function"); + }); + + test("syncs the requested repo with its connections", async () => { + await createWorkload().process(processContext); + + expect(repoFindUniqueOrThrow).toHaveBeenCalledWith({ + where: { id: 42 }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + expect(mocks.getAuthCredentialsForRepo).toHaveBeenCalledWith( + repo, + jobLogger, + ); + expect(mocks.createOctokitFromToken).toHaveBeenCalledWith({ + token: "token", + url: undefined, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("does not run without the permission syncing entitlement", async () => { + mocks.hasEntitlement.mockResolvedValue(false); + + await expect(createWorkload().process(processContext)).rejects.toThrow( + "Permission syncing entitlement is not currently available.", + ); + + expect(repoFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + + test("replaces all permissions for a complete GitHub sync", async () => { + const githubRepo = { + ...repo, + external_codeHostType: "github", + external_id: "123", + metadata: {}, + }; + repoFindUniqueOrThrow.mockResolvedValue(githubRepo); + mocks.getAuthCredentialsForRepo.mockResolvedValue({ + hostUrl: "https://github.com", + token: "token", + }); + const octokit = {}; + mocks.createOctokitFromToken.mockResolvedValue({ octokit }); + mocks.getRepoCollaborators.mockResolvedValue([{ id: 101 }]); + accountFindMany.mockResolvedValue([{ id: "account_1" }]); + + await createWorkload().process(processContext); + + expect(accountFindMany).toHaveBeenCalledWith({ + where: { + providerType: "github", + providerAccountId: { + in: ["101"], + }, + issuerUrl: "https://github.com", + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { id: 42 }, + data: { + permittedAccounts: { + deleteMany: {}, + }, + }, + }); + expect(permissionCreateMany).toHaveBeenCalledWith({ + data: [ + { + accountId: "account_1", + repoId: 42, + source: "REPO_DRIVEN", + }, + ], + skipDuplicates: true, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("preserves account-driven permissions for a partial Bitbucket Cloud sync", async () => { + const bitbucketRepo = { + ...repo, + external_codeHostType: "bitbucketCloud", + external_id: "repo-uuid", + metadata: { + codeHostMetadata: { + bitbucketCloud: { + workspace: "sourcebot", + repoSlug: "sourcebot", + }, + }, + }, + }; + repoFindUniqueOrThrow.mockResolvedValue(bitbucketRepo); + mocks.getAuthCredentialsForRepo.mockResolvedValue({ + hostUrl: "https://bitbucket.org", + token: "token", + connectionConfig: { + user: "service-account", + }, + }); + mocks.createBitbucketCloudClient.mockReturnValue({}); + mocks.getExplicitUserPermissionsForCloudRepo.mockResolvedValue([ + { accountId: "upstream-account" }, + ]); + accountFindMany.mockResolvedValue([{ id: "account_1" }]); + + await createWorkload().process(processContext); + + expect(repoUpdate).toHaveBeenCalledWith({ + where: { id: 42 }, + data: { + permittedAccounts: { + deleteMany: { + source: "REPO_DRIVEN", + }, + }, + }, + }); + expect(permissionCreateMany).toHaveBeenCalledWith({ + data: [ + { + accountId: "account_1", + repoId: 42, + source: "REPO_DRIVEN", + }, + ], + skipDuplicates: true, + }); + }); + + test("marks a job as in progress when started", async () => { + await createWorkload().onStarted?.(lifecycleContext); + + expect(permissionSyncJobUpsert).toHaveBeenCalledWith({ + where: { id: "job_1" }, + update: { + status: "IN_PROGRESS", + completedAt: null, + errorMessage: null, + }, + create: { + id: "job_1", + repoId: 42, + status: "IN_PROGRESS", + }, + }); + }); + + test("marks a job completed and updates the repo sync timestamp", async () => { + await createWorkload().onCompleted?.(lifecycleContext, undefined); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith({ + where: { id: "job_1" }, + data: { + status: "COMPLETED", + completedAt: expect.any(Date), + errorMessage: null, + repo: { + update: { + permissionSyncedAt: expect.any(Date), + }, + }, + }, + select: { + repo: true, + }, + }); + }); + + test("marks a job failed after terminal failure", async () => { + const error = new Error("Upstream unavailable"); + + await createWorkload().onTerminalFailure?.(lifecycleContext, error); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith({ + where: { id: "job_1" }, + data: { + status: "FAILED", + completedAt: expect.any(Date), + errorMessage: "Upstream unavailable", + }, + select: { + repo: true, + }, + }); + expect(mocks.captureException).toHaveBeenCalledWith(error, { + tags: { + jobId: "job_1", + queue: "repo-permission-sync", + }, + }); + }); +}); diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.ts new file mode 100644 index 000000000..7fd1e59df --- /dev/null +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.ts @@ -0,0 +1,407 @@ +import * as Sentry from "@sentry/node"; +import { + PermissionSyncSource, + PrismaClient, + RepoPermissionSyncJobStatus, +} from "@sourcebot/db"; +import { + JobLogSink, + REPO_PERMISSION_SYNC_QUEUE, + repoMetadataSchema, +} from "@sourcebot/shared"; +import { hasEntitlement } from "../entitlements.js"; +import { + createOctokitFromToken, + getRepoCollaborators, + GITHUB_CLOUD_HOSTNAME, +} from "../github.js"; +import { + createGitLabFromPersonalAccessToken, + getProjectMembers, +} from "../gitlab.js"; +import { + createBitbucketCloudClient, + createBitbucketServerClient, + getExplicitUserPermissionsForCloudRepo, + getUserPermissionsForServerRepo, +} from "../bitbucket.js"; +import { + RepoAuthCredentials, + RepoWithConnections, + Settings, + Workload, +} from "../types.js"; +import { getAuthCredentialsForRepo } from "../utils.js"; +import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/index.type"; + +interface RepoPermissionSyncWorkloadDependencies { + db: PrismaClient; + settings: Settings; +} + +export const createRepoPermissionSyncWorkload = ({ + db, + settings, +}: RepoPermissionSyncWorkloadDependencies): Workload<"repo-permission-sync"> => ({ + queueSpec: REPO_PERMISSION_SYNC_QUEUE, + concurrency: settings.maxRepoPermissionSyncJobConcurrency, + process: async ({ data: { repoId }, logger }) => { + if (!(await hasEntitlement("permission-syncing"))) { + throw new Error( + "Permission syncing entitlement is not currently available.", + ); + } + + const repo = await db.repo.findUniqueOrThrow({ + where: { + id: repoId, + }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + + const id = repo.id; + logger.debug(`Syncing permissions for repo ${repo.displayName}...`); + + const credentials = await getAuthCredentialsForRepo(repo, logger); + if (!credentials) { + throw new Error(`No credentials found for repo ${id}`); + } + + const { accountIds, isPartialSync = false } = + await getPermissionSyncResult({ + db, + repo, + credentials, + logger: logger, + }); + + await db.$transaction([ + db.repo.update({ + where: { + id: repo.id, + }, + data: { + permittedAccounts: { + // @note: if this is a partial sync, we only want to delete the repo-driven permissions + // since we don't want to overwrite the account-driven permissions. + deleteMany: isPartialSync + ? { + source: PermissionSyncSource.REPO_DRIVEN, + } + : {}, + }, + }, + }), + db.accountToRepoPermission.createMany({ + data: accountIds.map((accountId) => ({ + accountId, + repoId: repo.id, + source: PermissionSyncSource.REPO_DRIVEN, + })), + skipDuplicates: true, + }), + ]); + }, + onStarted: async ({ data: { repoId }, jobId }) => { + await db.repoPermissionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: RepoPermissionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + repoId, + status: RepoPermissionSyncJobStatus.IN_PROGRESS, + }, + }); + }, + onCompleted: async ({ jobId, logger }) => { + const { repo } = await db.repoPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: RepoPermissionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + repo: { + update: { + permissionSyncedAt: new Date(), + }, + }, + }, + select: { + repo: true, + }, + }); + + logger.debug( + `Permissions synced for repo ${repo.displayName ?? repo.name}`, + ); + }, + onTerminalFailure: async ({ jobId, logger }, error) => { + Sentry.captureException(error, { + tags: { + jobId, + queue: REPO_PERMISSION_SYNC_QUEUE.name, + }, + }); + + const { repo } = await db.repoPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: RepoPermissionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + select: { + repo: true, + }, + }); + + logger.error( + `Repo permission sync job failed for repo ${repo.displayName ?? repo.name}: ${error.message}`, + ); + }, +}); + +interface ProviderPermissionSyncProps { + db: PrismaClient; + repo: RepoWithConnections; + credentials: RepoAuthCredentials; + logger: JobLogSink; +} + +interface PermissionSyncResult { + accountIds: string[]; + isPartialSync?: boolean; +} + +const getPermissionSyncResult = async ( + props: ProviderPermissionSyncProps, +): Promise => { + switch (props.repo.external_codeHostType) { + case "github": + return getGitHubPermissionSyncResult(props); + case "gitlab": + return getGitLabPermissionSyncResult(props); + case "bitbucketCloud": + return getBitbucketCloudPermissionSyncResult(props); + case "bitbucketServer": + return getBitbucketServerPermissionSyncResult(props); + default: + throw new Error( + `Unsupported code host type: ${props.repo.external_codeHostType}`, + ); + } +}; + +const getGitHubPermissionSyncResult = async ({ + db, + repo, + credentials, + logger, +}: ProviderPermissionSyncProps): Promise => { + const isGitHubCloud = credentials.hostUrl + ? new URL(credentials.hostUrl).hostname === GITHUB_CLOUD_HOSTNAME + : true; + const { octokit } = await createOctokitFromToken({ + token: credentials.token, + url: isGitHubCloud ? undefined : credentials.hostUrl, + }); + + // @note: this is a bit of a hack since the displayName _might_ not be set.. + // however, this property was introduced many versions ago and _should_ be set + // on each connection sync. Let's throw an error just in case. + if (!repo.displayName) { + throw new Error(`Repo ${repo.id} does not have a displayName`); + } + + const [owner, repoName] = repo.displayName.split("/"); + const collaborators = await getRepoCollaborators(owner, repoName, octokit); + const githubUserIds = collaborators.map((collaborator) => + collaborator.id.toString(), + ); + + logger.debug(`Found ${collaborators.length} collaborator(s)`, { + collaborators: collaborators.flatMap(({ email, login }) => ({email, login})), + }); + + const accounts = await db.account.findMany({ + where: { + providerType: "github", + providerAccountId: { + in: githubUserIds, + }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + }; +}; + +const getGitLabPermissionSyncResult = async ({ + db, + repo, + credentials, +}: ProviderPermissionSyncProps): Promise => { + const api = await createGitLabFromPersonalAccessToken({ + token: credentials.token, + url: credentials.hostUrl, + }); + + const projectId = repo.external_id; + if (!projectId) { + throw new Error(`Repo ${repo.id} does not have an external_id`); + } + + const members = await getProjectMembers(projectId, api); + const gitlabUserIds = members.map((member) => member.id.toString()); + + const accounts = await db.account.findMany({ + where: { + providerType: "gitlab", + providerAccountId: { + in: gitlabUserIds, + }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + }; +}; + +const getBitbucketCloudPermissionSyncResult = async ({ + db, + repo, + credentials, +}: ProviderPermissionSyncProps): Promise => { + const config = credentials.connectionConfig as + | BitbucketConnectionConfig + | undefined; + if (!config) { + throw new Error(`No connection config found for repo ${repo.id}`); + } + + const client = createBitbucketCloudClient(config.user, credentials.token); + + const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); + if (!parsedMetadata.success) { + throw new Error( + `Repo ${repo.id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`, + ); + } + const bitbucketCloudMetadata = + parsedMetadata.data.codeHostMetadata?.bitbucketCloud; + if (!bitbucketCloudMetadata) { + throw new Error( + `Repo ${repo.id} is missing required Bitbucket Cloud metadata (workspace/repoSlug)`, + ); + } + + const { workspace, repoSlug } = bitbucketCloudMetadata; + + // @note: The Bitbucket Cloud permissions API only returns users who have been *directly* + // granted access to this repository. Users who have access via a group added to the repo, + // via project-level membership, or via a group in a project are NOT captured here. + // These users will still gain access through account-driven permission syncing, + // but there may be a delay of up to `userDrivenPermissionSyncIntervalMs` before + // they see the repository in Sourcebot. + // @see: https://developer.atlassian.com/cloud/bitbucket/rest/api-group-repositories/#api-repositories-workspace-repo-slug-permissions-config-users-get + const users = await getExplicitUserPermissionsForCloudRepo( + client, + workspace, + repoSlug, + ); + const userAccountIds = users.map((user) => user.accountId); + + const accounts = await db.account.findMany({ + where: { + providerType: "bitbucket-cloud", + providerAccountId: { + in: userAccountIds, + }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + // Since we only fetch users who have been explicitly granted access to the repo, + // this is a partial sync. + isPartialSync: true, + }; +}; + +const getBitbucketServerPermissionSyncResult = async ({ + db, + repo, + credentials, +}: ProviderPermissionSyncProps): Promise => { + const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); + if (!parsedMetadata.success) { + throw new Error( + `Repo ${repo.id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`, + ); + } + const bitbucketServerMetadata = + parsedMetadata.data.codeHostMetadata?.bitbucketServer; + if (!bitbucketServerMetadata) { + throw new Error( + `Repo ${repo.id} is missing required Bitbucket Server metadata (projectKey/repoSlug)`, + ); + } + + const { projectKey, repoSlug } = bitbucketServerMetadata; + const hostUrl = credentials.hostUrl; + + if (!hostUrl) { + throw new Error( + `No host URL found for Bitbucket Server repo ${repo.id}`, + ); + } + + // @note: This covers users with direct repo-level and project-level permissions. + // Users with access only via groups are NOT captured here. Those users will + // still gain access through account-driven permission syncing. + const client = createBitbucketServerClient( + hostUrl, + /* user = */ undefined, + credentials.token, + ); + const users = await getUserPermissionsForServerRepo( + client, + projectKey, + repoSlug, + ); + const userIds = users.map((user) => user.userId); + + const accounts = await db.account.findMany({ + where: { + providerType: "bitbucket-server", + providerAccountId: { in: userIds }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + isPartialSync: true, + }; +}; diff --git a/packages/backend/src/ee/repoPermissionSyncer.ts b/packages/backend/src/ee/repoPermissionSyncer.ts deleted file mode 100644 index 536f48a09..000000000 --- a/packages/backend/src/ee/repoPermissionSyncer.ts +++ /dev/null @@ -1,436 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { PermissionSyncSource, PrismaClient, Repo, RepoPermissionSyncJobStatus } from "@sourcebot/db"; -import { createLogger, PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "@sourcebot/shared"; -import { env } from "@sourcebot/shared"; -import { hasEntitlement } from "../entitlements.js"; -import { DelayedError, Job, Queue, Worker } from 'bullmq'; -import { Redis } from 'ioredis'; -import { createOctokitFromToken, getRepoCollaborators, GITHUB_CLOUD_HOSTNAME } from "../github.js"; -import { createGitLabFromPersonalAccessToken, getProjectMembers } from "../gitlab.js"; -import { createBitbucketCloudClient, createBitbucketServerClient, getExplicitUserPermissionsForCloudRepo, getUserPermissionsForServerRepo } from "../bitbucket.js"; -import { repoMetadataSchema } from "@sourcebot/shared"; -import { Settings } from "../types.js"; -import { getAuthCredentialsForRepo, setIntervalAsync } from "../utils.js"; -import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/index.type"; - -type RepoPermissionSyncJob = { - jobId: string; -} - -const QUEUE_NAME = 'repoPermissionSyncQueue'; -const POLLING_INTERVAL_MS = 1000; -const ENTITLEMENT_RETRY_DELAY_MS = 30 * 1000; -const LOG_TAG = 'repo-permission-syncer'; - -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); - -export class RepoPermissionSyncer { - private queue: Queue; - private worker: Worker; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - ) { - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - }); - this.worker = new Worker(QUEUE_NAME, this.runJob.bind(this), { - connection: redis, - concurrency: this.settings.maxRepoPermissionSyncJobConcurrency, - }); - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobFailed.bind(this)); - } - - public async startScheduler() { - logger.debug('Starting scheduler'); - - this.interval = setIntervalAsync(async () => { - if (!await hasEntitlement('permission-syncing')) { - return; - } - - // @todo: make this configurable - const thresholdDate = new Date(Date.now() - this.settings.repoDrivenPermissionSyncIntervalMs); - - const repos = await this.db.repo.findMany({ - // Repos need their permissions to be synced against the code host when... - where: { - AND: [ - // They are not public. Public repositories are always visible to all users, therefore we don't - // need to explicitly perform permission syncing for them. - // @see: packages/web/src/prisma.ts - { - isPublic: false - }, - // They belong to a code host that supports permissions syncing - { - external_codeHostType: { - in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, - } - }, - // They have at least one connection with permission enforcement enabled - { - connections: { - some: { - connection: { - enforcePermissions: true, - } - } - } - }, - // They have not been synced within the threshold date. - { - OR: [ - { permissionSyncedAt: null }, - { permissionSyncedAt: { lt: thresholdDate } }, - ], - }, - // There aren't any active or recently failed jobs. - { - NOT: { - permissionSyncJobs: { - some: { - OR: [ - // Don't schedule if there are active jobs - { - status: { - in: [ - RepoPermissionSyncJobStatus.PENDING, - RepoPermissionSyncJobStatus.IN_PROGRESS, - ], - } - }, - // Don't schedule if there are recent failed jobs (within the threshold date). Note `gt` is used here since this is a inverse condition. - { - AND: [ - { status: RepoPermissionSyncJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - } - } - }, - ] - } - }); - - await this.schedulePermissionSync(repos); - }, POLLING_INTERVAL_MS); - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - await this.worker.close(/* force = */ true); - await this.queue.close(); - } - - private async schedulePermissionSync(repos: Repo[]) { - // @note: we don't perform this in a transaction because - // we want to avoid the situation where a job is created and run - // prior to the transaction being committed. - const jobs = await this.db.repoPermissionSyncJob.createManyAndReturn({ - data: repos.map(repo => ({ - repoId: repo.id, - })), - include: { - repo: true, - } - }); - - await this.queue.addBulk(jobs.map((job) => ({ - name: 'repoPermissionSyncJob', - data: { - jobId: job.id, - }, - opts: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - // Priority 1 (high) for never-synced, Priority 2 (normal) for re-sync - priority: job.repo.permissionSyncedAt === null ? 1 : 2, - } - }))) - } - - private async runJob(job: Job) { - if (!await hasEntitlement('permission-syncing')) { - await job.moveToDelayed(Date.now() + ENTITLEMENT_RETRY_DELAY_MS, job.token); - throw new DelayedError('Permission syncing entitlement is not currently available.'); - } - - const id = job.data.jobId; - const logger = createJobLogger(id); - - const { repo } = await this.db.repoPermissionSyncJob.update({ - where: { - id, - }, - data: { - status: RepoPermissionSyncJobStatus.IN_PROGRESS, - }, - select: { - repo: { - include: { - connections: { - include: { - connection: true, - } - } - } - } - } - }); - - if (!repo) { - throw new Error(`Repo ${id} not found`); - } - - logger.debug(`Syncing permissions for repo ${repo.displayName}...`); - - const credentials = await getAuthCredentialsForRepo(repo, logger); - if (!credentials) { - throw new Error(`No credentials found for repo ${id}`); - } - - const { - accountIds, - isPartialSync = false, - } = await (async (): Promise<{ - accountIds: string[], - isPartialSync?: boolean - }> => { - if (repo.external_codeHostType === 'github') { - const isGitHubCloud = credentials.hostUrl ? new URL(credentials.hostUrl).hostname === GITHUB_CLOUD_HOSTNAME : true; - const { octokit } = await createOctokitFromToken({ - token: credentials.token, - url: isGitHubCloud ? undefined : credentials.hostUrl, - }); - - // @note: this is a bit of a hack since the displayName _might_ not be set.. - // however, this property was introduced many versions ago and _should_ be set - // on each connection sync. Let's throw an error just in case. - if (!repo.displayName) { - throw new Error(`Repo ${id} does not have a displayName`); - } - - const [owner, repoName] = repo.displayName.split('/'); - - const collaborators = await getRepoCollaborators(owner, repoName, octokit); - const githubUserIds = collaborators.map(collaborator => collaborator.id.toString()); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'github', - providerAccountId: { - in: githubUserIds, - }, - issuerUrl: credentials.hostUrl, - }, - }); - - return { - accountIds: accounts.map(account => account.id), - } - } else if (repo.external_codeHostType === 'gitlab') { - const api = await createGitLabFromPersonalAccessToken({ - token: credentials.token, - url: credentials.hostUrl, - }); - - const projectId = repo.external_id; - if (!projectId) { - throw new Error(`Repo ${id} does not have an external_id`); - } - - const members = await getProjectMembers(projectId, api); - const gitlabUserIds = members.map(member => member.id.toString()); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'gitlab', - providerAccountId: { - in: gitlabUserIds, - }, - issuerUrl: credentials.hostUrl, - }, - }); - - return { - accountIds: accounts.map(account => account.id), - } - } else if (repo.external_codeHostType === 'bitbucketCloud') { - const config = credentials.connectionConfig as BitbucketConnectionConfig | undefined; - if (!config) { - throw new Error(`No connection config found for repo ${id}`); - } - - const client = createBitbucketCloudClient(config.user, credentials.token); - - const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); - if (!parsedMetadata.success) { - throw new Error(`Repo ${id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`); - } - const bitbucketCloudMetadata = parsedMetadata.data.codeHostMetadata?.bitbucketCloud; - if (!bitbucketCloudMetadata) { - throw new Error(`Repo ${id} is missing required Bitbucket Cloud metadata (workspace/repoSlug)`); - } - - const { workspace, repoSlug } = bitbucketCloudMetadata; - - // @note: The Bitbucket Cloud permissions API only returns users who have been *directly* - // granted access to this repository. Users who have access via a group added to the repo, - // via project-level membership, or via a group in a project are NOT captured here. - // These users will still gain access through user-driven syncing (accountPermissionSyncer), - // but there may be a delay of up to `userDrivenPermissionSyncIntervalMs` before - // they see the repository in Sourcebot. - // @see: https://developer.atlassian.com/cloud/bitbucket/rest/api-group-repositories/#api-repositories-workspace-repo-slug-permissions-config-users-get - const users = await getExplicitUserPermissionsForCloudRepo(client, workspace, repoSlug); - const userAccountIds = users.map(u => u.accountId); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'bitbucket-cloud', - providerAccountId: { - in: userAccountIds, - }, - issuerUrl: credentials.hostUrl, - }, - }); - - return { - accountIds: accounts.map(account => account.id), - // Since we only fetch users who have been explicitly granted access to the repo, - // this is a partial sync. - isPartialSync: true, - } - } else if (repo.external_codeHostType === 'bitbucketServer') { - const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); - if (!parsedMetadata.success) { - throw new Error(`Repo ${id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`); - } - const bitbucketServerMetadata = parsedMetadata.data.codeHostMetadata?.bitbucketServer; - if (!bitbucketServerMetadata) { - throw new Error(`Repo ${id} is missing required Bitbucket Server metadata (projectKey/repoSlug)`); - } - - const { projectKey, repoSlug } = bitbucketServerMetadata; - const hostUrl = credentials.hostUrl; - - if (!hostUrl) { - throw new Error(`No host URL found for Bitbucket Server repo ${id}`); - } - - // @note: This covers users with direct repo-level and project-level permissions. - // Users with access only via groups are NOT captured here. Those users will - // still gain access through account-driven syncing (accountPermissionSyncer). - const client = createBitbucketServerClient(hostUrl, /* user = */ undefined, credentials.token); - const users = await getUserPermissionsForServerRepo(client, projectKey, repoSlug); - const userIds = users.map(u => u.userId); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'bitbucket-server', - providerAccountId: { in: userIds }, - issuerUrl: credentials.hostUrl, - } - }); - - return { - accountIds: accounts.map(account => account.id), - isPartialSync: true, - } - } - - throw new Error(`Unsupported code host type: ${repo.external_codeHostType}`); - })(); - - await this.db.$transaction([ - this.db.repo.update({ - where: { - id: repo.id, - }, - data: { - permittedAccounts: { - // @note: if this is a partial sync, we only want to delete the repo-driven permissions - // since we don't want to overwrite the account-driven permissions. - deleteMany: isPartialSync ? { - source: PermissionSyncSource.REPO_DRIVEN, - } : {}, - } - } - }), - this.db.accountToRepoPermission.createMany({ - data: accountIds.map(accountId => ({ - accountId, - repoId: repo.id, - source: PermissionSyncSource.REPO_DRIVEN, - })), - skipDuplicates: true, - }) - ]); - } - - private async onJobCompleted(job: Job) { - const logger = createJobLogger(job.data.jobId); - - const { repo } = await this.db.repoPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: RepoPermissionSyncJobStatus.COMPLETED, - repo: { - update: { - permissionSyncedAt: new Date(), - } - }, - completedAt: new Date(), - }, - select: { - repo: true - } - }); - - logger.debug(`Permissions synced for repo ${repo.displayName ?? repo.name}`); - } - - private async onJobFailed(job: Job | undefined, err: Error) { - const logger = createJobLogger(job?.data.jobId ?? 'unknown'); - - Sentry.captureException(err, { - tags: { - jobId: job?.data.jobId, - queue: QUEUE_NAME, - } - }); - - const errorMessage = (repoName: string) => `Repo permission sync job failed for repo ${repoName}: ${err.message}`; - - if (job) { - const { repo } = await this.db.repoPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: RepoPermissionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: err.message, - }, - select: { - repo: true - }, - }); - logger.error(errorMessage(repo.displayName ?? repo.name)); - } else { - logger.error(errorMessage('unknown repo (id not found)')); - } - } -} diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index d45f24936..f53dee93a 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -16,6 +16,8 @@ import { redis } from "./redis.js"; import { createConnectionWorkload } from "./connectionWorkload.js"; import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; import { Api } from "./api.js"; +import { createAccountPermissionSyncWorkload } from "./ee/accountPermissionSyncWorkload.js"; +import { createRepoPermissionSyncWorkload } from "./ee/repoPermissionSyncWorkload.js"; const logger = createLogger('backend-entrypoint'); @@ -59,10 +61,20 @@ const repoIndexWorkload = createRepoIndexWorkload({ db: prisma, settings, }); +const accountPermissionSyncWorkload = createAccountPermissionSyncWorkload({ + db: prisma, + settings, +}); +const repoPermissionSyncWorkload = createRepoPermissionSyncWorkload({ + db: prisma, + settings, +}); jobManager.register(reconciliationWorkload); jobManager.register(connectionWorkload); jobManager.register(repoIndexWorkload); +jobManager.register(accountPermissionSyncWorkload); +jobManager.register(repoPermissionSyncWorkload); const api = new Api(promClient, jobManager.getQueues()); diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts index 2a6f1649f..5421c8a4b 100644 --- a/packages/backend/src/jobManager.test.ts +++ b/packages/backend/src/jobManager.test.ts @@ -1,34 +1,38 @@ -import { Redis } from 'ioredis'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; -import { ProcessContext, Workload } from './types.js'; - -const mocks = vi.hoisted(() => ({ - enqueue: vi.fn(), - producerClose: vi.fn(), - workerClose: vi.fn(), - jobLogger: { +import { Redis } from "ioredis"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { ProcessContext, Workload } from "./types.js"; + +const mocks = vi.hoisted(() => { + const jobLogger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), flush: vi.fn(), - }, - workers: [] as Array<{ - processor: (job: unknown) => Promise; - handlers: Map void>; - }>, -})); + }; + return { + enqueue: vi.fn(), + producerClose: vi.fn(), + workerClose: vi.fn(), + jobLogger, + createBullMQJobLogger: vi.fn(() => jobLogger), + workers: [] as Array<{ + processor: (job: unknown) => Promise; + handlers: Map void>; + }>, + }; +}); // The module under test creates a logger at import time; stub it so importing pure helpers // has no side effects (mirrors repoIndexManager.test.ts). -vi.mock('@sourcebot/shared', () => ({ +vi.mock("@sourcebot/shared", () => ({ createLogger: vi.fn(() => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), })), - createBullMQJobLogger: vi.fn(() => mocks.jobLogger), + createBullMQJobLogger: mocks.createBullMQJobLogger, BullMQClient: class { enqueue = mocks.enqueue; close = mocks.producerClose; @@ -40,19 +44,22 @@ vi.mock('@sourcebot/shared', () => ({ })); // Mock the constants module directly so its env-derived cache-dir paths don't load. -vi.mock('./constants.js', () => ({ +vi.mock("./constants.js", () => ({ WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, })); -vi.mock('@sentry/node', () => ({ +vi.mock("@sentry/node", () => ({ captureException: vi.fn(), })); -vi.mock('bullmq', () => ({ +vi.mock("bullmq", () => ({ Worker: class { private readonly record: (typeof mocks.workers)[number]; - constructor(_name: string, processor: (job: unknown) => Promise) { + constructor( + _name: string, + processor: (job: unknown) => Promise, + ) { this.record = { processor, handlers: new Map() }; mocks.workers.push(this.record); } @@ -65,60 +72,67 @@ vi.mock('bullmq', () => ({ }, })); -import { BullMQJobManager, normalizeJobState, parseDuration } from './jobManager.js'; +import { + BullMQJobManager, + normalizeJobState, + parseDuration, +} from "./jobManager.js"; -describe('parseDuration', () => { +describe("parseDuration", () => { test.each([ - ['500ms', 500], - ['30s', 30_000], - ['5m', 300_000], - ['6h', 21_600_000], - ['1d', 86_400_000], - ])('parses %s', (input, expected) => { + ["500ms", 500], + ["30s", 30_000], + ["5m", 300_000], + ["6h", 21_600_000], + ["1d", 86_400_000], + ])("parses %s", (input, expected) => { expect(parseDuration(input)).toBe(expected); }); - test('trims surrounding whitespace', () => { - expect(parseDuration(' 10m ')).toBe(600_000); + test("trims surrounding whitespace", () => { + expect(parseDuration(" 10m ")).toBe(600_000); }); - test.each(['', '5', 'm', '5x', '1.5h', '-5m', '5 m'])('throws on malformed "%s"', (input) => { - expect(() => parseDuration(input)).toThrow(); - }); + test.each(["", "5", "m", "5x", "1.5h", "-5m", "5 m"])( + 'throws on malformed "%s"', + (input) => { + expect(() => parseDuration(input)).toThrow(); + }, + ); }); -describe('normalizeJobState', () => { +describe("normalizeJobState", () => { test.each([ - 'waiting', - 'active', - 'delayed', - 'completed', - 'failed', - 'paused', + "waiting", + "active", + "delayed", + "completed", + "failed", + "paused", ])('passes through "%s"', (state) => { expect(normalizeJobState(state)).toBe(state); }); - test('collapses prioritized and waiting-children to waiting', () => { - expect(normalizeJobState('prioritized')).toBe('waiting'); - expect(normalizeJobState('waiting-children')).toBe('waiting'); + test("collapses prioritized and waiting-children to waiting", () => { + expect(normalizeJobState("prioritized")).toBe("waiting"); + expect(normalizeJobState("waiting-children")).toBe("waiting"); }); - test('maps anything unrecognized to unknown', () => { - expect(normalizeJobState('something-else')).toBe('unknown'); - expect(normalizeJobState('unknown')).toBe('unknown'); + test("maps anything unrecognized to unknown", () => { + expect(normalizeJobState("something-else")).toBe("unknown"); + expect(normalizeJobState("unknown")).toBe("unknown"); }); }); const createWorkload = ( - overrides: Partial> = {}, -): Workload<'connection-sync', { repoCount: number }> => ({ + overrides: Partial> = {}, +): Workload<"connection-sync", { repoCount: number }> => ({ queueSpec: { - name: 'connection-sync', + name: "connection-sync", dedupKey: ({ connectionId }) => `connection:${connectionId}`, jobOptions: { attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, + backoff: { type: "exponential", delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: 500, }, @@ -130,7 +144,8 @@ const createWorkload = ( const data = { connectionId: 42, orgId: 1 }; const job = { - id: 'job-1', + id: "job-1", + queueName: "connection-sync", data, attemptsMade: 2, opts: { attempts: 2 }, @@ -138,85 +153,127 @@ const job = { updateProgress: vi.fn(), }; -describe('BullMQJobManager lifecycle', () => { +describe("BullMQJobManager lifecycle", () => { beforeEach(() => { vi.clearAllMocks(); mocks.workers.length = 0; - mocks.enqueue.mockResolvedValue('job-1'); + mocks.enqueue.mockResolvedValue("job-1"); }); - test('delegates enqueueing to BullMQClient and returns its job id', async () => { + test("delegates enqueueing to BullMQClient and returns its job id", async () => { const manager = new BullMQJobManager({} as Redis); const workload = createWorkload(); manager.register(workload); - const result = await manager.trigger('connection-sync', data); + const result = await manager.trigger("connection-sync", data); - expect(result).toBe('job-1'); + expect(result).toBe("job-1"); expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data); }); - test('calls onStarted before processing and onCompleted after completion', async () => { + test("calls onStarted before processing and onCompleted after completion", async () => { const calls: string[] = []; const workload = createWorkload({ - onStarted: vi.fn(async () => { calls.push('started'); }), + onStarted: vi.fn(async ({ logger }) => { + logger.info("Lifecycle started"); + calls.push("started"); + }), process: vi.fn(async () => { - calls.push('processed'); + calls.push("processed"); return { repoCount: 3 }; }), - onCompleted: vi.fn(async () => { calls.push('completed'); }), + onCompleted: vi.fn(async ({ logger }) => { + logger.info("Lifecycle completed"); + calls.push("completed"); + }), }); const manager = new BullMQJobManager({} as Redis); manager.register(workload); await manager.start(); - const result = await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); - expect(calls).toEqual(['started', 'processed']); + const result = await mocks.workers[0].processor({ + ...job, + attemptsMade: 0, + }); + expect(calls).toEqual(["started", "processed"]); - mocks.workers[0].handlers.get('completed')?.(job, result); - await vi.waitFor(() => expect(calls).toEqual(['started', 'processed', 'completed'])); + mocks.workers[0].handlers.get("completed")?.(job, result); + await vi.waitFor(() => + expect(calls).toEqual(["started", "processed", "completed"]), + ); expect(workload.onCompleted).toHaveBeenCalledWith( - expect.objectContaining({ data, jobId: 'job-1', maxAttempts: 2 }), + expect.objectContaining({ + data, + jobId: "job-1", + maxAttempts: 2, + logger: mocks.jobLogger, + }), { repoCount: 3 }, ); - expect(mocks.jobLogger.flush).toHaveBeenCalled(); + expect(mocks.jobLogger.info).toHaveBeenCalledWith("Lifecycle started"); + expect(mocks.jobLogger.info).toHaveBeenCalledWith( + "Lifecycle completed", + ); + await vi.waitFor(() => { + expect(mocks.jobLogger.flush).toHaveBeenCalledTimes(2); + }); }); - test('provides the structured job logger to the workload processor', async () => { - const process = vi.fn(async (context: ProcessContext<'connection-sync'>) => { - context.logger.info('Processing connection'); - return { repoCount: 3 }; - }); + test("provides the structured job logger to the workload processor", async () => { + const process = vi.fn( + async (context: ProcessContext<"connection-sync">) => { + context.logger.info("Processing connection"); + return { repoCount: 3 }; + }, + ); const manager = new BullMQJobManager({} as Redis); manager.register(createWorkload({ process })); await manager.start(); await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); - expect(process).toHaveBeenCalledWith(expect.objectContaining({ - logger: mocks.jobLogger, - })); - expect(mocks.jobLogger.info).toHaveBeenCalledWith('Processing connection'); + expect(process).toHaveBeenCalledWith( + expect.objectContaining({ + logger: mocks.jobLogger, + }), + ); + expect(mocks.jobLogger.info).toHaveBeenCalledWith( + "Processing connection", + ); expect(mocks.jobLogger.flush).toHaveBeenCalled(); }); - test('reports lifecycle metadata after terminal failure', async () => { - const onTerminalFailure = vi.fn(); + test("reports lifecycle metadata after terminal failure", async () => { + const onTerminalFailure = vi.fn(async ({ logger }) => { + logger.error("Lifecycle failed"); + }); const workload = createWorkload({ onTerminalFailure }); const manager = new BullMQJobManager({} as Redis); manager.register(workload); await manager.start(); - const error = new Error('failed'); - mocks.workers[0].handlers.get('failed')?.(job, error); + const error = new Error("failed"); + mocks.workers[0].handlers.get("failed")?.(job, error); await vi.waitFor(() => { - expect(onTerminalFailure).toHaveBeenCalledWith(expect.objectContaining({ - data, - jobId: 'job-1', - attemptsMade: 2, - maxAttempts: 2, - }), error); + expect(onTerminalFailure).toHaveBeenCalledWith( + expect.objectContaining({ + data, + jobId: "job-1", + attemptsMade: 2, + maxAttempts: 2, + logger: mocks.jobLogger, + }), + error, + ); }); + expect(mocks.jobLogger.error).toHaveBeenCalledWith("Lifecycle failed"); + await vi.waitFor(() => { + expect(mocks.jobLogger.flush).toHaveBeenCalledOnce(); + }); + expect(mocks.createBullMQJobLogger).toHaveBeenCalledWith( + expect.objectContaining({ id: "job-1", attemptsMade: 2 }), + expect.objectContaining({ attempt: 2 }), + ); }); }); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index 97c44da09..a617a5812 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -1,12 +1,25 @@ import * as Sentry from "@sentry/node"; -import { BullMQClient, createBullMQJobLogger, createLogger, DataOf, JobLifecycleContext, QueueName } from "@sourcebot/shared"; +import { + BullMQClient, + createBullMQJobLogger, + createLogger, + DataOf, + JobLogSink, + QueueName, +} from "@sourcebot/shared"; import { Job, Queue, Worker } from "bullmq"; import { Redis } from "ioredis"; import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; -import { JobDetail, JobManager, Schedule, Workload } from "./types.js"; +import { + JobDetail, + JobManager, + Schedule, + JobLifecycleContext, + Workload, +} from "./types.js"; import { prisma } from "./prisma.js"; -const LOG_TAG = 'job-manager'; +const LOG_TAG = "job-manager"; const logger = createLogger(LOG_TAG); const DURATION_UNITS_MS: Record = { @@ -20,39 +33,46 @@ const DURATION_UNITS_MS: Record = { export const parseDuration = (value: string): number => { const match = /^(\d+)(ms|s|m|h|d)$/.exec(value.trim()); if (!match) { - throw new Error(`Invalid duration "${value}". Expected e.g. "500ms", "30s", "5m", "6h", "1d".`); + throw new Error( + `Invalid duration "${value}". Expected e.g. "500ms", "30s", "5m", "6h", "1d".`, + ); } return Number(match[1]) * DURATION_UNITS_MS[match[2]]; }; -export const normalizeJobState = (state: string): JobDetail['state'] => { +export const normalizeJobState = (state: string): JobDetail["state"] => { switch (state) { - case 'waiting': - case 'active': - case 'delayed': - case 'completed': - case 'failed': - case 'paused': + case "waiting": + case "active": + case "delayed": + case "completed": + case "failed": + case "paused": return state; - case 'prioritized': - case 'waiting-children': - return 'waiting'; + case "prioritized": + case "waiting-children": + return "waiting"; default: - return 'unknown'; + return "unknown"; } }; const scheduleToRepeat = (schedule: Schedule) => - 'pattern' in schedule ? { pattern: schedule.pattern } : { every: parseDuration(schedule.every) }; + "pattern" in schedule + ? { pattern: schedule.pattern } + : { every: parseDuration(schedule.every) }; export class BullMQJobManager implements JobManager { - private readonly workloads = new Map>(); + private readonly workloads = new Map< + string, + Workload + >(); private readonly workers = new Map(); private readonly bullmqClient: BullMQClient; private readonly abortController = new AbortController(); constructor(private readonly connection: Redis) { - this.bullmqClient = new BullMQClient(connection, prisma); + this.bullmqClient = new BullMQClient(connection); } register(workload: Workload): void { @@ -71,7 +91,9 @@ export class BullMQJobManager implements JobManager { async start(): Promise { if (this.workloads.size === 0) { - logger.debug('start() called with nothing registered; nothing to do'); + logger.debug( + "start() called with nothing registered; nothing to do", + ); return; } @@ -80,17 +102,21 @@ export class BullMQJobManager implements JobManager { } logger.info( - `Started ${this.workloads.size} workload(s) [${[...this.workloads.keys()].join(', ')}]`, + `Started ${this.workloads.size} workload(s) [${[...this.workloads.keys()].join(", ")}]`, ); } async trigger( workloadName: TName, - data: DataOf + data: DataOf, ): Promise { - const workload = this.workloads.get(workloadName) as Workload | undefined; + const workload = this.workloads.get(workloadName) as + | Workload + | undefined; if (!workload) { - throw new Error(`Cannot trigger unknown workload "${workloadName}"`); + throw new Error( + `Cannot trigger unknown workload "${workloadName}"`, + ); } return this.bullmqClient.enqueue(workload.queueSpec, data); } @@ -98,19 +124,25 @@ export class BullMQJobManager implements JobManager { async stop(): Promise { this.abortController.abort(); - await Promise.all([...this.workers.values()].map((worker) => - Promise.race([ - worker.close(), - new Promise((resolve) => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)), - ]), - )); + await Promise.all( + [...this.workers.values()].map((worker) => + Promise.race([ + worker.close(), + new Promise((resolve) => + setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS), + ), + ]), + ), + ); await this.bullmqClient.close(); - logger.info('Job manager stopped'); + logger.info("Job manager stopped"); } - private async startWorkload(workload: Workload): Promise { + private async startWorkload( + workload: Workload, + ): Promise { const { queueSpec: spec, concurrency, rateLimit, schedule } = workload; const queue = this.bullmqClient.getQueue(spec); @@ -118,24 +150,29 @@ export class BullMQJobManager implements JobManager { const worker = new Worker( spec.name, async (job) => { - const jobLogger = createBullMQJobLogger( + const jobLogger = createBullMQJobLogger(job, { + label: `${LOG_TAG}:${spec.name}:job:${job.id ?? "unknown"}`, + }); + const lifecycleContext = this.jobLifecycleContext( job, - `${LOG_TAG}:${spec.name}:job:${job.id ?? 'unknown'}`, + jobLogger, ); - const lifecycleContext = this.jobLifecycleContext(job); try { await workload.onStarted?.(lifecycleContext); const result = await workload.process({ ...lifecycleContext, signal: this.abortController.signal, - logger: jobLogger, - updateProgress: (progress) => job.updateProgress(progress), + updateProgress: (progress) => + job.updateProgress(progress), trigger: (target, data) => this.trigger(target, data), }); return result; } catch (error) { - jobLogger.error(`Workload "${spec.name}" attempt failed`, error); + jobLogger.error( + `Workload "${spec.name}" attempt failed`, + error, + ); throw error; } finally { await jobLogger.flush(); @@ -146,18 +183,23 @@ export class BullMQJobManager implements JobManager { concurrency, maxStalledCount: 1, ...(rateLimit - ? { limiter: { max: rateLimit.max, duration: parseDuration(rateLimit.per) } } + ? { + limiter: { + max: rateLimit.max, + duration: parseDuration(rateLimit.per), + }, + } : {}), }, ); - worker.on('failed', (job, error) => { + worker.on("failed", (job, error) => { void this.onWorkloadJobFailed(workload, job, error); }); - worker.on('completed', (job, result) => { + worker.on("completed", (job, result) => { void this.onWorkloadJobCompleted(workload, job, result); }); - worker.on('error', (error) => { + worker.on("error", (error) => { logger.error(`Worker "${spec.name}" error:`, error); }); @@ -176,7 +218,9 @@ export class BullMQJobManager implements JobManager { name: spec.name, opts: { attempts: spec.jobOptions.attempts, - removeOnComplete: { count: spec.jobOptions.keep.completed }, + removeOnComplete: { + count: spec.jobOptions.keep.completed, + }, removeOnFail: { count: spec.jobOptions.keep.failed }, keepLogs: spec.jobOptions.keepLogs, }, @@ -196,16 +240,36 @@ export class BullMQJobManager implements JobManager { const maxAttempts = job.opts.attempts ?? 1; const isTerminal = job.attemptsMade >= maxAttempts; if (!isTerminal) { - logger.warn(`Workload "${workload.queueSpec.name}" job ${job.id} failed attempt ${job.attemptsMade}/${maxAttempts}; will retry: ${error.message}`); + logger.warn( + `Workload "${workload.queueSpec.name}" job ${job.id} failed attempt ${job.attemptsMade}/${maxAttempts}; will retry: ${error.message}`, + ); return; } - logger.error(`Workload "${workload.queueSpec.name}" job ${job.id} failed terminally after ${job.attemptsMade} attempt(s): ${error.message}`); + logger.error( + `Workload "${workload.queueSpec.name}" job ${job.id} failed terminally after ${job.attemptsMade} attempt(s): ${error.message}`, + ); + const jobLogger = createBullMQJobLogger(job, { + label: `${LOG_TAG}:${workload.queueSpec.name}:job:${job.id ?? "unknown"}`, + attempt: Math.max(job.attemptsMade, 1), + }); try { - await workload.onTerminalFailure?.(this.jobLifecycleContext(job), error); + await workload.onTerminalFailure?.( + this.jobLifecycleContext(job, jobLogger), + error, + ); } catch (hookError) { Sentry.captureException(hookError); - logger.error(`onTerminalFailure for workload "${workload.queueSpec.name}" threw:`, hookError); + jobLogger.error( + `onTerminalFailure for workload "${workload.queueSpec.name}" threw`, + hookError, + ); + logger.error( + `onTerminalFailure for workload "${workload.queueSpec.name}" threw:`, + hookError, + ); + } finally { + await jobLogger.flush(); } } @@ -214,21 +278,41 @@ export class BullMQJobManager implements JobManager { job: Job, result: TResult, ): Promise { + const jobLogger = createBullMQJobLogger(job, { + label: `${LOG_TAG}:${workload.queueSpec.name}:job:${job.id ?? "unknown"}`, + attempt: Math.max(job.attemptsMade, 1), + }); try { - await workload.onCompleted?.(this.jobLifecycleContext(job), result); + await workload.onCompleted?.( + this.jobLifecycleContext(job, jobLogger), + result, + ); } catch (hookError) { Sentry.captureException(hookError); - logger.error(`onCompleted for workload "${workload.queueSpec.name}" threw:`, hookError); + jobLogger.error( + `onCompleted for workload "${workload.queueSpec.name}" threw`, + hookError, + ); + logger.error( + `onCompleted for workload "${workload.queueSpec.name}" threw:`, + hookError, + ); + } finally { + await jobLogger.flush(); } } - private jobLifecycleContext(job: Job): JobLifecycleContext { + private jobLifecycleContext( + job: Job, + logger: JobLogSink, + ): JobLifecycleContext { return { data: job.data, - jobId: job.id ?? '', + jobId: job.id ?? "", attemptsMade: job.attemptsMade, maxAttempts: job.opts.attempts ?? 1, prisma, + logger, }; } } diff --git a/packages/backend/src/reconciliationWorkload.test.ts b/packages/backend/src/reconciliationWorkload.test.ts index eadb37b88..2391cc6ee 100644 --- a/packages/backend/src/reconciliationWorkload.test.ts +++ b/packages/backend/src/reconciliationWorkload.test.ts @@ -2,7 +2,27 @@ import type { PrismaClient } from '@sourcebot/db'; import type { JobLogger } from '@sourcebot/shared'; import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +const mocks = vi.hoisted(() => ({ + env: { + PERMISSION_SYNC_ENABLED: 'true', + }, + hasEntitlement: vi.fn(), +})); + vi.mock('@sourcebot/shared', () => ({ + env: mocks.env, + PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS: [ + 'github', + 'gitlab', + 'bitbucket-cloud', + 'bitbucket-server', + ], + PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES: [ + 'github', + 'gitlab', + 'bitbucketCloud', + 'bitbucketServer', + ], RECONCILIATION_QUEUE: { name: 'reconciliation', jobOptions: { @@ -14,12 +34,18 @@ vi.mock('@sourcebot/shared', () => ({ }, })); +vi.mock('./entitlements.js', () => ({ + hasEntitlement: mocks.hasEntitlement, +})); + import { createReconciliationWorkload } from './reconciliationWorkload.js'; const settings = { resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, reindexIntervalMs: 60 * 60 * 1000, repoGarbageCollectionGracePeriodMs: 10 * 1000, + userDrivenPermissionSyncIntervalMs: 24 * 60 * 60 * 1000, + repoDrivenPermissionSyncIntervalMs: 24 * 60 * 60 * 1000, }; const logger = { @@ -33,6 +59,7 @@ const logger = { describe('reconciliationWorkload', () => { const connectionFindMany = vi.fn(); const repoFindMany = vi.fn(); + const accountFindMany = vi.fn(); const db = { connection: { findMany: connectionFindMany, @@ -40,6 +67,9 @@ describe('reconciliationWorkload', () => { repo: { findMany: repoFindMany, }, + account: { + findMany: accountFindMany, + }, } as unknown as PrismaClient; beforeEach(() => { @@ -47,6 +77,9 @@ describe('reconciliationWorkload', () => { vi.setSystemTime(new Date('2026-07-27T12:00:00.000Z')); connectionFindMany.mockReset().mockResolvedValue([]); repoFindMany.mockReset().mockResolvedValue([]); + accountFindMany.mockReset().mockResolvedValue([]); + mocks.env.PERMISSION_SYNC_ENABLED = 'true'; + mocks.hasEntitlement.mockReset().mockResolvedValue(false); }); afterEach(() => { @@ -173,4 +206,170 @@ describe('reconciliationWorkload', () => { type: 'CLEANUP', }); }); + + test('submits due accounts for permission syncing when entitled', async () => { + mocks.hasEntitlement.mockResolvedValue(true); + accountFindMany.mockResolvedValue([ + { id: 'account_1' }, + { id: 'account_2' }, + ]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger, + }); + + expect(accountFindMany).toHaveBeenCalledWith({ + where: { + AND: [ + { + providerType: { + in: ['github', 'gitlab', 'bitbucket-cloud', 'bitbucket-server'], + }, + }, + { + OR: [ + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: new Date('2026-07-26T12:00:00.000Z') } }, + ], + }, + ], + }, + select: { + id: true, + }, + }); + expect(trigger).toHaveBeenCalledWith('account-permission-sync', { + accountId: 'account_1', + }); + expect(trigger).toHaveBeenCalledWith('account-permission-sync', { + accountId: 'account_2', + }); + }); + + test('does not query accounts without the permission syncing entitlement', async () => { + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger: vi.fn(), + }); + + expect(accountFindMany).not.toHaveBeenCalled(); + expect(repoFindMany).toHaveBeenCalledTimes(2); + }); + + test('does not query accounts when permission syncing is disabled', async () => { + mocks.env.PERMISSION_SYNC_ENABLED = 'false'; + mocks.hasEntitlement.mockResolvedValue(true); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger: vi.fn(), + }); + + expect(mocks.hasEntitlement).not.toHaveBeenCalled(); + expect(accountFindMany).not.toHaveBeenCalled(); + expect(repoFindMany).toHaveBeenCalledTimes(2); + }); + + test('submits due private repos for permission syncing when entitled', async () => { + mocks.hasEntitlement.mockResolvedValue(true); + repoFindMany + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { id: 42 }, + { id: 84 }, + ]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger, + }); + + expect(repoFindMany).toHaveBeenNthCalledWith(3, { + where: { + AND: [ + { + isPublic: false, + }, + { + external_codeHostType: { + in: ['github', 'gitlab', 'bitbucketCloud', 'bitbucketServer'], + }, + }, + { + connections: { + some: { + connection: { + enforcePermissions: true, + }, + }, + }, + }, + { + OR: [ + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: new Date('2026-07-26T12:00:00.000Z') } }, + ], + }, + ], + }, + select: { + id: true, + }, + }); + expect(trigger).toHaveBeenCalledWith('repo-permission-sync', { + repoId: 42, + }); + expect(trigger).toHaveBeenCalledWith('repo-permission-sync', { + repoId: 84, + }); + }); }); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts index 90a14f14e..fa2f29cfa 100644 --- a/packages/backend/src/reconciliationWorkload.ts +++ b/packages/backend/src/reconciliationWorkload.ts @@ -1,5 +1,6 @@ import { PrismaClient } from "@sourcebot/db"; -import { RECONCILIATION_QUEUE } from "@sourcebot/shared"; +import { env, PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS, RECONCILIATION_QUEUE } from "@sourcebot/shared"; +import { hasEntitlement } from "./entitlements.js"; import { Settings, Workload } from "./types.js"; interface ReconciliationWorkloadDependencies { @@ -15,72 +16,153 @@ export const createReconciliationWorkload = ({ schedule: { every: '10s' }, queueSpec: RECONCILIATION_QUEUE, process: async ({ logger, trigger }) => { - const connectionThreshold = new Date(Date.now() - settings.resyncConnectionIntervalMs); - const connections = await db.connection.findMany({ - where: { - OR: [ - { syncedAt: null }, - { syncedAt: { lt: connectionThreshold } }, - ], - }, - select: { - id: true, - orgId: true, - }, - }); + // Connections + { + const connectionThreshold = new Date(Date.now() - settings.resyncConnectionIntervalMs); + const connections = await db.connection.findMany({ + where: { + OR: [ + { syncedAt: null }, + { syncedAt: { lt: connectionThreshold } }, + ], + }, + select: { + id: true, + orgId: true, + }, + }); + + await Promise.all(connections.map(async (connection) => { + await trigger('connection-sync', { + connectionId: connection.id, + orgId: connection.orgId, + }); + })); + } - await Promise.all(connections.map(async (connection) => { - logger.debug(`Scheduling connection sync for connection ${connection.id}`); - await trigger('connection-sync', { - connectionId: connection.id, - orgId: connection.orgId, + // Repo garbage collection + { + const cleanupThreshold = new Date(Date.now() - settings.repoGarbageCollectionGracePeriodMs); + const reposToCleanup = await db.repo.findMany({ + where: { + connections: { + none: {}, + }, + isAutoCleanupDisabled: false, + OR: [ + { indexedAt: null }, + { indexedAt: { lt: cleanupThreshold } }, + ], + }, + select: { + id: true, + }, }); - })); - const cleanupThreshold = new Date(Date.now() - settings.repoGarbageCollectionGracePeriodMs); - const reposToCleanup = await db.repo.findMany({ - where: { - connections: { - none: {}, + await Promise.all(reposToCleanup.map(async ({ id }) => { + logger.debug(`Scheduling cleanup for repo ${id}`); + await trigger('repo-index', { + repoId: id, + type: 'CLEANUP', + }); + })); + } + + // Repo indexing + { + const indexThreshold = new Date(Date.now() - settings.reindexIntervalMs); + const reposToIndex = await db.repo.findMany({ + where: { + OR: [ + { indexedAt: null }, + { indexedAt: { lt: indexThreshold } }, + ], + }, + select: { + id: true, }, - isAutoCleanupDisabled: false, - OR: [ - { indexedAt: null }, - { indexedAt: { lt: cleanupThreshold } }, - ], - }, - select: { - id: true, - }, - }); + }); + + await Promise.all(reposToIndex.map(async ({ id }) => { + await trigger('repo-index', { + repoId: id, + type: 'INDEX', + }); + })); + } - await Promise.all(reposToCleanup.map(async ({ id }) => { - logger.debug(`Scheduling cleanup for repo ${id}`); - await trigger('repo-index', { - repoId: id, - type: 'CLEANUP', + // Permission syncing + if ( + env.PERMISSION_SYNC_ENABLED === 'true' && + await hasEntitlement('permission-syncing') + ) { + const accountThreshold = new Date(Date.now() - settings.userDrivenPermissionSyncIntervalMs); + const accounts = await db.account.findMany({ + where: { + AND: [ + { + providerType: { + in: PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS, + }, + }, + { + OR: [ + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: accountThreshold } }, + ], + }, + ], + }, + select: { + id: true, + }, }); - })); - const indexThreshold = new Date(Date.now() - settings.reindexIntervalMs); - const reposToIndex = await db.repo.findMany({ - where: { - OR: [ - { indexedAt: null }, - { indexedAt: { lt: indexThreshold } }, - ], - }, - select: { - id: true, - }, - }); + await Promise.all(accounts.map(async ({ id }) => { + await trigger('account-permission-sync', { + accountId: id, + }); + })); - await Promise.all(reposToIndex.map(async ({ id }) => { - logger.debug(`Scheduling index for repo ${id}`); - await trigger('repo-index', { - repoId: id, - type: 'INDEX', + const repoThreshold = new Date(Date.now() - settings.repoDrivenPermissionSyncIntervalMs); + const repos = await db.repo.findMany({ + where: { + AND: [ + { + isPublic: false, + }, + { + external_codeHostType: { + in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, + }, + }, + { + connections: { + some: { + connection: { + enforcePermissions: true, + }, + }, + }, + }, + { + OR: [ + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: repoThreshold } }, + ], + }, + ], + }, + select: { + id: true, + }, }); - })); + + await Promise.all(repos.map(async ({ id }) => { + await trigger('repo-permission-sync', { + repoId: id, + }); + })); + } }, }); diff --git a/packages/backend/src/repoIndexWorkload.test.ts b/packages/backend/src/repoIndexWorkload.test.ts index 076814b3d..023a71e07 100644 --- a/packages/backend/src/repoIndexWorkload.test.ts +++ b/packages/backend/src/repoIndexWorkload.test.ts @@ -1,19 +1,21 @@ -import type { PrismaClient } from '@sourcebot/db'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; -import { createRepoIndexWorkload } from './repoIndexWorkload.js'; +import type { PrismaClient } from "@sourcebot/db"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; const repoIndexingJobUpsert = vi.fn(); const repoIndexingJobUpdate = vi.fn(); const repoUpdate = vi.fn(); -const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => callback({ - repoIndexingJob: { - upsert: repoIndexingJobUpsert, - update: repoIndexingJobUpdate, - }, - repo: { - update: repoUpdate, - }, -})); +const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => + callback({ + repoIndexingJob: { + upsert: repoIndexingJobUpsert, + update: repoIndexingJobUpdate, + }, + repo: { + update: repoUpdate, + }, + }), +); const db = { $transaction: transaction, @@ -26,45 +28,53 @@ const workload = createRepoIndexWorkload({ } as never, }); +const lifecycleLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + const lifecycleContext = { data: { repoId: 42, - type: 'INDEX' as const, + type: "INDEX" as const, }, - jobId: 'job-1', + jobId: "job-1", attemptsMade: 0, maxAttempts: 2, prisma: db, + logger: lifecycleLogger, }; -describe('repoIndexWorkload lifecycle', () => { +describe("repoIndexWorkload lifecycle", () => { beforeEach(() => { vi.clearAllMocks(); }); - test('declares database-backed lifecycle hooks', () => { - expect(workload.onStarted).toBeTypeOf('function'); - expect(workload.onCompleted).toBeTypeOf('function'); - expect(workload.onTerminalFailure).toBeTypeOf('function'); + test("declares database-backed lifecycle hooks", () => { + expect(workload.onStarted).toBeTypeOf("function"); + expect(workload.onCompleted).toBeTypeOf("function"); + expect(workload.onTerminalFailure).toBeTypeOf("function"); }); - test('marks the repo indexing job and repo as in progress when started', async () => { + test("marks the repo indexing job and repo as in progress when started", async () => { await workload.onStarted?.(lifecycleContext); expect(repoIndexingJobUpsert).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, update: { - status: 'IN_PROGRESS', + status: "IN_PROGRESS", completedAt: null, errorMessage: null, }, create: { - id: 'job-1', + id: "job-1", repoId: 42, - type: 'INDEX', - status: 'IN_PROGRESS', + type: "INDEX", + status: "IN_PROGRESS", }, }); expect(repoUpdate).toHaveBeenCalledWith({ @@ -72,20 +82,20 @@ describe('repoIndexWorkload lifecycle', () => { id: 42, }, data: { - latestIndexingJobStatus: 'IN_PROGRESS', + latestIndexingJobStatus: "IN_PROGRESS", }, }); }); - test('marks the repo indexing job and repo as completed', async () => { + test("marks the repo indexing job and repo as completed", async () => { await workload.onCompleted?.(lifecycleContext, undefined); expect(repoIndexingJobUpdate).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, data: { - status: 'COMPLETED', + status: "COMPLETED", completedAt: expect.any(Date), errorMessage: null, }, @@ -95,37 +105,40 @@ describe('repoIndexWorkload lifecycle', () => { id: 42, }, data: { - latestIndexingJobStatus: 'COMPLETED', + latestIndexingJobStatus: "COMPLETED", }, }); }); - test('does not update a completed cleanup job after its repo cascades the job row', async () => { - await workload.onCompleted?.({ - ...lifecycleContext, - data: { - repoId: 42, - type: 'CLEANUP', + test("does not update a completed cleanup job after its repo cascades the job row", async () => { + await workload.onCompleted?.( + { + ...lifecycleContext, + data: { + repoId: 42, + type: "CLEANUP", + }, }, - }, undefined); + undefined, + ); expect(transaction).not.toHaveBeenCalled(); }); - test('marks the repo indexing job and repo as failed after terminal failure', async () => { + test("marks the repo indexing job and repo as failed after terminal failure", async () => { await workload.onTerminalFailure?.( lifecycleContext, - new Error('Unable to clone repository'), + new Error("Unable to clone repository"), ); expect(repoIndexingJobUpdate).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, data: { - status: 'FAILED', + status: "FAILED", completedAt: expect.any(Date), - errorMessage: 'Unable to clone repository', + errorMessage: "Unable to clone repository", }, }); expect(repoUpdate).toHaveBeenCalledWith({ @@ -133,7 +146,7 @@ describe('repoIndexWorkload lifecycle', () => { id: 42, }, data: { - latestIndexingJobStatus: 'FAILED', + latestIndexingJobStatus: "FAILED", }, }); }); diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 848d9f48f..1cc29e147 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -1,19 +1,28 @@ -import { Connection, Repo, RepoToConnection } from "@sourcebot/db"; +import { + Connection, + PrismaClient, + Repo, + RepoToConnection, +} from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type"; -import { DataOf, JobLifecycleContext, JobLogger, QueueName, QueueSpec } from "@sourcebot/shared"; +import { DataOf, JobLogSink, QueueName, QueueSpec } from "@sourcebot/shared"; export type Settings = Required; // @see : https://stackoverflow.com/a/61132308 -export type DeepPartial = T extends object ? { - [P in keyof T]?: DeepPartial; -} : T; +export type DeepPartial = T extends object + ? { + [P in keyof T]?: DeepPartial; + } + : T; // @see: https://stackoverflow.com/a/69328045 export type WithRequired = T & { [P in K]-?: T[P] }; -export type RepoWithConnections = Repo & { connections: (RepoToConnection & { connection: Connection })[] }; +export type RepoWithConnections = Repo & { + connections: (RepoToConnection & { connection: Connection })[]; +}; export type RepoAuthCredentials = { hostUrl?: string; @@ -21,12 +30,20 @@ export type RepoAuthCredentials = { cloneUrlWithToken?: string; authHeader?: string; connectionConfig?: ConnectionConfig; -} +}; +export interface JobLifecycleContext { + data: DataOf; + jobId: string; + attemptsMade: number; + maxAttempts: number; + prisma: PrismaClient; + logger: JobLogSink; +} -export interface ProcessContext extends JobLifecycleContext { +export interface ProcessContext + extends JobLifecycleContext { signal: AbortSignal; - logger: JobLogger; updateProgress(progress: number | object): Promise; trigger(workload: T, data: DataOf): Promise; } @@ -56,9 +73,15 @@ export interface Workload { /** Called before `process` on every attempt. */ onStarted?(ctx: JobLifecycleContext): Promise; /** Called after BullMQ marks the job as completed. */ - onCompleted?(ctx: JobLifecycleContext, result: TResult): Promise; + onCompleted?( + ctx: JobLifecycleContext, + result: TResult, + ): Promise; /** Called after BullMQ exhausts all attempts and marks the job as failed. */ - onTerminalFailure?(ctx: JobLifecycleContext, err: Error): Promise; + onTerminalFailure?( + ctx: JobLifecycleContext, + err: Error, + ): Promise; } export interface JobManager { @@ -69,12 +92,10 @@ export interface JobManager { trigger( workload: TName, - data: DataOf + data: DataOf, ): Promise; } - - export interface QueueCounts { waiting: number; active: number; @@ -83,13 +104,20 @@ export interface QueueCounts { failed: number; paused: number; prioritized?: number; - 'waiting-children'?: number; + "waiting-children"?: number; } export interface JobDetail { id: string; name: string; - state: 'waiting' | 'active' | 'delayed' | 'completed' | 'failed' | 'paused' | 'unknown'; + state: + | "waiting" + | "active" + | "delayed" + | "completed" + | "failed" + | "paused" + | "unknown"; data: TData; attemptsMade: number; maxAttempts: number; diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index 15296f33a..a15b31ea3 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -41,17 +41,17 @@ const schema = { }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", "minimum": 1, "deprecated": true }, @@ -95,12 +95,12 @@ const schema = { }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 } }, @@ -227,17 +227,17 @@ const schema = { }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", "minimum": 1, "deprecated": true }, @@ -281,12 +281,12 @@ const schema = { }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 } }, diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index 901e7762b..c18bc9301 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -110,16 +110,16 @@ export interface Settings { */ reindexRepoPollingIntervalMs?: number; /** - * The number of connection sync jobs to run concurrently. Defaults to 2. + * The number of connection sync jobs to run concurrently. Defaults to 8. */ maxConnectionSyncJobConcurrency?: number; /** - * The number of repo indexing jobs to run concurrently. Defaults to 2. + * The number of repo indexing jobs to run concurrently. Defaults to 8. */ maxRepoIndexingJobConcurrency?: number; /** * @deprecated - * The number of repo GC jobs to run concurrently. Defaults to 2. + * The number of repo GC jobs to run concurrently. Defaults to 8. */ maxRepoGarbageCollectionJobConcurrency?: number; /** @@ -154,11 +154,11 @@ export interface Settings { */ experiment_userDrivenPermissionSyncIntervalMs?: number; /** - * The number of account permission sync jobs to run concurrently. Defaults to 2. + * The number of account permission sync jobs to run concurrently. Defaults to 8. */ maxAccountPermissionSyncJobConcurrency?: number; /** - * The number of repo permission sync jobs to run concurrently. Defaults to 2. + * The number of repo permission sync jobs to run concurrently. Defaults to 8. */ maxRepoPermissionSyncJobConcurrency?: number; } diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts deleted file mode 100644 index 34955363c..000000000 --- a/packages/shared/src/bullmqClient.test.ts +++ /dev/null @@ -1,251 +0,0 @@ -import type { Redis } from 'ioredis'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; -import type { QueueSpec } from './queue.js'; -import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from './jobLogger.js'; -import { CONNECTION_QUEUE, REPO_INDEX_QUEUE } from './queue.js'; - -const queueMocks = vi.hoisted(() => ({ - add: vi.fn(), - close: vi.fn(), - getJob: vi.fn(), - getJobLogs: vi.fn(), -})); - -vi.mock('@sentry/node', () => ({ - captureException: vi.fn(), -})); - -vi.mock('./logger.js', () => ({ - createLogger: vi.fn(() => ({ - error: vi.fn(), - })), -})); - -vi.mock('bullmq', () => ({ - Queue: class { - constructor() { - return queueMocks; - } - }, -})); - -import { BullMQClient } from './bullmqClient.js'; -import { PrismaClient } from '@sourcebot/db'; - -const connectionSpec: QueueSpec<'connection-sync'> = { - name: 'connection-sync', - dedupKey: ({ connectionId }) => `connection:${connectionId}`, - jobOptions: { - attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, - }, -}; - -const data = { connectionId: 42, orgId: 1 }; - -describe('BullMQClient', () => { - const redis = {} as Redis; - const prisma = {} as PrismaClient; - - beforeEach(() => { - vi.clearAllMocks(); - queueMocks.add.mockImplementation(async (_name, _data, options) => ({ id: options.jobId })); - }); - - test('returns the job id when BullMQ accepts the proposed id', async () => { - const client = new BullMQClient(redis, prisma); - - const result = await client.enqueue(connectionSpec, data); - - expect(result).toEqual(expect.any(String)); - expect(queueMocks.add).toHaveBeenCalledWith( - 'connection-sync', - data, - expect.objectContaining({ - jobId: result, - deduplication: { id: 'connection:42' }, - keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, - }), - ); - }); - - test('calls onEnqueued when a new job is created', async () => { - const onEnqueued = vi.fn(); - const client = new BullMQClient(redis, prisma); - - const result = await client.enqueue({ ...connectionSpec, onEnqueued }, data); - - expect(onEnqueued).toHaveBeenCalledWith({ - data, - jobId: result, - attemptsMade: 0, - maxAttempts: 2, - prisma, - }); - }); - - test('returns the existing job id without calling onEnqueued when BullMQ deduplicates the enqueue', async () => { - const onEnqueued = vi.fn(); - queueMocks.add.mockResolvedValue({ id: 'existing-job' }); - const client = new BullMQClient(redis, prisma); - - const result = await client.enqueue({ ...connectionSpec, onEnqueued }, data); - - expect(result).toBe('existing-job'); - expect(onEnqueued).not.toHaveBeenCalled(); - }); - - test('upserts a pending connection sync job when enqueueing a connection sync', async () => { - const connectionSyncJobUpsert = vi.fn(); - const client = new BullMQClient(redis, { - connectionSyncJob: { - upsert: connectionSyncJobUpsert, - }, - } as unknown as PrismaClient); - - const result = await client.enqueue(CONNECTION_QUEUE, data); - - expect(connectionSyncJobUpsert).toHaveBeenCalledWith({ - where: { - id: result, - }, - update: {}, - create: { - id: result, - connectionId: 42, - status: 'PENDING', - warningMessages: [], - }, - }); - }); - - test('upserts a pending repo indexing job when enqueueing with repo-level deduplication', async () => { - const repoIndexingJobUpsert = vi.fn(); - const client = new BullMQClient(redis, { - repoIndexingJob: { - upsert: repoIndexingJobUpsert, - }, - } as unknown as PrismaClient); - - const result = await client.enqueue(REPO_INDEX_QUEUE, { - repoId: 42, - type: 'INDEX', - }); - - expect(queueMocks.add).toHaveBeenCalledWith( - 'repo-index', - { - repoId: 42, - type: 'INDEX', - }, - expect.objectContaining({ - deduplication: { id: 'repo:42' }, - }), - ); - expect(repoIndexingJobUpsert).toHaveBeenCalledWith({ - where: { - id: result, - }, - update: {}, - create: { - id: result, - repoId: 42, - type: 'INDEX', - status: 'PENDING', - }, - }); - }); - - test.each([ - ['waiting', 'PENDING'], - ['waiting-children', 'PENDING'], - ['delayed', 'PENDING'], - ['prioritized', 'PENDING'], - ['paused', 'PENDING'], - ['active', 'IN_PROGRESS'], - ['completed', 'COMPLETED'], - ])('maps BullMQ state %s to %s', async (state, expectedStatus) => { - queueMocks.getJob.mockResolvedValue({ - id: 'job-1', - data, - getState: vi.fn().mockResolvedValue(state), - }); - const client = new BullMQClient(redis, prisma); - - await expect(client.getJob(connectionSpec, 'job-1')).resolves.toEqual({ - id: 'job-1', - data, - status: expectedStatus, - errorMessage: null, - }); - }); - - test('returns the failure reason for a failed job', async () => { - queueMocks.getJob.mockResolvedValue({ - id: 'job-1', - data, - failedReason: 'Connection credentials expired', - getState: vi.fn().mockResolvedValue('failed'), - }); - const client = new BullMQClient(redis, prisma); - - await expect(client.getJob(connectionSpec, 'job-1')).resolves.toEqual({ - id: 'job-1', - data, - status: 'FAILED', - errorMessage: 'Connection credentials expired', - }); - }); - - test.each([ - ['missing job', undefined], - ['unknown state', { - id: 'job-1', - data, - getState: vi.fn().mockResolvedValue('unknown'), - }], - ])('returns null for a %s', async (_label, job) => { - queueMocks.getJob.mockResolvedValue(job); - const client = new BullMQClient(redis, prisma); - - await expect(client.getJob(connectionSpec, 'job-1')).resolves.toBeNull(); - }); - - test('reads and parses incremental job logs', async () => { - queueMocks.getJobLogs.mockResolvedValue({ - logs: [ - JSON.stringify({ - version: 1, - timestamp: '2026-07-28T12:00:00.000Z', - level: 'warn', - message: 'Repository skipped', - attempt: 1, - }), - ], - count: 4, - }); - const client = new BullMQClient(redis, prisma); - - await expect(client.getJobLogs(connectionSpec, 'job-1', { - start: 3, - ascending: true, - })).resolves.toEqual({ - logs: [{ - version: 1, - timestamp: '2026-07-28T12:00:00.000Z', - level: 'warn', - message: 'Repository skipped', - attempt: 1, - }], - count: 4, - }); - expect(queueMocks.getJobLogs).toHaveBeenCalledWith( - 'job-1', - 3, - undefined, - true, - ); - }); -}); diff --git a/packages/shared/src/bullmqClient.ts b/packages/shared/src/bullmqClient.ts index 52a6bff54..5e5e2e43a 100644 --- a/packages/shared/src/bullmqClient.ts +++ b/packages/shared/src/bullmqClient.ts @@ -1,15 +1,10 @@ -import * as Sentry from "@sentry/node"; import { Queue } from "bullmq"; import { randomUUID } from "crypto"; import { Redis } from "ioredis"; -import { createLogger } from "./logger.js"; import { DataOf, QueueName, QueueSpec } from "./queue.js"; -import { PrismaClient } from "@sourcebot/db"; import { readBullMQJobLogs } from "./jobLogger.js"; import type { GetJobLogsOptions, JobLogs } from "./jobLogger.js"; -const logger = createLogger('job-producer'); - export type WorkloadJobStatus = "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED"; export interface WorkloadJob { @@ -52,7 +47,6 @@ export class BullMQClient { constructor( private readonly connection: Redis, - private readonly prisma: PrismaClient, ) {} getQueue(spec: QueueSpec): WorkloadQueue { @@ -117,22 +111,6 @@ export class BullMQClient { throw new Error(`BullMQ did not return an id for workload "${spec.name}"`); } - const isEnqueued = job.id === requestedJobId; - if (isEnqueued && spec.onEnqueued) { - try { - await spec.onEnqueued({ - data, - jobId: job.id, - attemptsMade: 0, - maxAttempts: spec.jobOptions.attempts, - prisma: this.prisma, - }); - } catch (error) { - Sentry.captureException(error); - logger.error(`onEnqueued for workload "${spec.name}" threw:`, error); - } - } - return job.id; } diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index fce116549..134774a0c 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -23,8 +23,8 @@ export const DEFAULT_CONFIG_SETTINGS: ConfigSettings = { resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, // 24 hours resyncConnectionPollingIntervalMs: 1000 * 1, // 1 second reindexRepoPollingIntervalMs: 1000 * 1, // 1 second - maxConnectionSyncJobConcurrency: 2, - maxRepoIndexingJobConcurrency: 2, + maxConnectionSyncJobConcurrency: 8, + maxRepoIndexingJobConcurrency: 8, maxRepoGarbageCollectionJobConcurrency: 2, repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours @@ -33,8 +33,8 @@ export const DEFAULT_CONFIG_SETTINGS: ConfigSettings = { userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours (deprecated) experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours (deprecated) - maxAccountPermissionSyncJobConcurrency: 2, - maxRepoPermissionSyncJobConcurrency: 2, + maxAccountPermissionSyncJobConcurrency: 8, + maxRepoPermissionSyncJobConcurrency: 8, } export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES: CodeHostType[] = [ diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts index 8e082fe63..d374e9f18 100644 --- a/packages/shared/src/env.server.ts +++ b/packages/shared/src/env.server.ts @@ -397,7 +397,6 @@ const options = { REDIS_TLS_HONOR_CIPHER_ORDER: booleanSchema.optional(), REDIS_TLS_KEY_PASSPHRASE: z.string().optional(), - CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000), REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60), GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: numberSchema.default(60 * 10), diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 91d42e05b..d9fe61e1c 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -94,14 +94,15 @@ export { export type { Version } from "./versionUtils.js"; export type { QueueName, - JobLifecycleContext, DataOf, QueueSpec, } from "./queue.js" export { + ACCOUNT_PERMISSION_SYNC_QUEUE, CONNECTION_QUEUE, RECONCILIATION_QUEUE, REPO_INDEX_QUEUE, + REPO_PERMISSION_SYNC_QUEUE, } from "./queue.js"; export { BullMQClient, diff --git a/packages/shared/src/jobLogger.test.ts b/packages/shared/src/jobLogger.test.ts index c6435ea00..cf727c2ab 100644 --- a/packages/shared/src/jobLogger.test.ts +++ b/packages/shared/src/jobLogger.test.ts @@ -71,6 +71,25 @@ describe("createBullMQJobLogger", () => { await expect(logger.flush()).resolves.toBeUndefined(); expect(mocks.applicationError).toHaveBeenCalled(); }); + + test("uses the supplied attempt for post-processing lifecycle logs", async () => { + const log = vi.fn().mockResolvedValue(1); + const logger = createBullMQJobLogger( + { + id: "job-1", + name: "connection", + queueName: "connection", + attemptsMade: 2, + log, + }, + { attempt: 2 }, + ); + + logger.info("Completed"); + await logger.flush(); + + expect(JSON.parse(log.mock.calls[0][0])).toMatchObject({ attempt: 2 }); + }); }); describe("readBullMQJobLogs", () => { diff --git a/packages/shared/src/jobLogger.ts b/packages/shared/src/jobLogger.ts index 57df9311b..f496cdab6 100644 --- a/packages/shared/src/jobLogger.ts +++ b/packages/shared/src/jobLogger.ts @@ -37,11 +37,15 @@ export interface JobLogs { count: number; } -type BullMQLogJob = Pick; +type BullMQLogJob = Pick< + Job, + "id" | "name" | "queueName" | "attemptsMade" | "log" +>; type JobLogQueue = Pick; const JOB_LOG_LEVELS = new Set(["debug", "info", "warn", "error"]); -const SENSITIVE_FIELD_NAME = /authorization|cookie|credential|password|private.?key|secret|token/i; +const SENSITIVE_FIELD_NAME = + /authorization|cookie|credential|password|private.?key|secret|token/i; const MAX_FIELD_DEPTH = 6; const sanitizeValue = ( @@ -104,7 +108,11 @@ const sanitizeFields = (fields: unknown): JobLogFields | undefined => { } const sanitized = sanitizeValue(fields, new WeakSet(), 0); - if (sanitized !== null && typeof sanitized === "object" && !Array.isArray(sanitized)) { + if ( + sanitized !== null && + typeof sanitized === "object" && + !Array.isArray(sanitized) + ) { return sanitized as JobLogFields; } return { value: sanitized }; @@ -163,12 +171,22 @@ export const readBullMQJobLogs = async ( export const createBullMQJobLogger = ( job: BullMQLogJob, - label = `${job.queueName}:job:${job.id ?? "unknown"}`, + options: { + label?: string; + attempt?: number; + } = {}, ): JobLogger => { + const label = + options.label ?? `${job.queueName}:job:${job.id ?? "unknown"}`; + const attempt = options.attempt ?? job.attemptsMade + 1; const applicationLogger = createLogger(label); const pendingWrites = new Set>(); - const write = (level: JobLogLevel, message: string, rawFields?: unknown): void => { + const write = ( + level: JobLogLevel, + message: string, + rawFields?: unknown, + ): void => { const fields = sanitizeFields(rawFields); applicationLogger.log(level, message, fields); @@ -177,10 +195,11 @@ export const createBullMQJobLogger = ( timestamp: new Date().toISOString(), level, message, - attempt: job.attemptsMade + 1, + attempt, ...(fields ? { fields } : {}), }; - const pendingWrite = job.log(JSON.stringify(entry)) + const pendingWrite = job + .log(JSON.stringify(entry)) .then(() => undefined) .catch((error: unknown) => { applicationLogger.error( diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 7c65ac5ef..d7b8d0271 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -1,5 +1,3 @@ - -import { ConnectionSyncJobStatus, PrismaClient, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from "./jobLogger.js"; export type QueueName = keyof QueueRegistry; @@ -7,75 +5,76 @@ export type DataOf = QueueRegistry[TName]; type EmptyJobData = Record; interface QueueRegistry { - 'reconciliation': EmptyJobData, - 'connection-sync': { - connectionId: number, - orgId: number - }, - 'repo-index': { - repoId: number, - type: 'INDEX' | 'CLEANUP', - }, + reconciliation: EmptyJobData; + "connection-sync": { + connectionId: number; + orgId: number; + }; + "repo-index": { + repoId: number; + type: "INDEX" | "CLEANUP"; + }; + "account-permission-sync": { + accountId: string; + }; + "repo-permission-sync": { + repoId: number; + }; } -export const RECONCILIATION_QUEUE: QueueSpec<'reconciliation'> = { - name: 'reconciliation', +export const RECONCILIATION_QUEUE: QueueSpec<"reconciliation"> = { + name: "reconciliation", jobOptions: { attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, + backoff: { type: "exponential", delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, }; -export const CONNECTION_QUEUE: QueueSpec<'connection-sync'> = { - name: 'connection-sync', +export const CONNECTION_QUEUE: QueueSpec<"connection-sync"> = { + name: "connection-sync", jobOptions: { attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, + backoff: { type: "exponential", delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, dedupKey: (data) => `connection:${data.connectionId}`, - onEnqueued: async ({ prisma, data: { connectionId }, jobId }) => { - await prisma.connectionSyncJob.upsert({ - where: { - id: jobId, - }, - update: {}, - create: { - id: jobId, - connectionId, - status: ConnectionSyncJobStatus.PENDING, - warningMessages: [], - }, - }); - } }; -export const REPO_INDEX_QUEUE: QueueSpec<'repo-index'> = { - name: 'repo-index', +export const REPO_INDEX_QUEUE: QueueSpec<"repo-index"> = { + name: "repo-index", jobOptions: { attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, + backoff: { type: "exponential", delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, dedupKey: (data) => `repo:${data.repoId}`, - onEnqueued: async ({ prisma, data: { repoId, type }, jobId }) => { - await prisma.repoIndexingJob.upsert({ - where: { - id: jobId, - }, - update: {}, - create: { - id: jobId, - repoId, - type: RepoIndexingJobType[type], - status: RepoIndexingJobStatus.PENDING, - }, - }); +}; + +export const ACCOUNT_PERMISSION_SYNC_QUEUE: QueueSpec<"account-permission-sync"> = + { + name: "account-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }, + dedupKey: (data) => `account:${data.accountId}`, + }; + +export const REPO_PERMISSION_SYNC_QUEUE: QueueSpec<"repo-permission-sync"> = { + name: "repo-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, + dedupKey: (data) => `repo:${data.repoId}`, }; export interface QueueSpec { @@ -83,17 +82,8 @@ export interface QueueSpec { dedupKey?(data: DataOf): string; jobOptions: { attempts: number; - backoff: { type: 'fixed' | 'exponential'; delayMs: number }; + backoff: { type: "fixed" | "exponential"; delayMs: number }; keep: { completed: number; failed: number }; keepLogs: number; }; - onEnqueued?(ctx: JobLifecycleContext): Promise; -} - -export interface JobLifecycleContext { - data: DataOf; - jobId: string; - attemptsMade: number; - maxAttempts: number; - prisma: PrismaClient; } diff --git a/packages/web/src/lib/bullmqClient.ts b/packages/web/src/lib/bullmqClient.ts index ef5d9524f..a891bf4bd 100644 --- a/packages/web/src/lib/bullmqClient.ts +++ b/packages/web/src/lib/bullmqClient.ts @@ -2,11 +2,10 @@ import 'server-only'; import { BullMQClient } from '@sourcebot/shared'; import { getRedisClient } from './redis'; -import { __unsafePrisma } from '@/prisma'; let client: BullMQClient | undefined; export function getBullMQClient() { - client ??= new BullMQClient(getRedisClient(), __unsafePrisma); + client ??= new BullMQClient(getRedisClient()); return client; } diff --git a/schemas/v3/index.json b/schemas/v3/index.json index 6b5b4fbb1..21499ef0f 100644 --- a/schemas/v3/index.json +++ b/schemas/v3/index.json @@ -40,17 +40,17 @@ }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", "minimum": 1, "deprecated": true }, @@ -94,12 +94,12 @@ }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 } }, From 0cdacda66a9c0cb22a82c0f22fd6aa1c9be8c093 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 10 Aug 2026 12:30:21 -0700 Subject: [PATCH 11/15] further wip --- packages/backend/src/api.ts | 86 ++- packages/backend/src/configManager.test.ts | 170 ++++++ packages/backend/src/configManager.ts | 72 ++- .../backend/src/connectionWorkload.test.ts | 471 +++++++++++------ packages/backend/src/connectionWorkload.ts | 492 +++++++++++++----- .../ee/accountPermissionSyncWorkload.test.ts | 5 +- .../src/ee/permissionSyncEligibility.ts | 25 + .../src/ee/repoPermissionSyncWorkload.test.ts | 5 +- packages/backend/src/index.ts | 26 +- packages/backend/src/jobManager.test.ts | 160 ++++-- packages/backend/src/jobManager.ts | 139 +++-- .../reconcileJobSchedulersAtStartup.test.ts | 166 ++++++ .../src/reconcileJobSchedulersAtStartup.ts | 175 +++++++ .../src/reconciliationWorkload.test.ts | 375 ------------- .../backend/src/reconciliationWorkload.ts | 168 ------ packages/backend/src/types.ts | 99 ++-- packages/shared/src/bullmqClient.test.ts | 135 +++++ packages/shared/src/bullmqClient.ts | 106 +++- packages/shared/src/index.server.ts | 14 +- packages/shared/src/queue.ts | 90 ++-- packages/shared/src/redis.ts | 12 +- packages/shared/src/schedule.test.ts | 42 ++ packages/shared/src/schedule.ts | 43 ++ packages/shared/src/utils.test.ts | 17 +- packages/shared/src/utils.ts | 21 +- packages/shared/vitest.config.ts | 1 + .../repos/components/repoActionsDropdown.tsx | 4 +- .../(app)/repos/components/repoJobsTable.tsx | 4 +- .../app/(app)/repos/components/reposTable.tsx | 4 +- .../components/connectionJobsTable.tsx | 4 +- packages/web/src/auth.ts | 19 +- .../accountPermissionSyncQueue.server.test.ts | 85 +++ .../accountPermissionSyncQueue.server.ts | 41 ++ .../web/src/ee/features/sso/actions.test.ts | 156 ++++++ packages/web/src/ee/features/sso/actions.ts | 58 ++- .../linkedAccountProviderCard.test.tsx | 5 +- .../components/linkedAccountProviderCard.tsx | 4 +- .../src/features/connections/actions.test.ts | 92 ++++ .../web/src/features/connections/actions.ts | 40 ++ .../web/src/features/repos/actions.test.ts | 92 ++++ packages/web/src/features/repos/actions.ts | 40 ++ .../web/src/features/workerApi/actions.ts | 67 +-- .../features/workerApi/client.server.test.ts | 38 -- .../src/features/workerApi/client.server.ts | 26 - .../src/lib/encryptedPrismaAdapter.test.ts | 79 +++ .../web/src/lib/encryptedPrismaAdapter.ts | 25 +- packages/web/src/lib/redis.ts | 8 +- 47 files changed, 2704 insertions(+), 1302 deletions(-) create mode 100644 packages/backend/src/configManager.test.ts create mode 100644 packages/backend/src/ee/permissionSyncEligibility.ts create mode 100644 packages/backend/src/reconcileJobSchedulersAtStartup.test.ts create mode 100644 packages/backend/src/reconcileJobSchedulersAtStartup.ts delete mode 100644 packages/backend/src/reconciliationWorkload.test.ts delete mode 100644 packages/backend/src/reconciliationWorkload.ts create mode 100644 packages/shared/src/bullmqClient.test.ts create mode 100644 packages/shared/src/schedule.test.ts create mode 100644 packages/shared/src/schedule.ts create mode 100644 packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.test.ts create mode 100644 packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.ts create mode 100644 packages/web/src/ee/features/sso/actions.test.ts create mode 100644 packages/web/src/features/connections/actions.test.ts create mode 100644 packages/web/src/features/connections/actions.ts create mode 100644 packages/web/src/features/repos/actions.test.ts create mode 100644 packages/web/src/features/repos/actions.ts delete mode 100644 packages/web/src/features/workerApi/client.server.test.ts delete mode 100644 packages/web/src/features/workerApi/client.server.ts create mode 100644 packages/web/src/lib/encryptedPrismaAdapter.test.ts diff --git a/packages/backend/src/api.ts b/packages/backend/src/api.ts index 4c829a3e9..43ce13a8e 100644 --- a/packages/backend/src/api.ts +++ b/packages/backend/src/api.ts @@ -1,13 +1,19 @@ -import { createLogger, env } from '@sourcebot/shared'; import { createBullBoard } from '@bull-board/api'; import { BullMQAdapter } from '@bull-board/api/bullMQAdapter.js'; import { ExpressAdapter } from '@bull-board/express'; -import { Queue } from 'bullmq'; +import { Octokit } from '@octokit/rest'; +import * as Sentry from "@sentry/node"; +import { PrismaClient, RepoIndexingJobType } from '@sourcebot/db'; +import { createLogger, env, JOB_PRIORITIES } from '@sourcebot/shared'; import express, { NextFunction, Request, Response } from 'express'; import 'express-async-errors'; import * as http from "http"; +import z from 'zod'; +import { SINGLE_TENANT_ORG_ID } from './constants.js'; +import { isGitHubRateLimitError, isNotFound } from './errors.js'; import { PromClient } from './promClient.js'; -import * as Sentry from "@sentry/node"; +import { createGitHubRepoRecord } from './repoCompileUtils.js'; +import type { JobManager } from './types.js'; const logger = createLogger('api'); @@ -17,7 +23,11 @@ const PORT = Number(workerApiUrl.port) || (workerApiUrl.protocol === "https:" ? export class Api { private server: http.Server; - constructor(promClient: PromClient, queues: Queue[]) { + constructor( + promClient: PromClient, + private prisma: PrismaClient, + private jobManager: JobManager, + ) { const app = express(); app.use(express.json()); app.use(express.urlencoded({ extended: true })); @@ -25,7 +35,7 @@ export class Api { const bullBoardAdapter = new ExpressAdapter(); bullBoardAdapter.setBasePath('/admin/queues'); createBullBoard({ - queues: queues.map(queue => new BullMQAdapter(queue, { readOnlyMode: true })), + queues: jobManager.getQueues().map(queue => new BullMQAdapter(queue, { readOnlyMode: true })), serverAdapter: bullBoardAdapter, }); app.use('/admin/queues', bullBoardAdapter.getRouter()); @@ -37,6 +47,8 @@ export class Api { res.end(metrics); }); + app.post(`/api/experimental/add-github-repo`, this.experimental_addGithubRepo.bind(this)); + app.use((error: unknown, _req: Request, _res: Response, next: NextFunction) => { Sentry.captureException(error); next(error); @@ -48,6 +60,70 @@ export class Api { }); } + private async experimental_addGithubRepo(req: Request, res: Response) { + const schema = z.object({ + owner: z.string(), + repo: z.string(), + }).strict(); + + const parsed = schema.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: parsed.error.message }); + return; + } + + const octokit = new Octokit({ + auth: env.EXPERIMENT_ASK_GH_GITHUB_TOKEN, + }); + let response; + try { + response = await octokit.rest.repos.get({ + owner: parsed.data.owner, + repo: parsed.data.repo, + }); + } catch (error) { + if (isNotFound(error)) { + res.status(404).json({ error: 'Repository not found on GitHub' }); + return; + } + if (isGitHubRateLimitError(error)) { + logger.warn(`GitHub API rate limit exceeded while adding ${parsed.data.owner}/${parsed.data.repo}`); + res.status(429).json({ error: 'GitHub API rate limit exceeded' }); + return; + } + throw error; + } + + const record = createGitHubRepoRecord({ + repo: response.data, + hostUrl: 'https://github.com', + isAutoCleanupDisabled: true, + }); + + const repo = await this.prisma.repo.upsert({ + where: { + external_id_external_codeHostUrl_orgId: { + external_id: record.external_id, + external_codeHostUrl: record.external_codeHostUrl, + orgId: SINGLE_TENANT_ORG_ID, + } + }, + update: record, + create: record, + }); + + const jobId = await this.jobManager.trigger( + 'repo-index', + { + repoId: repo.id, + type: RepoIndexingJobType.INDEX, + }, + { priority: JOB_PRIORITIES.INTERACTIVE }, + ); + + res.status(200).json({ jobId, repoId: repo.id }); + } + public async dispose() { return new Promise((resolve, reject) => { this.server.close((err) => { diff --git a/packages/backend/src/configManager.test.ts b/packages/backend/src/configManager.test.ts new file mode 100644 index 000000000..2dc50c320 --- /dev/null +++ b/packages/backend/src/configManager.test.ts @@ -0,0 +1,170 @@ +import type { JobManager } from "./types.js"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => { + const watcher = { + on: vi.fn(), + close: vi.fn(), + }; + watcher.on.mockReturnValue(watcher); + + return { + watcher, + loadConfig: vi.fn(), + resolveConfigSettings: vi.fn(), + syncSearchContexts: vi.fn(), + trigger: vi.fn(), + upsertJobScheduler: vi.fn(), + removeJobScheduler: vi.fn(), + connectionFindUnique: vi.fn(), + connectionCreate: vi.fn(), + connectionUpdate: vi.fn(), + connectionFindMany: vi.fn(), + connectionDelete: vi.fn(), + }; +}); + +vi.mock("@sourcebot/shared", () => ({ + createLogger: vi.fn(() => ({ + debug: vi.fn(), + error: vi.fn(), + })), + env: { + DATA_CACHE_DIR: "test-data", + PERMISSION_SYNC_ENABLED: "false", + }, + JOB_PRIORITIES: { + INTERACTIVE: 1, + SCHEDULED: 10, + }, + loadConfig: mocks.loadConfig, + resolveConfigSettings: mocks.resolveConfigSettings, +})); + +vi.mock("chokidar", () => ({ + default: { + watch: vi.fn(() => mocks.watcher), + }, +})); + +vi.mock("./ee/syncSearchContexts.js", () => ({ + syncSearchContexts: mocks.syncSearchContexts, +})); + +vi.mock("./prisma.js", () => ({ + prisma: { + connection: { + findUnique: mocks.connectionFindUnique, + create: mocks.connectionCreate, + update: mocks.connectionUpdate, + findMany: mocks.connectionFindMany, + delete: mocks.connectionDelete, + }, + }, +})); + +import { ConfigManager } from "./configManager.js"; + +const jobManager = { + trigger: mocks.trigger, + upsertJobScheduler: mocks.upsertJobScheduler, + removeJobScheduler: mocks.removeJobScheduler, +} as unknown as JobManager; + +describe("ConfigManager", () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.clearAllMocks(); + mocks.watcher.on.mockReturnValue(mocks.watcher); + mocks.resolveConfigSettings.mockReturnValue({ + resyncConnectionIntervalMs: 86_400_000, + }); + mocks.syncSearchContexts.mockResolvedValue(undefined); + mocks.trigger.mockResolvedValue("triggered-job"); + mocks.upsertJobScheduler.mockResolvedValue("scheduled-job"); + mocks.removeJobScheduler.mockResolvedValue(true); + mocks.connectionFindUnique.mockResolvedValue(null); + mocks.connectionCreate.mockResolvedValue({ id: 42 }); + mocks.connectionUpdate.mockResolvedValue({ id: 42 }); + mocks.connectionFindMany.mockResolvedValue([]); + mocks.connectionDelete.mockResolvedValue(undefined); + }); + + test("triggers a new connection during config sync", async () => { + const connectionConfig = { + type: "github", + url: "https://github.com", + }; + mocks.loadConfig.mockResolvedValue({ + connections: { sourcebot: connectionConfig }, + contexts: {}, + }); + const manager = new ConfigManager(jobManager, "/config.json"); + + await manager.syncConfig(); + + expect(mocks.trigger).toHaveBeenCalledWith( + "connection-sync", + { connectionId: 42 }, + { priority: 1 }, + ); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "connection-sync", + "connection-sync-v1-42", + 86_400_000, + { connectionId: 42 }, + { priority: 10 }, + ); + expect(mocks.connectionCreate.mock.invocationCallOrder[0]).toBeLessThan( + mocks.upsertJobScheduler.mock.invocationCallOrder[0], + ); + expect( + mocks.upsertJobScheduler.mock.invocationCallOrder[0], + ).toBeLessThan(mocks.trigger.mock.invocationCallOrder[0]); + expect(mocks.syncSearchContexts).toHaveBeenCalledWith({ + contexts: {}, + orgId: 1, + }); + }); + + test("deletes connections removed from the config", async () => { + mocks.loadConfig.mockResolvedValue({}); + mocks.connectionFindMany.mockResolvedValue([ + { id: 42, name: "removed-connection" }, + ]); + const manager = new ConfigManager(jobManager, "/config.json"); + + await manager.syncConfig(); + + expect(mocks.connectionDelete).toHaveBeenCalledWith({ + where: { id: 42 }, + }); + expect(mocks.removeJobScheduler).toHaveBeenCalledWith( + "connection-sync", + "connection-sync-v1-42", + ); + expect( + mocks.removeJobScheduler.mock.invocationCallOrder[0], + ).toBeLessThan(mocks.connectionDelete.mock.invocationCallOrder[0]); + }); + + test("does not trigger an unchanged connection", async () => { + const connectionConfig = { + type: "github", + url: "https://github.com", + }; + mocks.loadConfig.mockResolvedValue({ + connections: { sourcebot: connectionConfig }, + }); + mocks.connectionFindUnique.mockResolvedValue({ + id: 42, + config: connectionConfig, + }); + const manager = new ConfigManager(jobManager, "/config.json"); + + await manager.syncConfig(); + + expect(mocks.upsertJobScheduler).not.toHaveBeenCalled(); + expect(mocks.trigger).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/backend/src/configManager.ts b/packages/backend/src/configManager.ts index 8d04778a5..26e1aaf9e 100644 --- a/packages/backend/src/configManager.ts +++ b/packages/backend/src/configManager.ts @@ -1,22 +1,29 @@ import { Prisma } from "@sourcebot/db"; -import { createLogger, env } from "@sourcebot/shared"; -import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; -import { loadConfig } from "@sourcebot/shared"; +import { + createLogger, + env, + JOB_PRIORITIES, + loadConfig, + resolveConfigSettings, +} from "@sourcebot/shared"; +import type { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import chokidar, { FSWatcher } from 'chokidar'; import { SINGLE_TENANT_ORG_ID } from "./constants.js"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import isEqual from 'fast-deep-equal'; -import { JobManager } from "./types.js"; import { prisma } from "./prisma.js"; +import type { JobManager } from "./types.js"; const logger = createLogger('config-manager'); +const getConnectionSyncSchedulerId = (connectionId: number) => + `connection-sync-v1-${connectionId}`; export class ConfigManager { private watcher: FSWatcher; constructor( - private jobManager: JobManager, - configPath: string, + private readonly jobManager: JobManager, + private readonly configPath: string, ) { this.watcher = chokidar.watch(configPath, { ignoreInitial: true, // Don't fire events for existing files @@ -28,28 +35,35 @@ export class ConfigManager { }); this.watcher.on('change', async () => { - logger.debug(`Config file ${configPath} changed. Syncing config.`); + logger.debug(`Config file ${this.configPath} changed. Syncing config.`); try { - await this.syncConfig(configPath); + await this.syncConfig(); } catch (error) { logger.error(`Failed to sync config: ${error}`); } }); - - this.syncConfig(configPath); } - private syncConfig = async (configPath: string) => { - const config = await loadConfig(configPath); + public syncConfig = async (): Promise => { + const config = await loadConfig(this.configPath); + const settings = resolveConfigSettings(config); - await this.syncConnections(config.connections); + await this.syncConnections( + config.connections, + settings.resyncConnectionIntervalMs, + ); await syncSearchContexts({ contexts: config.contexts, orgId: SINGLE_TENANT_ORG_ID, }); } - private syncConnections = async (connections?: { [key: string]: ConnectionConfig }) => { + private syncConnections = async ( + connections: { [key: string]: ConnectionConfig } | undefined, + intervalMs: number, + ) => { + const connectionIdsToSync: number[] = []; + if (connections) { for (const [key, newConnectionConfig] of Object.entries(connections)) { const existingConnection = await prisma.connection.findUnique({ @@ -99,12 +113,18 @@ export class ConfigManager { } }); + if (!existingConnection) { + await this.jobManager.upsertJobScheduler( + "connection-sync", + getConnectionSyncSchedulerId(connection.id), + intervalMs, + { connectionId: connection.id }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ); + } + if (connectionNeedsSyncing) { - logger.debug(`Change detected for connection '${key}' (id: ${connection.id}). Creating sync job.`); - await this.jobManager.trigger('connection-sync', { - connectionId: connection.id, - orgId: SINGLE_TENANT_ORG_ID, - }) + connectionIdsToSync.push(connection.id); } } } @@ -120,8 +140,22 @@ export class ConfigManager { } }); + await Promise.all( + connectionIdsToSync.map((connectionId) => + this.jobManager.trigger( + "connection-sync", + { connectionId }, + { priority: JOB_PRIORITIES.INTERACTIVE }, + ), + ), + ); + for (const connection of deletedConnections) { logger.debug(`Deleting connection with name '${connection.name}'. Connection ID: ${connection.id}`); + await this.jobManager.removeJobScheduler( + "connection-sync", + getConnectionSyncSchedulerId(connection.id), + ); await prisma.connection.delete({ where: { id: connection.id, diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionWorkload.test.ts index d55deb752..c50cd3cac 100644 --- a/packages/backend/src/connectionWorkload.test.ts +++ b/packages/backend/src/connectionWorkload.test.ts @@ -1,17 +1,22 @@ -import { beforeEach, describe, expect, test, vi } from "vitest"; import type { PrismaClient } from "@sourcebot/db"; +import type { RepoData } from "./repoCompileUtils.js"; +import type { JobManager, ProcessContext } from "./types.js"; +import { beforeEach, describe, expect, test, vi } from "vitest"; const mocks = vi.hoisted(() => ({ connectionFindUniqueOrThrow: vi.fn(), connectionUpdate: vi.fn(), connectionSyncJobUpsert: vi.fn(), connectionSyncJobUpdate: vi.fn(), - repoFindMany: vi.fn(), - repoUpsert: vi.fn(), - repoToConnectionDeleteMany: vi.fn(), compileGithubConfig: vi.fn(), loadConfig: vi.fn(), syncSearchContexts: vi.fn(), + repoFindMany: vi.fn(), + repoUpsert: vi.fn(), + repoToConnectionDeleteMany: vi.fn(), + getJobSchedulerIds: vi.fn(), + upsertJobScheduler: vi.fn(), + removeJobScheduler: vi.fn(), })); vi.mock("@sentry/node", () => ({ @@ -26,18 +31,32 @@ vi.mock("@sourcebot/shared", () => ({ jobOptions: { attempts: 2, backoff: { type: "exponential", delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, + keepJobs: { + completed: { count: 50 }, + failed: { count: 50 }, + }, keepLogs: 500, }, }, - createLogger: vi.fn(() => ({ - debug: vi.fn(), - error: vi.fn(), - })), + JOB_PRIORITIES: { + SCHEDULED: 10, + }, env: { CONFIG_PATH: "/config.json", CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: 60_000, }, + PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES: [ + "github", + "gitlab", + "bitbucketCloud", + "bitbucketServer", + ], + PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS: [ + "github", + "gitlab", + "bitbucket-cloud", + "bitbucket-server", + ], loadConfig: mocks.loadConfig, })); @@ -55,13 +74,23 @@ vi.mock("./ee/syncSearchContexts.js", () => ({ syncSearchContexts: mocks.syncSearchContexts, })); -import { createConnectionWorkload } from "./connectionWorkload.js"; +import { + createConnectionWorkload, + persistConnectionRepositories, + reconcileRepoIndexWork, + reconcileRepoPermissionSyncWork, +} from "./connectionWorkload.js"; +import { REPO_PERMISSION_SYNC_WHERE } from "./ee/permissionSyncEligibility.js"; const db = { connection: { findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, update: mocks.connectionUpdate, }, + connectionSyncJob: { + upsert: mocks.connectionSyncJobUpsert, + update: mocks.connectionSyncJobUpdate, + }, repo: { findMany: mocks.repoFindMany, upsert: mocks.repoUpsert, @@ -69,22 +98,26 @@ const db = { repoToConnection: { deleteMany: mocks.repoToConnectionDeleteMany, }, - connectionSyncJob: { - upsert: mocks.connectionSyncJobUpsert, - update: mocks.connectionSyncJobUpdate, - }, } as unknown as PrismaClient; +const jobManager = { + getJobSchedulerIds: mocks.getJobSchedulerIds, + upsertJobScheduler: mocks.upsertJobScheduler, + removeJobScheduler: mocks.removeJobScheduler, +} as unknown as JobManager; const connectionWorkload = createConnectionWorkload({ db, + jobManager, + permissionSyncEnabled: true, settings: { maxConnectionSyncJobConcurrency: 2, + reindexIntervalMs: 3_600_000, + repoDrivenPermissionSyncIntervalMs: 21_600_000, } as never, }); const data = { connectionId: 42, - orgId: 7, }; const lifecycleLogger = { @@ -105,7 +138,15 @@ const lifecycleContext = { describe("connectionWorkload", () => { beforeEach(() => { + vi.restoreAllMocks(); vi.clearAllMocks(); + mocks.connectionUpdate.mockResolvedValue({}); + mocks.repoFindMany.mockResolvedValue([]); + mocks.getJobSchedulerIds.mockResolvedValue([]); + mocks.upsertJobScheduler.mockResolvedValue("scheduled-job"); + mocks.removeJobScheduler.mockResolvedValue(true); + mocks.loadConfig.mockResolvedValue({ contexts: undefined }); + mocks.syncSearchContexts.mockResolvedValue(undefined); }); test("declares database-backed lifecycle hooks", () => { @@ -172,70 +213,130 @@ describe("connectionWorkload", () => { }); }); - test("discovers repositories using the connection provider", async () => { + test("orchestrates discovery, persistence, and repo work reconciliation", async () => { const config = { type: "github" as const, }; + const discoveredRepo = { + external_id: "repo-4", + external_codeHostUrl: "https://github.com", + }; mocks.connectionFindUniqueOrThrow.mockResolvedValue({ id: 42, name: "github", + orgId: 7, config, }); mocks.compileGithubConfig.mockResolvedValue({ - repoData: [], + repoData: [discoveredRepo], warnings: ["Repository was archived"], }); - mocks.connectionUpdate.mockResolvedValue({}); - mocks.repoFindMany.mockResolvedValue([]); - mocks.loadConfig.mockResolvedValue({ contexts: undefined }); - mocks.syncSearchContexts.mockResolvedValue(undefined); + mocks.repoUpsert.mockResolvedValue({ + id: 4, + name: "github.com/sourcebot/repo-4", + indexedAt: null, + }); + const trigger = vi.fn(); const updateProgress = vi.fn(); - const logger = { - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - flush: vi.fn(), - }; - const signal = new AbortController().signal; const result = await connectionWorkload.process({ ...lifecycleContext, - signal, - logger, + signal: new AbortController().signal, updateProgress, - trigger: vi.fn(), + trigger, }); expect(mocks.compileGithubConfig).toHaveBeenCalledWith( config, 42, - signal, + expect.any(AbortSignal), ); - expect(logger.info).toHaveBeenCalledWith("Discovered 0 repositories", { - connectionId: 42, - repositoryCount: 0, - }); - expect(updateProgress).not.toHaveBeenCalled(); expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ - where: { - id: "job-1", - }, - data: { - warningMessages: ["Repository was archived"], + where: { id: "job-1" }, + data: { warningMessages: ["Repository was archived"] }, + }); + expect(mocks.repoUpsert).toHaveBeenCalledOnce(); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "repo-index", + "repo-index-v1-4", + 3_600_000, + { repoId: 4, type: "INDEX" }, + { priority: 10 }, + ); + expect(trigger).toHaveBeenCalledWith( + "repo-index", + { + repoId: 4, + type: "INDEX", }, + { priority: 10 }, + ); + expect(mocks.connectionUpdate).toHaveBeenCalledWith({ + where: { id: 42 }, + data: { syncedAt: expect.any(Date) }, + }); + expect(mocks.syncSearchContexts).toHaveBeenCalledWith({ + orgId: 7, + contexts: undefined, }); expect(result).toEqual({ reposToCleanup: [], - reposToIndex: [], + reposToIndex: [ + { id: 4, name: "github.com/sourcebot/repo-4" }, + ], }); + expect(updateProgress).not.toHaveBeenCalled(); }); - test("finds orphaned repositories and repositories needing a first index", async () => { - const config = { - type: "github" as const, - }; - const existingIndexedAt = new Date("2026-07-30T12:00:00.000Z"); + test("does not mark the connection synced when repo work reconciliation fails", async () => { + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + orgId: 7, + config: { type: "github" }, + }); + mocks.compileGithubConfig.mockResolvedValue({ + repoData: [ + { + external_id: "repo-4", + external_codeHostUrl: "https://github.com", + }, + ], + warnings: [], + }); + mocks.repoUpsert.mockResolvedValue({ + id: 4, + name: "github.com/sourcebot/repo-4", + indexedAt: null, + }); + mocks.upsertJobScheduler.mockRejectedValueOnce( + new Error("Redis unavailable"), + ); + + await expect( + connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), + }), + ).rejects.toThrow("Redis unavailable"); + + expect(mocks.connectionUpdate).not.toHaveBeenCalled(); + expect(mocks.syncSearchContexts).not.toHaveBeenCalled(); + }); +}); + +describe("connectionWorkload repo sync helpers", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.getJobSchedulerIds.mockResolvedValue([]); + mocks.upsertJobScheduler.mockResolvedValue("scheduled-job"); + mocks.removeJobScheduler.mockResolvedValue(true); + }); + + test("persists the discovered repository snapshot", async () => { + const indexedAt = new Date("2026-07-30T12:00:00.000Z"); const existingRepo = { external_id: "repo-1", external_codeHostUrl: "https://github.com", @@ -256,55 +357,35 @@ describe("connectionWorkload", () => { }, }, }; - - mocks.connectionFindUniqueOrThrow.mockResolvedValue({ - id: 42, - name: "github", - config, - }); - mocks.compileGithubConfig.mockResolvedValue({ - repoData: [existingRepo, newRepo], - warnings: [], - }); mocks.repoFindMany .mockResolvedValueOnce([{ id: 1 }, { id: 2 }, { id: 3 }]) - .mockResolvedValueOnce([{ id: 2, name: "github.com/sourcebot/repo-2" }]); + .mockResolvedValueOnce([ + { id: 2, name: "github.com/sourcebot/repo-2" }, + ]); mocks.repoUpsert .mockResolvedValueOnce({ id: 1, name: "github.com/sourcebot/repo-1", - indexedAt: existingIndexedAt, + indexedAt, }) .mockResolvedValueOnce({ id: 4, name: "github.com/sourcebot/repo-4", indexedAt: null, }); - mocks.repoToConnectionDeleteMany.mockResolvedValue({ count: 2 }); - mocks.connectionUpdate.mockResolvedValue({}); - mocks.loadConfig.mockResolvedValue({ contexts: undefined }); - mocks.syncSearchContexts.mockResolvedValue(undefined); - const trigger = vi.fn(); - const result = await connectionWorkload.process({ - ...lifecycleContext, - signal: new AbortController().signal, - updateProgress: vi.fn(), - trigger, + const result = await persistConnectionRepositories({ + db, + connectionId: 42, + orgId: 7, + discoveredRepos: [ + existingRepo, + newRepo, + newRepo, + ] as unknown as RepoData[], }); - expect(mocks.repoFindMany).toHaveBeenNthCalledWith(1, { - where: { - connections: { - some: { - connectionId: 42, - }, - }, - }, - select: { - id: true, - }, - }); + expect(mocks.repoUpsert).toHaveBeenCalledTimes(2); expect(mocks.repoUpsert).toHaveBeenNthCalledWith(1, { where: { external_id_external_codeHostUrl_orgId: { @@ -317,9 +398,7 @@ describe("connectionWorkload", () => { ...existingRepo, connections: { createMany: { - data: { - connectionId: 42, - }, + data: { connectionId: 42 }, skipDuplicates: true, }, }, @@ -339,88 +418,176 @@ describe("connectionWorkload", () => { }, }, }); - expect(mocks.repoFindMany).toHaveBeenNthCalledWith(2, { - where: { - id: { - in: [2, 3], + expect(result).toEqual({ + currentRepos: [ + { + id: 1, + name: "github.com/sourcebot/repo-1", + indexedAt, }, - connections: { - none: {}, + { + id: 4, + name: "github.com/sourcebot/repo-4", + indexedAt: null, }, - }, - select: { - id: true, - name: true, - }, - }); - expect(result).toEqual({ - reposToCleanup: [ - { id: 2, name: "github.com/sourcebot/repo-2" }, ], - reposToIndex: [ + unindexedRepos: [ { id: 4, name: "github.com/sourcebot/repo-4" }, ], + orphanedRepos: [ + { id: 2, name: "github.com/sourcebot/repo-2" }, + ], + affectedRepoIds: [1, 4, 2, 3], }); - expect(trigger).toHaveBeenNthCalledWith(1, "repo-index", { - repoId: 2, - type: "CLEANUP", - }); - expect(trigger).toHaveBeenNthCalledWith(2, "repo-index", { - repoId: 4, - type: "INDEX", + }); + + test("reconciles repo indexing schedules and immediate work", async () => { + const trigger = vi.fn().mockResolvedValue("job"); + const indexedAt = new Date("2026-07-30T12:00:00.000Z"); + + await reconcileRepoIndexWork({ + jobManager, + trigger: trigger as ProcessContext<"connection-sync">["trigger"], + currentRepos: [ + { id: 1, name: "repo-1", indexedAt }, + { id: 4, name: "repo-4", indexedAt: null }, + ], + unindexedRepos: [{ id: 4, name: "repo-4" }], + orphanedRepos: [{ id: 2, name: "repo-2" }], + intervalMs: 3_600_000, }); - expect(mocks.repoUpsert.mock.invocationCallOrder[1]).toBeLessThan( - mocks.repoToConnectionDeleteMany.mock.invocationCallOrder[0], + + expect(mocks.upsertJobScheduler).toHaveBeenNthCalledWith( + 1, + "repo-index", + "repo-index-v1-1", + 3_600_000, + { repoId: 1, type: "INDEX" }, + { priority: 10 }, ); - expect(mocks.repoFindMany.mock.invocationCallOrder[1]).toBeLessThan( - trigger.mock.invocationCallOrder[0], + expect(mocks.upsertJobScheduler).toHaveBeenNthCalledWith( + 2, + "repo-index", + "repo-index-v1-4", + 3_600_000, + { repoId: 4, type: "INDEX" }, + { priority: 10 }, + ); + expect(mocks.removeJobScheduler).toHaveBeenCalledWith( + "repo-index", + "repo-index-v1-2", + ); + expect(trigger).toHaveBeenNthCalledWith( + 1, + "repo-index", + { + repoId: 2, + type: "CLEANUP", + }, + { priority: 10 }, + ); + expect(trigger).toHaveBeenNthCalledWith( + 2, + "repo-index", + { + repoId: 4, + type: "INDEX", + }, + { priority: 10 }, ); + expect( + mocks.upsertJobScheduler.mock.invocationCallOrder[1], + ).toBeLessThan(trigger.mock.invocationCallOrder[0]); }); - test("does not mark the connection synced when scheduling fails", async () => { - const config = { - type: "github" as const, - }; - const newRepo = { - external_id: "repo-4", - external_codeHostUrl: "https://github.com", - displayName: "sourcebot/repo-4", - connections: { - create: { - connectionId: 42, - }, - }, - }; + test("reconciles repo permission schedules and immediate work", async () => { + const trigger = vi.fn().mockResolvedValue("job"); + const permissionSyncedAt = new Date("2026-07-30T12:00:00.000Z"); + mocks.repoFindMany.mockResolvedValue([ + { id: 1, permissionSyncedAt: null }, + { id: 4, permissionSyncedAt }, + ]); + mocks.getJobSchedulerIds.mockResolvedValue([ + "repo-permission-sync-v1-1", + ]); - mocks.connectionFindUniqueOrThrow.mockResolvedValue({ - id: 42, - name: "github", - config, + await reconcileRepoPermissionSyncWork({ + db, + jobManager, + trigger: trigger as ProcessContext<"connection-sync">["trigger"], + enabled: true, + affectedRepoIds: [1, 4, 2, 3], + intervalMs: 21_600_000, }); - mocks.compileGithubConfig.mockResolvedValue({ - repoData: [newRepo], - warnings: [], - }); - mocks.repoFindMany.mockResolvedValueOnce([]); - mocks.repoUpsert.mockResolvedValueOnce({ - id: 4, - name: "github.com/sourcebot/repo-4", - indexedAt: null, + + expect(mocks.repoFindMany).toHaveBeenCalledWith({ + where: { + id: { + in: [1, 4, 2, 3], + }, + ...REPO_PERMISSION_SYNC_WHERE, + }, + select: { + id: true, + permissionSyncedAt: true, + }, }); - const trigger = vi.fn().mockRejectedValue(new Error("Redis unavailable")); + expect(mocks.upsertJobScheduler).toHaveBeenNthCalledWith( + 1, + "repo-permission-sync", + "repo-permission-sync-v1-1", + 21_600_000, + { repoId: 1 }, + { priority: 10 }, + ); + expect(mocks.upsertJobScheduler).toHaveBeenNthCalledWith( + 2, + "repo-permission-sync", + "repo-permission-sync-v1-4", + 21_600_000, + { repoId: 4 }, + { priority: 10 }, + ); + expect(mocks.removeJobScheduler).toHaveBeenNthCalledWith( + 1, + "repo-permission-sync", + "repo-permission-sync-v1-2", + ); + expect(mocks.removeJobScheduler).toHaveBeenNthCalledWith( + 2, + "repo-permission-sync", + "repo-permission-sync-v1-3", + ); + expect(trigger).toHaveBeenNthCalledWith( + 1, + "repo-permission-sync", + { repoId: 1 }, + { priority: 10 }, + ); + expect(trigger).toHaveBeenNthCalledWith( + 2, + "repo-permission-sync", + { repoId: 4 }, + { priority: 10 }, + ); + }); - await expect(connectionWorkload.process({ - ...lifecycleContext, - signal: new AbortController().signal, - updateProgress: vi.fn(), - trigger, - })).rejects.toThrow("Redis unavailable"); + test("removes permission schedules without querying eligibility when disabled", async () => { + const trigger = vi.fn(); - expect(trigger).toHaveBeenCalledWith("repo-index", { - repoId: 4, - type: "INDEX", + await reconcileRepoPermissionSyncWork({ + db, + jobManager, + trigger: trigger as ProcessContext<"connection-sync">["trigger"], + enabled: false, + affectedRepoIds: [1, 2], + intervalMs: 21_600_000, }); - expect(mocks.connectionUpdate).not.toHaveBeenCalled(); - expect(mocks.syncSearchContexts).not.toHaveBeenCalled(); + + expect(mocks.repoFindMany).not.toHaveBeenCalled(); + expect(mocks.getJobSchedulerIds).not.toHaveBeenCalled(); + expect(mocks.upsertJobScheduler).not.toHaveBeenCalled(); + expect(mocks.removeJobScheduler).toHaveBeenCalledTimes(2); + expect(trigger).not.toHaveBeenCalled(); }); }); diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts index b6e429b50..01b149444 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionWorkload.ts @@ -1,13 +1,30 @@ -import { Settings, Workload } from "./types.js"; -import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; -import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; -import { CONNECTION_QUEUE, env, loadConfig } from "@sourcebot/shared"; -import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import * as Sentry from "@sentry/node"; import { ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db"; +import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; +import { + CONNECTION_QUEUE, + env, + JOB_PRIORITIES, + loadConfig, +} from "@sourcebot/shared"; +import { REPO_PERMISSION_SYNC_WHERE } from "./ee/permissionSyncEligibility.js"; +import { syncSearchContexts } from "./ee/syncSearchContexts.js"; +import { + compileAzureDevOpsConfig, + compileBitbucketConfig, + compileGenericGitHostConfig, + compileGerritConfig, + compileGiteaConfig, + compileGithubConfig, + compileGitlabConfig, +} from "./repoCompileUtils.js"; +import type { RepoData } from "./repoCompileUtils.js"; +import { JobManager, ProcessContext, Settings, Workload } from "./types.js"; interface Props { - db: PrismaClient, + db: PrismaClient; + jobManager: JobManager; + permissionSyncEnabled: boolean; settings: Settings; } @@ -18,40 +35,37 @@ interface ConnectionSyncResult { export const createConnectionWorkload = ({ db, - settings -}: Props): Workload<'connection-sync', ConnectionSyncResult> => ({ + jobManager, + permissionSyncEnabled, + settings, +}: Props): Workload<"connection-sync", ConnectionSyncResult> => ({ queueSpec: CONNECTION_QUEUE, concurrency: settings.maxConnectionSyncJobConcurrency, process: async ({ - data: { - connectionId, - orgId - }, + data: { connectionId }, logger, signal, jobId, trigger, }) => { - logger.info(`Syncing connection ${connectionId}`, { - connectionId, - orgId, - }); const connection = await db.connection.findUniqueOrThrow({ where: { - id: connectionId - } + id: connectionId, + }, }); + const { orgId } = connection; - const config = connection.config as unknown as ConnectionConfig; + logger.info(`Syncing connection ${connectionId}`, { + connectionId, + orgId, + }); - const result = await discoverConnectionRepositories({ - config, + const { repoData, warnings } = await discoverConnectionRepositories({ + config: connection.config as unknown as ConnectionConfig, connectionId, signal, }); - let { repoData, warnings } = result; - await db.connectionSyncJob.update({ where: { id: jobId, @@ -66,123 +80,47 @@ export const createConnectionWorkload = ({ repositoryCount: repoData.length, }); - // Filter out any duplicates by external_id and external_codeHostUrl. - repoData = repoData.filter((repo, index, self) => { - return index === self.findIndex(r => - r.external_id === repo.external_id && - r.external_codeHostUrl === repo.external_codeHostUrl - ); - }) - - const previouslyAssociatedRepos = await db.repo.findMany({ - where: { - connections: { - some: { - connectionId, - }, - }, - }, - select: { - id: true, - }, + const repoChanges = await persistConnectionRepositories({ + db, + connectionId, + orgId, + discoveredRepos: repoData, }); - const upsertedRepos: { id: number; name: string; indexedAt: Date | null }[] = []; - - for (const repo of repoData) { - const upsertedRepo = await db.repo.upsert({ - where: { - external_id_external_codeHostUrl_orgId: { - external_id: repo.external_id, - external_codeHostUrl: repo.external_codeHostUrl, - orgId: orgId, - } - }, - update: { - ...repo, - connections: { - createMany: { - data: { - connectionId, - }, - skipDuplicates: true, - }, - }, - }, - create: repo, - select: { - id: true, - name: true, - indexedAt: true, - }, - }) - upsertedRepos.push(upsertedRepo); - } - - const currentRepoIds = new Set(upsertedRepos.map(({ id }) => id)); - const staleRepoIds = previouslyAssociatedRepos - .map(({ id }) => id) - .filter((id) => !currentRepoIds.has(id)); - - if (staleRepoIds.length > 0) { - await db.repoToConnection.deleteMany({ - where: { - connectionId, - repoId: { - in: staleRepoIds, - }, - }, - }); - } - - const reposToCleanup = staleRepoIds.length > 0 - ? await db.repo.findMany({ - where: { - id: { - in: staleRepoIds, - }, - connections: { - none: {}, - }, - }, - select: { - id: true, - name: true, - }, - }) - : []; + await reconcileRepoIndexWork({ + jobManager, + trigger, + currentRepos: repoChanges.currentRepos, + unindexedRepos: repoChanges.unindexedRepos, + orphanedRepos: repoChanges.orphanedRepos, + intervalMs: settings.reindexIntervalMs, + }); - const reposToIndex = upsertedRepos - .filter(({ indexedAt }) => indexedAt === null) - .map(({ id, name }) => ({ id, name })); - - await Promise.all(reposToCleanup.map(({ id }) => - trigger('repo-index', { - repoId: id, - type: 'CLEANUP', - }) - )); - - await Promise.all(reposToIndex.map(({ id }) => - trigger('repo-index', { - repoId: id, - type: 'INDEX', - }) - )); - - logger.info(`Stored ${repoData.length} repositories`, { - connectionId, - connectionName: connection.name, - repositoryCount: repoData.length, + await reconcileRepoPermissionSyncWork({ + db, + jobManager, + trigger, + enabled: permissionSyncEnabled, + affectedRepoIds: repoChanges.affectedRepoIds, + intervalMs: settings.repoDrivenPermissionSyncIntervalMs, }); + logger.info( + `Stored ${repoChanges.currentRepos.length} repositories`, + { + connectionId, + connectionName: connection.name, + repositoryCount: repoChanges.currentRepos.length, + }, + ); + await db.connection.update({ where: { id: connectionId, }, data: { syncedAt: new Date(), - } + }, }); // After a connection has synced, we need to re-sync the org's search contexts as @@ -194,9 +132,12 @@ export const createConnectionWorkload = ({ orgId, contexts: config.contexts, }); - } catch (err) { - logger.error(`Failed to sync search contexts for connection ${connectionId}`, err); - Sentry.captureException(err); + } catch (error) { + logger.error( + `Failed to sync search contexts for connection ${connectionId}`, + error, + ); + Sentry.captureException(error); } logger.info(`Connection ${connectionId} sync finished`, { @@ -204,8 +145,8 @@ export const createConnectionWorkload = ({ }); return { - reposToCleanup, - reposToIndex, + reposToCleanup: repoChanges.orphanedRepos, + reposToIndex: repoChanges.unindexedRepos, }; }, onStarted: async ({ data: { connectionId }, jobId }) => { @@ -253,6 +194,275 @@ export const createConnectionWorkload = ({ }, }); +export interface CurrentRepo { + id: number; + name: string; + indexedAt: Date | null; +} + +export interface ConnectionRepoChanges { + currentRepos: CurrentRepo[]; + unindexedRepos: { id: number; name: string }[]; + orphanedRepos: { id: number; name: string }[]; + affectedRepoIds: number[]; +} + +type Trigger = ProcessContext<"connection-sync">["trigger"]; + +const deduplicateRepos = (repos: RepoData[]): RepoData[] => + repos.filter( + (repo, index, allRepos) => + index === + allRepos.findIndex( + (candidate) => + candidate.external_id === repo.external_id && + candidate.external_codeHostUrl === + repo.external_codeHostUrl, + ), + ); + +export const persistConnectionRepositories = async ({ + db, + connectionId, + orgId, + discoveredRepos, +}: { + db: PrismaClient; + connectionId: number; + orgId: number; + discoveredRepos: RepoData[]; +}): Promise => { + const previouslyAssociatedRepos = await db.repo.findMany({ + where: { + connections: { + some: { + connectionId, + }, + }, + }, + select: { + id: true, + }, + }); + + const currentRepos: CurrentRepo[] = []; + for (const repo of deduplicateRepos(discoveredRepos)) { + currentRepos.push( + await db.repo.upsert({ + where: { + external_id_external_codeHostUrl_orgId: { + external_id: repo.external_id, + external_codeHostUrl: repo.external_codeHostUrl, + orgId, + }, + }, + update: { + ...repo, + connections: { + createMany: { + data: { + connectionId, + }, + skipDuplicates: true, + }, + }, + }, + create: repo, + select: { + id: true, + name: true, + indexedAt: true, + }, + }), + ); + } + + const currentRepoIds = new Set(currentRepos.map(({ id }) => id)); + const staleRepoIds = previouslyAssociatedRepos + .map(({ id }) => id) + .filter((id) => !currentRepoIds.has(id)); + + if (staleRepoIds.length > 0) { + await db.repoToConnection.deleteMany({ + where: { + connectionId, + repoId: { + in: staleRepoIds, + }, + }, + }); + } + + const orphanedRepos = + staleRepoIds.length > 0 + ? await db.repo.findMany({ + where: { + id: { + in: staleRepoIds, + }, + connections: { + none: {}, + }, + }, + select: { + id: true, + name: true, + }, + }) + : []; + + return { + currentRepos, + unindexedRepos: currentRepos + .filter(({ indexedAt }) => indexedAt === null) + .map(({ id, name }) => ({ id, name })), + orphanedRepos, + affectedRepoIds: [...new Set([...currentRepoIds, ...staleRepoIds])], + }; +}; + +export const reconcileRepoIndexWork = async ({ + jobManager, + trigger, + currentRepos, + unindexedRepos, + orphanedRepos, + intervalMs, +}: { + jobManager: JobManager; + trigger: Trigger; + currentRepos: CurrentRepo[]; + unindexedRepos: { id: number; name: string }[]; + orphanedRepos: { id: number; name: string }[]; + intervalMs: number; +}): Promise => { + await Promise.all( + currentRepos.map(({ id }) => + jobManager.upsertJobScheduler( + "repo-index", + `repo-index-v1-${id}`, + intervalMs, + { repoId: id, type: "INDEX" }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ), + ), + ); + + await Promise.all( + orphanedRepos.map(({ id }) => + jobManager.removeJobScheduler( + "repo-index", + `repo-index-v1-${id}`, + ), + ), + ); + + await Promise.all( + orphanedRepos.map(({ id }) => + trigger( + "repo-index", + { + repoId: id, + type: "CLEANUP", + }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ), + ), + ); + + await Promise.all( + unindexedRepos.map(({ id }) => + trigger( + "repo-index", + { + repoId: id, + type: "INDEX", + }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ), + ), + ); +}; + +export const reconcileRepoPermissionSyncWork = async ({ + db, + jobManager, + trigger, + enabled, + affectedRepoIds, + intervalMs, +}: { + db: PrismaClient; + jobManager: JobManager; + trigger: Trigger; + enabled: boolean; + affectedRepoIds: number[]; + intervalMs: number; +}): Promise => { + const [eligibleRepos, existingSchedulerIds] = + enabled && affectedRepoIds.length > 0 + ? await Promise.all([ + db.repo.findMany({ + where: { + id: { + in: affectedRepoIds, + }, + ...REPO_PERMISSION_SYNC_WHERE, + }, + select: { + id: true, + permissionSyncedAt: true, + }, + }), + jobManager.getJobSchedulerIds("repo-permission-sync"), + ]) + : [[], []]; + const existingSchedulerIdSet = new Set(existingSchedulerIds); + const eligibleRepoIds = new Set(eligibleRepos.map(({ id }) => id)); + const ineligibleRepoIds = affectedRepoIds.filter( + (id) => !eligibleRepoIds.has(id), + ); + + await Promise.all( + eligibleRepos.map(({ id }) => + jobManager.upsertJobScheduler( + "repo-permission-sync", + `repo-permission-sync-v1-${id}`, + intervalMs, + { repoId: id }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ), + ), + ); + + await Promise.all( + ineligibleRepoIds.map((id) => + jobManager.removeJobScheduler( + "repo-permission-sync", + `repo-permission-sync-v1-${id}`, + ), + ), + ); + + await Promise.all( + eligibleRepos + .filter( + ({ id, permissionSyncedAt }) => + permissionSyncedAt === null || + !existingSchedulerIdSet.has( + `repo-permission-sync-v1-${id}`, + ), + ) + .map(({ id }) => + trigger( + "repo-permission-sync", + { repoId: id }, + { priority: JOB_PRIORITIES.SCHEDULED }, + ), + ), + ); +}; + const discoverConnectionRepositories = async ({ config, connectionId, @@ -263,25 +473,25 @@ const discoverConnectionRepositories = async ({ signal: AbortSignal; }) => { switch (config.type) { - case 'github': { + case "github": { return compileGithubConfig(config, connectionId, signal); } - case 'gitlab': { + case "gitlab": { return compileGitlabConfig(config, connectionId); } - case 'gitea': { + case "gitea": { return compileGiteaConfig(config, connectionId); } - case 'gerrit': { + case "gerrit": { return compileGerritConfig(config, connectionId); } - case 'bitbucket': { + case "bitbucket": { return compileBitbucketConfig(config, connectionId); } - case 'azuredevops': { + case "azuredevops": { return compileAzureDevOpsConfig(config, connectionId); } - case 'git': { + case "git": { return compileGenericGitHostConfig(config, connectionId); } } diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts index 7326524a6..4633dda27 100644 --- a/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts @@ -24,7 +24,10 @@ vi.mock("@sourcebot/shared", async (importOriginal) => ({ jobOptions: { attempts: 2, backoff: { type: "exponential", delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, + keepJobs: { + completed: { count: 50 }, + failed: { count: 50 }, + }, keepLogs: 500, }, }, diff --git a/packages/backend/src/ee/permissionSyncEligibility.ts b/packages/backend/src/ee/permissionSyncEligibility.ts new file mode 100644 index 000000000..74dfcfbbf --- /dev/null +++ b/packages/backend/src/ee/permissionSyncEligibility.ts @@ -0,0 +1,25 @@ +import type { Prisma } from "@sourcebot/db"; +import { + PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, + PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS, +} from "@sourcebot/shared"; + +export const ACCOUNT_PERMISSION_SYNC_WHERE = { + providerType: { + in: [...PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS], + }, +} satisfies Prisma.AccountWhereInput; + +export const REPO_PERMISSION_SYNC_WHERE = { + isPublic: false, + external_codeHostType: { + in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, + }, + connections: { + some: { + connection: { + enforcePermissions: true, + }, + }, + }, +} satisfies Prisma.RepoWhereInput; diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts index 648c5b88d..96796e65d 100644 --- a/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts @@ -27,7 +27,10 @@ vi.mock("@sourcebot/shared", async (importOriginal) => ({ jobOptions: { attempts: 2, backoff: { type: "exponential", delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, + keepJobs: { + completed: { count: 50 }, + failed: { count: 50 }, + }, keepLogs: 500, }, }, diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index f53dee93a..7d32466a5 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -11,13 +11,14 @@ import { BullMQJobManager } from "./jobManager.js"; import { shutdownPosthog } from "./posthog.js"; import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; -import { createReconciliationWorkload } from "./reconciliationWorkload.js"; import { redis } from "./redis.js"; import { createConnectionWorkload } from "./connectionWorkload.js"; import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; import { Api } from "./api.js"; import { createAccountPermissionSyncWorkload } from "./ee/accountPermissionSyncWorkload.js"; import { createRepoPermissionSyncWorkload } from "./ee/repoPermissionSyncWorkload.js"; +import { reconcileJobSchedulersAtStartup } from "./reconcileJobSchedulersAtStartup.js"; +import { hasEntitlement } from "./entitlements.js"; const logger = createLogger('backend-entrypoint'); @@ -42,6 +43,9 @@ try { const settings = await getConfigSettings(env.CONFIG_PATH); +const permissionSyncEnabled = + env.PERMISSION_SYNC_ENABLED === "true" && + (await hasEntitlement("permission-syncing")); const promClient = new PromClient(); @@ -49,12 +53,10 @@ logger.info('Worker started.'); const jobManager = new BullMQJobManager(redis); -const reconciliationWorkload = createReconciliationWorkload({ - db: prisma, - settings, -}); const connectionWorkload = createConnectionWorkload({ db: prisma, + jobManager, + permissionSyncEnabled, settings, }); const repoIndexWorkload = createRepoIndexWorkload({ @@ -70,18 +72,26 @@ const repoPermissionSyncWorkload = createRepoPermissionSyncWorkload({ settings, }); -jobManager.register(reconciliationWorkload); jobManager.register(connectionWorkload); jobManager.register(repoIndexWorkload); jobManager.register(accountPermissionSyncWorkload); jobManager.register(repoPermissionSyncWorkload); -const api = new Api(promClient, jobManager.getQueues()); +const api = new Api(promClient, prisma, jobManager); await cleanupOrphanedRepoResources(prisma); -await jobManager.start(); const configManager = new ConfigManager(jobManager, env.CONFIG_PATH); +await configManager.syncConfig(); + +await reconcileJobSchedulersAtStartup({ + db: prisma, + jobManager, + permissionSyncEnabled, + settings, +}); + +await jobManager.start(); const listenToShutdownSignals = () => { diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts index 5421c8a4b..f2cc64b9d 100644 --- a/packages/backend/src/jobManager.test.ts +++ b/packages/backend/src/jobManager.test.ts @@ -12,6 +12,9 @@ const mocks = vi.hoisted(() => { }; return { enqueue: vi.fn(), + upsertJobScheduler: vi.fn(), + getJobSchedulerIds: vi.fn(), + removeJobScheduler: vi.fn(), producerClose: vi.fn(), workerClose: vi.fn(), jobLogger, @@ -33,8 +36,14 @@ vi.mock("@sourcebot/shared", () => ({ debug: vi.fn(), })), createBullMQJobLogger: mocks.createBullMQJobLogger, + scheduleToMs: vi.fn((schedule: string | number) => + typeof schedule === "number" ? schedule : 300_000, + ), BullMQClient: class { enqueue = mocks.enqueue; + upsertJobScheduler = mocks.upsertJobScheduler; + getJobSchedulerIds = mocks.getJobSchedulerIds; + removeJobScheduler = mocks.removeJobScheduler; close = mocks.producerClose; getQueue = vi.fn(() => ({ getJobCounts: vi.fn(), @@ -72,57 +81,7 @@ vi.mock("bullmq", () => ({ }, })); -import { - BullMQJobManager, - normalizeJobState, - parseDuration, -} from "./jobManager.js"; - -describe("parseDuration", () => { - test.each([ - ["500ms", 500], - ["30s", 30_000], - ["5m", 300_000], - ["6h", 21_600_000], - ["1d", 86_400_000], - ])("parses %s", (input, expected) => { - expect(parseDuration(input)).toBe(expected); - }); - - test("trims surrounding whitespace", () => { - expect(parseDuration(" 10m ")).toBe(600_000); - }); - - test.each(["", "5", "m", "5x", "1.5h", "-5m", "5 m"])( - 'throws on malformed "%s"', - (input) => { - expect(() => parseDuration(input)).toThrow(); - }, - ); -}); - -describe("normalizeJobState", () => { - test.each([ - "waiting", - "active", - "delayed", - "completed", - "failed", - "paused", - ])('passes through "%s"', (state) => { - expect(normalizeJobState(state)).toBe(state); - }); - - test("collapses prioritized and waiting-children to waiting", () => { - expect(normalizeJobState("prioritized")).toBe("waiting"); - expect(normalizeJobState("waiting-children")).toBe("waiting"); - }); - - test("maps anything unrecognized to unknown", () => { - expect(normalizeJobState("something-else")).toBe("unknown"); - expect(normalizeJobState("unknown")).toBe("unknown"); - }); -}); +import { BullMQJobManager } from "./jobManager.js"; const createWorkload = ( overrides: Partial> = {}, @@ -133,7 +92,10 @@ const createWorkload = ( jobOptions: { attempts: 2, backoff: { type: "exponential", delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, + keepJobs: { + completed: { count: 50 }, + failed: { count: 50 }, + }, keepLogs: 500, }, }, @@ -142,7 +104,7 @@ const createWorkload = ( ...overrides, }); -const data = { connectionId: 42, orgId: 1 }; +const data = { connectionId: 42 }; const job = { id: "job-1", queueName: "connection-sync", @@ -158,6 +120,11 @@ describe("BullMQJobManager lifecycle", () => { vi.clearAllMocks(); mocks.workers.length = 0; mocks.enqueue.mockResolvedValue("job-1"); + mocks.upsertJobScheduler.mockResolvedValue("scheduled-job-1"); + mocks.getJobSchedulerIds.mockResolvedValue([]); + mocks.removeJobScheduler.mockResolvedValue(true); + mocks.workerClose.mockResolvedValue(undefined); + mocks.producerClose.mockResolvedValue(undefined); }); test("delegates enqueueing to BullMQClient and returns its job id", async () => { @@ -165,10 +132,93 @@ describe("BullMQJobManager lifecycle", () => { const workload = createWorkload(); manager.register(workload); - const result = await manager.trigger("connection-sync", data); + const result = await manager.trigger( + "connection-sync", + data, + { priority: 1 }, + ); expect(result).toBe("job-1"); - expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data); + expect(mocks.enqueue).toHaveBeenCalledWith( + workload.queueSpec, + data, + { priority: 1 }, + ); + }); + + test("manages schedulers through the registered workload", async () => { + const manager = new BullMQJobManager({} as Redis); + const workload = createWorkload(); + manager.register(workload); + + await expect( + manager.upsertJobScheduler( + "connection-sync", + "connection-sync-v1-42", + 60_000, + data, + { priority: 10 }, + ), + ).resolves.toBe("scheduled-job-1"); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + workload.queueSpec, + "connection-sync-v1-42", + 60_000, + data, + { priority: 10 }, + ); + + mocks.getJobSchedulerIds.mockResolvedValue(["connection-sync-v1-42"]); + await expect( + manager.getJobSchedulerIds("connection-sync"), + ).resolves.toEqual(["connection-sync-v1-42"]); + expect(mocks.getJobSchedulerIds).toHaveBeenCalledWith( + workload.queueSpec, + ); + + await expect( + manager.removeJobScheduler( + "connection-sync", + "connection-sync-v1-42", + ), + ).resolves.toBe(true); + expect(mocks.removeJobScheduler).toHaveBeenCalledWith( + workload.queueSpec, + "connection-sync-v1-42", + ); + }); + + test("upserts a declared workload schedule when starting", async () => { + const manager = new BullMQJobManager({} as Redis); + const workload = createWorkload({ + schedule: { + interval: "5m", + data, + options: { priority: 10 }, + }, + }); + manager.register(workload); + + await manager.start(); + + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + workload.queueSpec, + "schedule:connection-sync", + "5m", + data, + { priority: 10 }, + ); + }); + + test("closes workers and producer queues when stopping", async () => { + const manager = new BullMQJobManager({} as Redis); + manager.register(createWorkload()); + await manager.start(); + + await manager.stop(); + + expect(mocks.workerClose).toHaveBeenCalledOnce(); + expect(mocks.producerClose).toHaveBeenCalledOnce(); }); test("calls onStarted before processing and onCompleted after completion", async () => { diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index a617a5812..980080a2b 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -4,64 +4,22 @@ import { createBullMQJobLogger, createLogger, DataOf, + JobEnqueueOptions, JobLogSink, QueueName, + Schedule, + scheduleToMs, } from "@sourcebot/shared"; import { Job, Queue, Worker } from "bullmq"; import { Redis } from "ioredis"; import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; -import { - JobDetail, - JobManager, - Schedule, - JobLifecycleContext, - Workload, -} from "./types.js"; +import { JobLifecycleContext, Workload } from "./types.js"; +import type { JobManager } from "./types.js"; import { prisma } from "./prisma.js"; const LOG_TAG = "job-manager"; const logger = createLogger(LOG_TAG); -const DURATION_UNITS_MS: Record = { - ms: 1, - s: 1000, - m: 1000 * 60, - h: 1000 * 60 * 60, - d: 1000 * 60 * 60 * 24, -}; - -export const parseDuration = (value: string): number => { - const match = /^(\d+)(ms|s|m|h|d)$/.exec(value.trim()); - if (!match) { - throw new Error( - `Invalid duration "${value}". Expected e.g. "500ms", "30s", "5m", "6h", "1d".`, - ); - } - return Number(match[1]) * DURATION_UNITS_MS[match[2]]; -}; - -export const normalizeJobState = (state: string): JobDetail["state"] => { - switch (state) { - case "waiting": - case "active": - case "delayed": - case "completed": - case "failed": - case "paused": - return state; - case "prioritized": - case "waiting-children": - return "waiting"; - default: - return "unknown"; - } -}; - -const scheduleToRepeat = (schedule: Schedule) => - "pattern" in schedule - ? { pattern: schedule.pattern } - : { every: parseDuration(schedule.every) }; - export class BullMQJobManager implements JobManager { private readonly workloads = new Map< string, @@ -109,16 +67,45 @@ export class BullMQJobManager implements JobManager { async trigger( workloadName: TName, data: DataOf, + options?: JobEnqueueOptions, ): Promise { - const workload = this.workloads.get(workloadName) as - | Workload - | undefined; - if (!workload) { - throw new Error( - `Cannot trigger unknown workload "${workloadName}"`, - ); - } - return this.bullmqClient.enqueue(workload.queueSpec, data); + const workload = this.getWorkload(workloadName); + return this.bullmqClient.enqueue(workload.queueSpec, data, options); + } + + async upsertJobScheduler( + workloadName: TName, + schedulerId: string, + schedule: Schedule, + data: DataOf, + options?: JobEnqueueOptions, + ): Promise { + const workload = this.getWorkload(workloadName); + return this.bullmqClient.upsertJobScheduler( + workload.queueSpec, + schedulerId, + schedule, + data, + options, + ); + } + + async getJobSchedulerIds( + workloadName: TName, + ): Promise { + const workload = this.getWorkload(workloadName); + return this.bullmqClient.getJobSchedulerIds(workload.queueSpec); + } + + async removeJobScheduler( + workloadName: TName, + schedulerId: string, + ): Promise { + const workload = this.getWorkload(workloadName); + return this.bullmqClient.removeJobScheduler( + workload.queueSpec, + schedulerId, + ); } async stop(): Promise { @@ -145,8 +132,6 @@ export class BullMQJobManager implements JobManager { ): Promise { const { queueSpec: spec, concurrency, rateLimit, schedule } = workload; - const queue = this.bullmqClient.getQueue(spec); - const worker = new Worker( spec.name, async (job) => { @@ -165,7 +150,8 @@ export class BullMQJobManager implements JobManager { signal: this.abortController.signal, updateProgress: (progress) => job.updateProgress(progress), - trigger: (target, data) => this.trigger(target, data), + trigger: (target, data, options) => + this.trigger(target, data, options), }); return result; } catch (error) { @@ -186,7 +172,7 @@ export class BullMQJobManager implements JobManager { ? { limiter: { max: rateLimit.max, - duration: parseDuration(rateLimit.per), + duration: scheduleToMs(rateLimit.per), }, } : {}), @@ -207,28 +193,31 @@ export class BullMQJobManager implements JobManager { if (schedule) { // @note: jobs produced by BullMQ's scheduler bypass the deduplication check that - // `Queue.add` goes through, so a dedup key would be silently ignored here. A - // scheduled workload gets its overlap protection from `concurrency` instead: the + // `Queue.add` goes through, so a dedup key would be silently ignored here. The // next tick's job is only created once the current one goes active, so at most one // run is ever queued behind the one in flight. - await queue.upsertJobScheduler( + await this.upsertJobScheduler( + spec.name, `schedule:${spec.name}`, - scheduleToRepeat(schedule), - { - name: spec.name, - opts: { - attempts: spec.jobOptions.attempts, - removeOnComplete: { - count: spec.jobOptions.keep.completed, - }, - removeOnFail: { count: spec.jobOptions.keep.failed }, - keepLogs: spec.jobOptions.keepLogs, - }, - }, + schedule.interval, + schedule.data, + schedule.options, ); } } + private getWorkload( + workloadName: TName, + ): Workload { + const workload = this.workloads.get(workloadName) as + | Workload + | undefined; + if (!workload) { + throw new Error(`Unknown workload "${workloadName}"`); + } + return workload; + } + private async onWorkloadJobFailed( workload: Workload, job: Job | undefined, diff --git a/packages/backend/src/reconcileJobSchedulersAtStartup.test.ts b/packages/backend/src/reconcileJobSchedulersAtStartup.test.ts new file mode 100644 index 000000000..810544242 --- /dev/null +++ b/packages/backend/src/reconcileJobSchedulersAtStartup.test.ts @@ -0,0 +1,166 @@ +import type { PrismaClient } from "@sourcebot/db"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { reconcileJobSchedulersAtStartup } from "./reconcileJobSchedulersAtStartup.js"; +import type { JobManager } from "./types.js"; + +const mocks = { + accountFindMany: vi.fn(), + connectionFindMany: vi.fn(), + repoFindMany: vi.fn(), + getJobSchedulerIds: vi.fn(), + upsertJobScheduler: vi.fn(), + removeJobScheduler: vi.fn(), +}; + +const db = { + account: { + findMany: mocks.accountFindMany, + }, + connection: { + findMany: mocks.connectionFindMany, + }, + repo: { + findMany: mocks.repoFindMany, + }, +} as unknown as PrismaClient; + +const jobManager = mocks as unknown as JobManager; + +describe("reconcileJobSchedulersAtStartup", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.accountFindMany.mockResolvedValue([ + { id: "account-1" }, + { id: "account-2" }, + ]); + mocks.connectionFindMany.mockResolvedValue([{ id: 1 }, { id: 2 }]); + mocks.repoFindMany.mockResolvedValue([{ id: 42 }, { id: 84 }]); + mocks.getJobSchedulerIds.mockResolvedValue([]); + mocks.upsertJobScheduler.mockResolvedValue("scheduled-job"); + mocks.removeJobScheduler.mockResolvedValue(true); + }); + + test("reconciles connection, repository, and permission schedulers", async () => { + await reconcileJobSchedulersAtStartup({ + db, + jobManager, + permissionSyncEnabled: true, + settings: { + resyncConnectionIntervalMs: 86_400_000, + reindexIntervalMs: 3_600_000, + userDrivenPermissionSyncIntervalMs: 43_200_000, + repoDrivenPermissionSyncIntervalMs: 21_600_000, + }, + }); + + expect(mocks.connectionFindMany).toHaveBeenCalledWith({ + select: { id: true }, + }); + expect(mocks.repoFindMany).toHaveBeenCalledWith({ + select: { id: true }, + }); + expect(mocks.accountFindMany).toHaveBeenCalledWith({ + where: { + providerType: { + in: [ + "github", + "gitlab", + "bitbucket-cloud", + "bitbucket-server", + ], + }, + }, + select: { id: true }, + }); + expect(mocks.repoFindMany).toHaveBeenCalledWith({ + where: { + isPublic: false, + external_codeHostType: { + in: [ + "github", + "gitlab", + "bitbucketCloud", + "bitbucketServer", + ], + }, + connections: { + some: { + connection: { + enforcePermissions: true, + }, + }, + }, + }, + select: { id: true }, + }); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "connection-sync", + "connection-sync-v1-1", + 86_400_000, + { connectionId: 1 }, + { priority: 10 }, + ); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "repo-index", + "repo-index-v1-42", + 3_600_000, + { repoId: 42, type: "INDEX" }, + { priority: 10 }, + ); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "account-permission-sync", + "account-permission-sync-v1-account-1", + 43_200_000, + { accountId: "account-1" }, + { priority: 10 }, + ); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "repo-permission-sync", + "repo-permission-sync-v1-42", + 21_600_000, + { repoId: 42 }, + { priority: 10 }, + ); + }); + + test("removes permission schedulers when permission syncing is disabled", async () => { + mocks.getJobSchedulerIds.mockImplementation(async (workloadName) => { + if (workloadName === "account-permission-sync") { + return ["account-permission-sync-v1-account-1"]; + } + if (workloadName === "repo-permission-sync") { + return ["repo-permission-sync-v1-42"]; + } + return []; + }); + + await reconcileJobSchedulersAtStartup({ + db, + jobManager, + permissionSyncEnabled: false, + settings: { + resyncConnectionIntervalMs: 86_400_000, + reindexIntervalMs: 3_600_000, + userDrivenPermissionSyncIntervalMs: 43_200_000, + repoDrivenPermissionSyncIntervalMs: 21_600_000, + }, + }); + + expect(mocks.accountFindMany).not.toHaveBeenCalled(); + expect(mocks.repoFindMany).toHaveBeenCalledOnce(); + expect(mocks.removeJobScheduler).toHaveBeenCalledWith( + "account-permission-sync", + "account-permission-sync-v1-account-1", + ); + expect(mocks.removeJobScheduler).toHaveBeenCalledWith( + "repo-permission-sync", + "repo-permission-sync-v1-42", + ); + expect(mocks.upsertJobScheduler).not.toHaveBeenCalledWith( + expect.stringContaining("permission-sync"), + expect.anything(), + expect.anything(), + expect.anything(), + ); + }); +}); diff --git a/packages/backend/src/reconcileJobSchedulersAtStartup.ts b/packages/backend/src/reconcileJobSchedulersAtStartup.ts new file mode 100644 index 000000000..6917388af --- /dev/null +++ b/packages/backend/src/reconcileJobSchedulersAtStartup.ts @@ -0,0 +1,175 @@ +import type { PrismaClient } from "@sourcebot/db"; +import { + ACCOUNT_PERMISSION_SYNC_SCHEDULER_ID_PREFIX, + getAccountPermissionSyncSchedulerId, + JOB_PRIORITIES, +} from "@sourcebot/shared"; +import type { + DataOf, + JobEnqueueOptions, + QueueName, + Schedule, +} from "@sourcebot/shared"; +import { + ACCOUNT_PERMISSION_SYNC_WHERE, + REPO_PERMISSION_SYNC_WHERE, +} from "./ee/permissionSyncEligibility.js"; +import type { JobManager, Settings } from "./types.js"; + +interface SchedulerTarget { + schedulerId: string; + data: DataOf; +} + +interface ReconcileOptions { + jobManager: JobManager; + workloadName: TName; + schedulerIdPrefix: string; + targets: SchedulerTarget[]; + schedule: Schedule; + jobOptions?: JobEnqueueOptions; +} + +const reconcileJobSchedulers = async ({ + jobManager, + workloadName, + schedulerIdPrefix, + targets, + schedule, + jobOptions, +}: ReconcileOptions): Promise => { + const existingSchedulerIds = new Set( + await jobManager.getJobSchedulerIds(workloadName), + ); + const desiredSchedulerIds = new Set( + targets.map(({ schedulerId }) => schedulerId), + ); + + await Promise.all( + targets.map(({ schedulerId, data }) => { + if (jobOptions) { + return jobManager.upsertJobScheduler( + workloadName, + schedulerId, + schedule, + data, + jobOptions, + ); + } + return jobManager.upsertJobScheduler( + workloadName, + schedulerId, + schedule, + data, + ); + }), + ); + + const obsoleteSchedulerIds = [...existingSchedulerIds].filter( + (schedulerId) => + schedulerId.startsWith(schedulerIdPrefix) && + !desiredSchedulerIds.has(schedulerId), + ); + await Promise.all( + obsoleteSchedulerIds.map((schedulerId) => + jobManager.removeJobScheduler(workloadName, schedulerId), + ), + ); +}; + +interface Props { + db: PrismaClient; + jobManager: JobManager; + settings: Pick< + Settings, + | "reindexIntervalMs" + | "repoDrivenPermissionSyncIntervalMs" + | "resyncConnectionIntervalMs" + | "userDrivenPermissionSyncIntervalMs" + >; + permissionSyncEnabled: boolean; +} + +export const reconcileJobSchedulersAtStartup = async ({ + db, + jobManager, + settings, + permissionSyncEnabled, +}: Props): Promise => { + const [connections, repos, accountsForPermissionSync, reposForPermissionSync] = + await Promise.all([ + db.connection.findMany({ + select: { + id: true, + }, + }), + db.repo.findMany({ + select: { + id: true, + }, + }), + permissionSyncEnabled + ? db.account.findMany({ + where: ACCOUNT_PERMISSION_SYNC_WHERE, + select: { + id: true, + }, + }) + : [], + permissionSyncEnabled + ? db.repo.findMany({ + where: REPO_PERMISSION_SYNC_WHERE, + select: { + id: true, + }, + }) + : [], + ]); + + await Promise.all([ + reconcileJobSchedulers({ + jobManager, + workloadName: "connection-sync", + schedulerIdPrefix: "connection-sync-v1-", + targets: connections.map(({ id }) => ({ + schedulerId: `connection-sync-v1-${id}`, + data: { connectionId: id }, + })), + schedule: settings.resyncConnectionIntervalMs, + jobOptions: { priority: JOB_PRIORITIES.SCHEDULED }, + }), + reconcileJobSchedulers({ + jobManager, + workloadName: "repo-index", + schedulerIdPrefix: "repo-index-v1-", + targets: repos.map(({ id }) => ({ + schedulerId: `repo-index-v1-${id}`, + data: { repoId: id, type: "INDEX" }, + })), + schedule: settings.reindexIntervalMs, + jobOptions: { priority: JOB_PRIORITIES.SCHEDULED }, + }), + reconcileJobSchedulers({ + jobManager, + workloadName: "account-permission-sync", + schedulerIdPrefix: ACCOUNT_PERMISSION_SYNC_SCHEDULER_ID_PREFIX, + targets: accountsForPermissionSync.map(({ id }) => ({ + schedulerId: getAccountPermissionSyncSchedulerId(id), + data: { accountId: id }, + })), + schedule: settings.userDrivenPermissionSyncIntervalMs, + jobOptions: { priority: JOB_PRIORITIES.SCHEDULED }, + }), + reconcileJobSchedulers({ + jobManager, + workloadName: "repo-permission-sync", + schedulerIdPrefix: "repo-permission-sync-v1-", + targets: reposForPermissionSync.map(({ id }) => ({ + schedulerId: `repo-permission-sync-v1-${id}`, + data: { repoId: id }, + })), + schedule: settings.repoDrivenPermissionSyncIntervalMs, + jobOptions: { priority: JOB_PRIORITIES.SCHEDULED }, + }), + ]); +}; diff --git a/packages/backend/src/reconciliationWorkload.test.ts b/packages/backend/src/reconciliationWorkload.test.ts deleted file mode 100644 index 2391cc6ee..000000000 --- a/packages/backend/src/reconciliationWorkload.test.ts +++ /dev/null @@ -1,375 +0,0 @@ -import type { PrismaClient } from '@sourcebot/db'; -import type { JobLogger } from '@sourcebot/shared'; -import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; - -const mocks = vi.hoisted(() => ({ - env: { - PERMISSION_SYNC_ENABLED: 'true', - }, - hasEntitlement: vi.fn(), -})); - -vi.mock('@sourcebot/shared', () => ({ - env: mocks.env, - PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS: [ - 'github', - 'gitlab', - 'bitbucket-cloud', - 'bitbucket-server', - ], - PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES: [ - 'github', - 'gitlab', - 'bitbucketCloud', - 'bitbucketServer', - ], - RECONCILIATION_QUEUE: { - name: 'reconciliation', - jobOptions: { - attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - keepLogs: 500, - }, - }, -})); - -vi.mock('./entitlements.js', () => ({ - hasEntitlement: mocks.hasEntitlement, -})); - -import { createReconciliationWorkload } from './reconciliationWorkload.js'; - -const settings = { - resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, - reindexIntervalMs: 60 * 60 * 1000, - repoGarbageCollectionGracePeriodMs: 10 * 1000, - userDrivenPermissionSyncIntervalMs: 24 * 60 * 60 * 1000, - repoDrivenPermissionSyncIntervalMs: 24 * 60 * 60 * 1000, -}; - -const logger = { - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - flush: vi.fn(), -} satisfies JobLogger; - -describe('reconciliationWorkload', () => { - const connectionFindMany = vi.fn(); - const repoFindMany = vi.fn(); - const accountFindMany = vi.fn(); - const db = { - connection: { - findMany: connectionFindMany, - }, - repo: { - findMany: repoFindMany, - }, - account: { - findMany: accountFindMany, - }, - } as unknown as PrismaClient; - - beforeEach(() => { - vi.useFakeTimers(); - vi.setSystemTime(new Date('2026-07-27T12:00:00.000Z')); - connectionFindMany.mockReset().mockResolvedValue([]); - repoFindMany.mockReset().mockResolvedValue([]); - accountFindMany.mockReset().mockResolvedValue([]); - mocks.env.PERMISSION_SYNC_ENABLED = 'true'; - mocks.hasEntitlement.mockReset().mockResolvedValue(false); - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - test('runs every 10 seconds on the reconciliation queue', () => { - const workload = createReconciliationWorkload({ - db, - settings, - }); - - expect(workload.queueSpec.name).toBe('reconciliation'); - expect(workload.schedule).toEqual({ every: '10s' }); - expect(workload.concurrency).toBe(1); - }); - - test('triggers connection syncs for connections that are due', async () => { - connectionFindMany.mockResolvedValue([ - { id: 42, orgId: 1 }, - { id: 84, orgId: 2 }, - ]); - const trigger = vi.fn().mockResolvedValue('job-id'); - const workload = createReconciliationWorkload({ - db, - settings, - }); - - await workload.process({ - data: {}, - jobId: 'reconciliation-job', - attemptsMade: 0, - maxAttempts: 2, - prisma: db, - signal: new AbortController().signal, - logger, - updateProgress: vi.fn(), - trigger, - }); - - expect(connectionFindMany).toHaveBeenCalledWith({ - where: { - OR: [ - { syncedAt: null }, - { syncedAt: { lt: new Date('2026-07-26T12:00:00.000Z') } }, - ], - }, - select: { - id: true, - orgId: true, - }, - }); - expect(trigger).toHaveBeenCalledTimes(2); - expect(trigger).toHaveBeenCalledWith('connection-sync', { - connectionId: 42, - orgId: 1, - }); - expect(trigger).toHaveBeenCalledWith('connection-sync', { - connectionId: 84, - orgId: 2, - }); - }); - - test('submits all due repos for indexing', async () => { - repoFindMany - .mockResolvedValueOnce([]) - .mockResolvedValueOnce([ - { id: 1 }, - { id: 2 }, - { id: 3 }, - { id: 4 }, - ]); - const trigger = vi.fn().mockResolvedValue('job-id'); - const workload = createReconciliationWorkload({ - db, - settings, - }); - - await workload.process({ - data: {}, - jobId: 'reconciliation-job', - attemptsMade: 0, - maxAttempts: 2, - prisma: db, - signal: new AbortController().signal, - logger, - updateProgress: vi.fn(), - trigger, - }); - - expect(trigger).toHaveBeenCalledTimes(4); - for (const repoId of [1, 2, 3, 4]) { - expect(trigger).toHaveBeenCalledWith('repo-index', { - repoId, - type: 'INDEX', - }); - } - }); - - test('schedules orphaned repos for cleanup', async () => { - repoFindMany - .mockResolvedValueOnce([{ id: 42 }]) - .mockResolvedValueOnce([]); - const trigger = vi.fn().mockResolvedValue('job-id'); - const workload = createReconciliationWorkload({ - db, - settings, - }); - - await workload.process({ - data: {}, - jobId: 'reconciliation-job', - attemptsMade: 0, - maxAttempts: 2, - prisma: db, - signal: new AbortController().signal, - logger, - updateProgress: vi.fn(), - trigger, - }); - - expect(trigger).toHaveBeenCalledWith('repo-index', { - repoId: 42, - type: 'CLEANUP', - }); - }); - - test('submits due accounts for permission syncing when entitled', async () => { - mocks.hasEntitlement.mockResolvedValue(true); - accountFindMany.mockResolvedValue([ - { id: 'account_1' }, - { id: 'account_2' }, - ]); - const trigger = vi.fn().mockResolvedValue('job-id'); - const workload = createReconciliationWorkload({ - db, - settings, - }); - - await workload.process({ - data: {}, - jobId: 'reconciliation-job', - attemptsMade: 0, - maxAttempts: 2, - prisma: db, - signal: new AbortController().signal, - logger, - updateProgress: vi.fn(), - trigger, - }); - - expect(accountFindMany).toHaveBeenCalledWith({ - where: { - AND: [ - { - providerType: { - in: ['github', 'gitlab', 'bitbucket-cloud', 'bitbucket-server'], - }, - }, - { - OR: [ - { permissionSyncedAt: null }, - { permissionSyncedAt: { lt: new Date('2026-07-26T12:00:00.000Z') } }, - ], - }, - ], - }, - select: { - id: true, - }, - }); - expect(trigger).toHaveBeenCalledWith('account-permission-sync', { - accountId: 'account_1', - }); - expect(trigger).toHaveBeenCalledWith('account-permission-sync', { - accountId: 'account_2', - }); - }); - - test('does not query accounts without the permission syncing entitlement', async () => { - const workload = createReconciliationWorkload({ - db, - settings, - }); - - await workload.process({ - data: {}, - jobId: 'reconciliation-job', - attemptsMade: 0, - maxAttempts: 2, - prisma: db, - signal: new AbortController().signal, - logger, - updateProgress: vi.fn(), - trigger: vi.fn(), - }); - - expect(accountFindMany).not.toHaveBeenCalled(); - expect(repoFindMany).toHaveBeenCalledTimes(2); - }); - - test('does not query accounts when permission syncing is disabled', async () => { - mocks.env.PERMISSION_SYNC_ENABLED = 'false'; - mocks.hasEntitlement.mockResolvedValue(true); - const workload = createReconciliationWorkload({ - db, - settings, - }); - - await workload.process({ - data: {}, - jobId: 'reconciliation-job', - attemptsMade: 0, - maxAttempts: 2, - prisma: db, - signal: new AbortController().signal, - logger, - updateProgress: vi.fn(), - trigger: vi.fn(), - }); - - expect(mocks.hasEntitlement).not.toHaveBeenCalled(); - expect(accountFindMany).not.toHaveBeenCalled(); - expect(repoFindMany).toHaveBeenCalledTimes(2); - }); - - test('submits due private repos for permission syncing when entitled', async () => { - mocks.hasEntitlement.mockResolvedValue(true); - repoFindMany - .mockResolvedValueOnce([]) - .mockResolvedValueOnce([]) - .mockResolvedValueOnce([ - { id: 42 }, - { id: 84 }, - ]); - const trigger = vi.fn().mockResolvedValue('job-id'); - const workload = createReconciliationWorkload({ - db, - settings, - }); - - await workload.process({ - data: {}, - jobId: 'reconciliation-job', - attemptsMade: 0, - maxAttempts: 2, - prisma: db, - signal: new AbortController().signal, - logger, - updateProgress: vi.fn(), - trigger, - }); - - expect(repoFindMany).toHaveBeenNthCalledWith(3, { - where: { - AND: [ - { - isPublic: false, - }, - { - external_codeHostType: { - in: ['github', 'gitlab', 'bitbucketCloud', 'bitbucketServer'], - }, - }, - { - connections: { - some: { - connection: { - enforcePermissions: true, - }, - }, - }, - }, - { - OR: [ - { permissionSyncedAt: null }, - { permissionSyncedAt: { lt: new Date('2026-07-26T12:00:00.000Z') } }, - ], - }, - ], - }, - select: { - id: true, - }, - }); - expect(trigger).toHaveBeenCalledWith('repo-permission-sync', { - repoId: 42, - }); - expect(trigger).toHaveBeenCalledWith('repo-permission-sync', { - repoId: 84, - }); - }); -}); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts deleted file mode 100644 index fa2f29cfa..000000000 --- a/packages/backend/src/reconciliationWorkload.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { PrismaClient } from "@sourcebot/db"; -import { env, PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS, RECONCILIATION_QUEUE } from "@sourcebot/shared"; -import { hasEntitlement } from "./entitlements.js"; -import { Settings, Workload } from "./types.js"; - -interface ReconciliationWorkloadDependencies { - db: PrismaClient; - settings: Settings; -} - -export const createReconciliationWorkload = ({ - db, - settings, -}: ReconciliationWorkloadDependencies): Workload<'reconciliation'> => ({ - concurrency: 1, - schedule: { every: '10s' }, - queueSpec: RECONCILIATION_QUEUE, - process: async ({ logger, trigger }) => { - // Connections - { - const connectionThreshold = new Date(Date.now() - settings.resyncConnectionIntervalMs); - const connections = await db.connection.findMany({ - where: { - OR: [ - { syncedAt: null }, - { syncedAt: { lt: connectionThreshold } }, - ], - }, - select: { - id: true, - orgId: true, - }, - }); - - await Promise.all(connections.map(async (connection) => { - await trigger('connection-sync', { - connectionId: connection.id, - orgId: connection.orgId, - }); - })); - } - - // Repo garbage collection - { - const cleanupThreshold = new Date(Date.now() - settings.repoGarbageCollectionGracePeriodMs); - const reposToCleanup = await db.repo.findMany({ - where: { - connections: { - none: {}, - }, - isAutoCleanupDisabled: false, - OR: [ - { indexedAt: null }, - { indexedAt: { lt: cleanupThreshold } }, - ], - }, - select: { - id: true, - }, - }); - - await Promise.all(reposToCleanup.map(async ({ id }) => { - logger.debug(`Scheduling cleanup for repo ${id}`); - await trigger('repo-index', { - repoId: id, - type: 'CLEANUP', - }); - })); - } - - // Repo indexing - { - const indexThreshold = new Date(Date.now() - settings.reindexIntervalMs); - const reposToIndex = await db.repo.findMany({ - where: { - OR: [ - { indexedAt: null }, - { indexedAt: { lt: indexThreshold } }, - ], - }, - select: { - id: true, - }, - }); - - await Promise.all(reposToIndex.map(async ({ id }) => { - await trigger('repo-index', { - repoId: id, - type: 'INDEX', - }); - })); - } - - // Permission syncing - if ( - env.PERMISSION_SYNC_ENABLED === 'true' && - await hasEntitlement('permission-syncing') - ) { - const accountThreshold = new Date(Date.now() - settings.userDrivenPermissionSyncIntervalMs); - const accounts = await db.account.findMany({ - where: { - AND: [ - { - providerType: { - in: PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS, - }, - }, - { - OR: [ - { permissionSyncedAt: null }, - { permissionSyncedAt: { lt: accountThreshold } }, - ], - }, - ], - }, - select: { - id: true, - }, - }); - - await Promise.all(accounts.map(async ({ id }) => { - await trigger('account-permission-sync', { - accountId: id, - }); - })); - - const repoThreshold = new Date(Date.now() - settings.repoDrivenPermissionSyncIntervalMs); - const repos = await db.repo.findMany({ - where: { - AND: [ - { - isPublic: false, - }, - { - external_codeHostType: { - in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, - }, - }, - { - connections: { - some: { - connection: { - enforcePermissions: true, - }, - }, - }, - }, - { - OR: [ - { permissionSyncedAt: null }, - { permissionSyncedAt: { lt: repoThreshold } }, - ], - }, - ], - }, - select: { - id: true, - }, - }); - - await Promise.all(repos.map(async ({ id }) => { - await trigger('repo-permission-sync', { - repoId: id, - }); - })); - } - }, -}); diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 1cc29e147..f5537f52a 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -6,7 +6,15 @@ import { } from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type"; -import { DataOf, JobLogSink, QueueName, QueueSpec } from "@sourcebot/shared"; +import { + DataOf, + JobEnqueueOptions, + JobLogSink, + QueueName, + QueueSpec, + Schedule, +} from "@sourcebot/shared"; +import type { Queue } from "bullmq"; export type Settings = Required; @@ -45,39 +53,28 @@ export interface ProcessContext extends JobLifecycleContext { signal: AbortSignal; updateProgress(progress: number | object): Promise; - trigger(workload: T, data: DataOf): Promise; + trigger( + workload: T, + data: DataOf, + options?: JobEnqueueOptions, + ): Promise; } -export type Schedule = { every: string } | { pattern: string }; - -/** - * A Workload is a single kind of background work, declared - * as the queue it runs on, the code that processes the job, - * and how much of it may run at once. - * - * Jobs reach a workload's queue in one of two ways: someone calls `trigger`, or - if the - * workload declares a `schedule` - the JobManager enqueues one on that cadence. A sweep is - * just a scheduled workload that carries no payload, and whose `process` scans for work and - * triggers it onto other workloads' queues. - */ export interface Workload { queueSpec: QueueSpec; concurrency: number; - /** - * If set, the JobManager enqueues a job on this cadence rather than waiting for someone to - * `trigger` one. Scheduled jobs carry no payload, so `TData` should be `void`. - */ - schedule?: Schedule; + schedule?: { + interval: Schedule; + data: DataOf; + options?: JobEnqueueOptions; + }; rateLimit?: { max: number; per: string }; process(ctx: ProcessContext): Promise; - /** Called before `process` on every attempt. */ onStarted?(ctx: JobLifecycleContext): Promise; - /** Called after BullMQ marks the job as completed. */ onCompleted?( ctx: JobLifecycleContext, result: TResult, ): Promise; - /** Called after BullMQ exhausts all attempts and marks the job as failed. */ onTerminalFailure?( ctx: JobLifecycleContext, err: Error, @@ -85,49 +82,27 @@ export interface Workload { } export interface JobManager { - register(w: Workload): void; - + register(workload: Workload): void; + getQueues(): Queue[]; start(): Promise; stop(): Promise; - trigger( - workload: TName, + workloadName: TName, data: DataOf, + options?: JobEnqueueOptions, ): Promise; -} - -export interface QueueCounts { - waiting: number; - active: number; - delayed: number; - completed: number; - failed: number; - paused: number; - prioritized?: number; - "waiting-children"?: number; -} - -export interface JobDetail { - id: string; - name: string; - state: - | "waiting" - | "active" - | "delayed" - | "completed" - | "failed" - | "paused" - | "unknown"; - data: TData; - attemptsMade: number; - maxAttempts: number; - result?: TResult | null; - failedReason?: string | null; - stacktrace?: string[]; - logs: string[]; - enqueuedAt: number; - startedAt: number | null; - finishedAt: number | null; - waitMs?: number | null; - runMs?: number | null; + upsertJobScheduler( + workloadName: TName, + schedulerId: string, + schedule: Schedule, + data: DataOf, + options?: JobEnqueueOptions, + ): Promise; + getJobSchedulerIds( + workloadName: TName, + ): Promise; + removeJobScheduler( + workloadName: TName, + schedulerId: string, + ): Promise; } diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts new file mode 100644 index 000000000..84e5efcb0 --- /dev/null +++ b/packages/shared/src/bullmqClient.test.ts @@ -0,0 +1,135 @@ +import { Redis } from "ioredis"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + add: vi.fn(async () => ({ id: "job-1" })), + upsertJobScheduler: vi.fn(async () => ({ id: "scheduled-job" })), + getJobSchedulers: vi.fn(async () => [ + { key: "scheduler-1" }, + { key: "scheduler-2" }, + ]), + removeJobScheduler: vi.fn(async () => true), +})); + +vi.mock("bullmq", () => ({ + Queue: class { + add = mocks.add; + upsertJobScheduler = mocks.upsertJobScheduler; + getJobSchedulers = mocks.getJobSchedulers; + removeJobScheduler = mocks.removeJobScheduler; + }, +})); + +vi.mock("./jobLogger.js", () => ({ + DEFAULT_JOB_LOGS_MAX_ENTRIES: 500, + readBullMQJobLogs: vi.fn(), +})); + +import { BullMQClient } from "./bullmqClient.js"; +import { CONNECTION_QUEUE } from "./queue.js"; + +describe("BullMQClient", () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.clearAllMocks(); + vi.spyOn(Date, "now").mockReturnValue(1_000_000); + }); + + test("includes workload data in scheduled jobs", async () => { + const client = new BullMQClient({} as Redis); + const data = { connectionId: 42 }; + + await client.upsertJobScheduler( + CONNECTION_QUEUE, + "schedule:42", + 1_000, + data, + ); + + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "schedule:42", + { every: 1_000, startDate: 1_001_000 }, + expect.objectContaining({ + name: "connection-sync", + data, + opts: { + attempts: 2, + backoff: { type: "exponential", delay: 5000 }, + removeOnComplete: { age: 1_209_600 }, + removeOnFail: { age: 1_209_600 }, + keepLogs: 500, + }, + }), + ); + }); + + test("adds enqueue priority to immediate jobs", async () => { + const client = new BullMQClient({} as Redis); + + await client.enqueue( + CONNECTION_QUEUE, + { connectionId: 42 }, + { priority: 1 }, + ); + + expect(mocks.add).toHaveBeenCalledWith( + "connection-sync", + { connectionId: 42 }, + expect.objectContaining({ priority: 1 }), + ); + }); + + test("adds enqueue priority to scheduled jobs", async () => { + const client = new BullMQClient({} as Redis); + + await client.upsertJobScheduler( + CONNECTION_QUEUE, + "schedule:42", + 1_000, + { connectionId: 42 }, + { priority: 10 }, + ); + + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "schedule:42", + expect.any(Object), + expect.objectContaining({ + opts: expect.objectContaining({ priority: 10 }), + }), + ); + }); + + test("parses string schedules and delays the first run by one interval", async () => { + const client = new BullMQClient({} as Redis); + + await client.upsertJobScheduler( + CONNECTION_QUEUE, + "schedule:42", + "5m", + { connectionId: 42 }, + ); + + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "schedule:42", + { every: 300_000, startDate: 1_300_000 }, + expect.any(Object), + ); + }); + + test("lists scheduler ids", async () => { + const client = new BullMQClient({} as Redis); + + await expect( + client.getJobSchedulerIds(CONNECTION_QUEUE), + ).resolves.toEqual(["scheduler-1", "scheduler-2"]); + }); + + test("removes a scheduler by id", async () => { + const client = new BullMQClient({} as Redis); + + await expect( + client.removeJobScheduler(CONNECTION_QUEUE, "scheduler-1"), + ).resolves.toBe(true); + expect(mocks.removeJobScheduler).toHaveBeenCalledWith("scheduler-1"); + }); +}); diff --git a/packages/shared/src/bullmqClient.ts b/packages/shared/src/bullmqClient.ts index 5e5e2e43a..bc960d76b 100644 --- a/packages/shared/src/bullmqClient.ts +++ b/packages/shared/src/bullmqClient.ts @@ -1,11 +1,22 @@ import { Queue } from "bullmq"; import { randomUUID } from "crypto"; import { Redis } from "ioredis"; -import { DataOf, QueueName, QueueSpec } from "./queue.js"; +import type { + DataOf, + JobEnqueueOptions, + QueueName, + QueueSpec, +} from "./queue.js"; +import { scheduleToMs } from "./schedule.js"; +import type { Schedule } from "./schedule.js"; import { readBullMQJobLogs } from "./jobLogger.js"; import type { GetJobLogsOptions, JobLogs } from "./jobLogger.js"; -export type WorkloadJobStatus = "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED"; +export type WorkloadJobStatus = + | "PENDING" + | "IN_PROGRESS" + | "COMPLETED" + | "FAILED"; export interface WorkloadJob { id: string; @@ -45,11 +56,11 @@ const normalizeJobState = (state: string): WorkloadJobStatus | null => { export class BullMQClient { private readonly queues = new Map(); - constructor( - private readonly connection: Redis, - ) {} + constructor(private readonly connection: Redis) {} - getQueue(spec: QueueSpec): WorkloadQueue { + getQueue( + spec: QueueSpec, + ): WorkloadQueue { const queueName = spec.name; let queue = this.queues.get(queueName); if (!queue) { @@ -91,7 +102,8 @@ export class BullMQClient { async enqueue( spec: QueueSpec, - data: DataOf + data: DataOf, + options: JobEnqueueOptions = {}, ): Promise { const dedupKey = spec.dedupKey?.(data); const queue = this.getQueue(spec); @@ -100,21 +112,91 @@ export class BullMQClient { const job = await queue.add(spec.name, data, { jobId: requestedJobId, ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), + ...(options.priority !== undefined + ? { priority: options.priority } + : {}), attempts: spec.jobOptions.attempts, - backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, - removeOnComplete: { count: spec.jobOptions.keep.completed }, - removeOnFail: { count: spec.jobOptions.keep.failed }, + backoff: { + type: spec.jobOptions.backoff.type, + delay: spec.jobOptions.backoff.delayMs, + }, + removeOnComplete: spec.jobOptions.keepJobs.completed, + removeOnFail: spec.jobOptions.keepJobs.failed, keepLogs: spec.jobOptions.keepLogs, }); if (!job.id) { - throw new Error(`BullMQ did not return an id for workload "${spec.name}"`); + throw new Error( + `BullMQ did not return an id for workload "${spec.name}"`, + ); } return job.id; } + async upsertJobScheduler( + spec: QueueSpec, + schedulerId: string, + schedule: Schedule, + data: DataOf, + options: JobEnqueueOptions = {}, + ): Promise { + const queue = this.getQueue(spec); + const intervalMs = scheduleToMs(schedule); + + // @note: jobs produced by BullMQ's scheduler bypass the deduplication check that + // `Queue.add` goes through, so a dedup key would be silently ignored here. + const job = await queue.upsertJobScheduler( + schedulerId, + { + every: intervalMs, + startDate: Date.now() + intervalMs, + }, + { + name: spec.name, + data, + opts: { + ...(options.priority !== undefined + ? { priority: options.priority } + : {}), + attempts: spec.jobOptions.attempts, + backoff: { + type: spec.jobOptions.backoff.type, + delay: spec.jobOptions.backoff.delayMs, + }, + removeOnComplete: spec.jobOptions.keepJobs.completed, + removeOnFail: spec.jobOptions.keepJobs.failed, + keepLogs: spec.jobOptions.keepLogs, + }, + }, + ); + + if (!job.id) { + throw new Error( + `BullMQ did not return an id for workload "${spec.name}"`, + ); + } + + return job.id; + } + + async getJobSchedulerIds( + spec: QueueSpec, + ): Promise { + const schedulers = await this.getQueue(spec).getJobSchedulers(); + return schedulers.map(({ key }) => key); + } + + removeJobScheduler( + spec: QueueSpec, + schedulerId: string, + ): Promise { + return this.getQueue(spec).removeJobScheduler(schedulerId); + } + async close(): Promise { - await Promise.all([...this.queues.values()].map((queue) => queue.close())); + await Promise.all( + [...this.queues.values()].map((queue) => queue.close()), + ); } } diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index d9fe61e1c..9eb86c7a0 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -31,6 +31,7 @@ export { base64Decode, loadJsonFile, getConfigSettings, + resolveConfigSettings, getRepoPath, getRepoIdFromPath, isCredentialsLoginEnabled, @@ -95,15 +96,24 @@ export type { Version } from "./versionUtils.js"; export type { QueueName, DataOf, + JobEnqueueOptions, QueueSpec, -} from "./queue.js" + JobOptions, +} from "./queue.js"; export { ACCOUNT_PERMISSION_SYNC_QUEUE, CONNECTION_QUEUE, - RECONCILIATION_QUEUE, + DEFAULT_JOB_OPTIONS, + JOB_PRIORITIES, REPO_INDEX_QUEUE, REPO_PERMISSION_SYNC_QUEUE, } from "./queue.js"; +export type { Schedule } from "./schedule.js"; +export { + ACCOUNT_PERMISSION_SYNC_SCHEDULER_ID_PREFIX, + getAccountPermissionSyncSchedulerId, + scheduleToMs, +} from "./schedule.js"; export { BullMQClient, } from "./bullmqClient.js"; diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index d7b8d0271..e688a342f 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -1,14 +1,49 @@ +import type { KeepJobs } from "bullmq"; import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from "./jobLogger.js"; +export interface QueueSpec { + name: TName; + dedupKey?(data: DataOf): string; + jobOptions: JobOptions; +} + +export type JobOptions = { + attempts: number; + backoff: { type: "fixed" | "exponential"; delayMs: number }; + keepJobs: { + completed: KeepJobs; + failed: KeepJobs; + }; + keepLogs: number; +}; + +export type JobEnqueueOptions = { + priority?: number; +}; + +export const JOB_PRIORITIES = { + INTERACTIVE: 1, + SCHEDULED: 10, +} as const; + +const TWO_WEEKS_IN_SECONDS = 14 * 24 * 60 * 60; + +export const DEFAULT_JOB_OPTIONS: JobOptions = { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keepJobs: { + completed: { age: TWO_WEEKS_IN_SECONDS }, + failed: { age: TWO_WEEKS_IN_SECONDS }, + }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, +}; + export type QueueName = keyof QueueRegistry; export type DataOf = QueueRegistry[TName]; -type EmptyJobData = Record; interface QueueRegistry { - reconciliation: EmptyJobData; "connection-sync": { connectionId: number; - orgId: number; }; "repo-index": { repoId: number; @@ -22,68 +57,27 @@ interface QueueRegistry { }; } -export const RECONCILIATION_QUEUE: QueueSpec<"reconciliation"> = { - name: "reconciliation", - jobOptions: { - attempts: 2, - backoff: { type: "exponential", delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, - }, -}; - export const CONNECTION_QUEUE: QueueSpec<"connection-sync"> = { name: "connection-sync", - jobOptions: { - attempts: 2, - backoff: { type: "exponential", delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, - }, + jobOptions: DEFAULT_JOB_OPTIONS, dedupKey: (data) => `connection:${data.connectionId}`, }; export const REPO_INDEX_QUEUE: QueueSpec<"repo-index"> = { name: "repo-index", - jobOptions: { - attempts: 2, - backoff: { type: "exponential", delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, - }, + jobOptions: DEFAULT_JOB_OPTIONS, dedupKey: (data) => `repo:${data.repoId}`, }; export const ACCOUNT_PERMISSION_SYNC_QUEUE: QueueSpec<"account-permission-sync"> = { name: "account-permission-sync", - jobOptions: { - attempts: 2, - backoff: { type: "exponential", delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, - }, + jobOptions: DEFAULT_JOB_OPTIONS, dedupKey: (data) => `account:${data.accountId}`, }; export const REPO_PERMISSION_SYNC_QUEUE: QueueSpec<"repo-permission-sync"> = { name: "repo-permission-sync", - jobOptions: { - attempts: 2, - backoff: { type: "exponential", delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, - }, + jobOptions: DEFAULT_JOB_OPTIONS, dedupKey: (data) => `repo:${data.repoId}`, }; - -export interface QueueSpec { - name: TName; - dedupKey?(data: DataOf): string; - jobOptions: { - attempts: number; - backoff: { type: "fixed" | "exponential"; delayMs: number }; - keep: { completed: number; failed: number }; - keepLogs: number; - }; -} diff --git a/packages/shared/src/redis.ts b/packages/shared/src/redis.ts index c016aa966..49e678f2a 100644 --- a/packages/shared/src/redis.ts +++ b/packages/shared/src/redis.ts @@ -1,5 +1,5 @@ import fs from "fs"; -import { Redis } from "ioredis"; +import { Redis, type RedisOptions } from "ioredis"; import { env } from "./env.server.js"; const buildTlsOptions = (): Record => { @@ -43,7 +43,9 @@ const buildTlsOptions = (): Record => { }; }; -export const createRedisClient = () => new Redis(env.REDIS_URL, { - maxRetriesPerRequest: null, - ...buildTlsOptions(), -}); +export const createRedisClient = (options: RedisOptions = {}) => + new Redis(env.REDIS_URL, { + maxRetriesPerRequest: null, + ...buildTlsOptions(), + ...options, + }); diff --git a/packages/shared/src/schedule.test.ts b/packages/shared/src/schedule.test.ts new file mode 100644 index 000000000..3aa8139c6 --- /dev/null +++ b/packages/shared/src/schedule.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "vitest"; +import { + getAccountPermissionSyncSchedulerId, + scheduleToMs, +} from "./schedule.js"; + +test("builds an account permission sync scheduler ID", () => { + expect(getAccountPermissionSyncSchedulerId("account-1")).toBe( + "account-permission-sync-v1-account-1", + ); +}); + +describe("scheduleToMs", () => { + test.each([ + [500, 500], + ["500ms", 500], + ["30s", 30_000], + ["5m", 300_000], + ["6h", 21_600_000], + ["1d", 86_400_000], + [" 10m ", 600_000], + ])("converts %s to milliseconds", (schedule, expected) => { + expect(scheduleToMs(schedule)).toBe(expected); + }); + + test.each([ + "", + "0s", + "5", + "m", + "5x", + "1.5h", + "-5m", + "5 m", + 0, + -1, + Number.NaN, + Number.POSITIVE_INFINITY, + ])('rejects invalid schedule "%s"', (schedule) => { + expect(() => scheduleToMs(schedule)).toThrow(); + }); +}); diff --git a/packages/shared/src/schedule.ts b/packages/shared/src/schedule.ts new file mode 100644 index 000000000..29f1df6a9 --- /dev/null +++ b/packages/shared/src/schedule.ts @@ -0,0 +1,43 @@ +export type Schedule = string | number; + +export const ACCOUNT_PERMISSION_SYNC_SCHEDULER_ID_PREFIX = + "account-permission-sync-v1-"; + +export const getAccountPermissionSyncSchedulerId = ( + accountId: string, +): string => `${ACCOUNT_PERMISSION_SYNC_SCHEDULER_ID_PREFIX}${accountId}`; + +const SCHEDULE_UNITS_MS: Record = { + ms: 1, + s: 1000, + m: 1000 * 60, + h: 1000 * 60 * 60, + d: 1000 * 60 * 60 * 24, +}; + +export const scheduleToMs = (schedule: Schedule): number => { + if (typeof schedule === "number") { + if (!Number.isFinite(schedule) || schedule <= 0) { + throw new Error( + `Invalid schedule "${schedule}". Expected a positive number of milliseconds.`, + ); + } + return schedule; + } + + const match = /^(\d+)(ms|s|m|h|d)$/.exec(schedule.trim()); + if (!match) { + throw new Error( + `Invalid schedule "${schedule}". Expected e.g. "500ms", "30s", "5m", "6h", or "1d".`, + ); + } + + const intervalMs = Number(match[1]) * SCHEDULE_UNITS_MS[match[2]]; + if (!Number.isFinite(intervalMs) || intervalMs <= 0) { + throw new Error( + `Invalid schedule "${schedule}". Expected a positive finite interval.`, + ); + } + + return intervalMs; +}; diff --git a/packages/shared/src/utils.test.ts b/packages/shared/src/utils.test.ts index c346c2c5a..14e1de4e8 100644 --- a/packages/shared/src/utils.test.ts +++ b/packages/shared/src/utils.test.ts @@ -1,7 +1,7 @@ import { readFile } from 'fs/promises'; import { beforeEach, describe, expect, test, vi } from 'vitest'; import { DEFAULT_CONFIG_SETTINGS } from './constants.js'; -import { getConfigSettings } from './utils.js'; +import { getConfigSettings, resolveConfigSettings } from './utils.js'; // Mock fs/promises so loadConfig doesn't hit the filesystem. // The config schema has no required fields, so '{}' is valid. @@ -95,3 +95,18 @@ describe('getConfigSettings', () => { }); }); }); + +describe('resolveConfigSettings', () => { + test('resolves settings from an already-loaded config', () => { + const result = resolveConfigSettings({ + settings: { + resyncConnectionIntervalMs: 12_345, + }, + }); + + expect(result.resyncConnectionIntervalMs).toBe(12_345); + expect(result.reindexIntervalMs).toBe( + DEFAULT_CONFIG_SETTINGS.reindexIntervalMs, + ); + }); +}); diff --git a/packages/shared/src/utils.ts b/packages/shared/src/utils.ts index 848b941eb..2a1867fc4 100644 --- a/packages/shared/src/utils.ts +++ b/packages/shared/src/utils.ts @@ -4,9 +4,9 @@ import { z } from "zod"; import { DEFAULT_CONFIG_SETTINGS } from "./constants.js"; import { ConfigSettings } from "./types.js"; import { Org, Repo } from "@sourcebot/db"; +import type { SourcebotConfig } from "@sourcebot/schemas/v3/index.type"; import path from "path"; import { env, isRemotePath, loadConfig } from "./env.server.js"; -import { isAnonymousAccessAvailable } from './entitlements.js'; // From https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem export const base64Decode = (base64: string): string => { @@ -75,14 +75,8 @@ export const loadJsonFile = async ( } -export const getConfigSettings = async (configPath?: string): Promise => { - if (!configPath) { - return DEFAULT_CONFIG_SETTINGS; - } - - const config = await loadConfig(configPath); - - return { +export const resolveConfigSettings = (config: SourcebotConfig): ConfigSettings => + ({ ...DEFAULT_CONFIG_SETTINGS, ...config.settings, // Fall back to deprecated experiment_ variants if new keys are not set. @@ -94,7 +88,14 @@ export const getConfigSettings = async (configPath?: string): Promise => { + if (!configPath) { + return DEFAULT_CONFIG_SETTINGS; } + + return resolveConfigSettings(await loadConfig(configPath)); } export const getRepoIdFromPath = (repoPath: string): number | undefined => { @@ -143,4 +144,4 @@ export const isMemberApprovalRequired = (org: Org): boolean => { } return org.memberApprovalRequired; -} \ No newline at end of file +} diff --git a/packages/shared/vitest.config.ts b/packages/shared/vitest.config.ts index 35f8c7016..30085023a 100644 --- a/packages/shared/vitest.config.ts +++ b/packages/shared/vitest.config.ts @@ -2,6 +2,7 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { + include: ['src/**/*.test.ts'], environment: 'node', watch: false, env: { diff --git a/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx b/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx index 4a0049892..878ba3f2b 100644 --- a/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx +++ b/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx @@ -6,7 +6,7 @@ import { getCodeHostInfoForRepo, isServiceError } from "@/lib/utils" import { ExternalLink, MoreHorizontal } from "lucide-react" import Link from "next/link" import { useState } from "react" -import { indexRepo } from "@/features/workerApi/actions" +import { indexRepo } from "@/features/repos/actions" import { useRouter } from "next/navigation" import { useToast } from "@/components/hooks/use-toast" import type { Repo } from "./reposTable" @@ -34,7 +34,7 @@ export const RepoActionsDropdown = ({ repo }: RepoActionsDropdownProps) => { if (!isServiceError(response)) { const { jobId } = response toast({ - description: `✅ Repository sync triggered successfully. Job ID: ${jobId}`, + description: `✅ Repository indexing scheduled. Job ID: ${jobId}`, }) router.refresh() } else { diff --git a/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx b/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx index a108d1b18..7f82ff796 100644 --- a/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx +++ b/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx @@ -27,7 +27,7 @@ import { useRouter } from "next/navigation" import { useToast } from "@/components/hooks/use-toast" import { DisplayDate } from "../../components/DisplayDate" import { LoadingButton } from "@/components/ui/loading-button" -import { indexRepo } from "@/features/workerApi/actions" +import { indexRepo } from "@/features/repos/actions" import { isServiceError } from "@/lib/utils" // @see: https://v0.app/chat/repo-indexing-status-uhjdDim8OUS @@ -202,7 +202,7 @@ export const RepoJobsTable = ({ if (!isServiceError(response)) { const { jobId } = response; toast({ - description: `✅ Repository sync triggered successfully. Job ID: ${jobId}`, + description: `✅ Repository indexing scheduled. Job ID: ${jobId}`, }) router.refresh(); } else { diff --git a/packages/web/src/app/(app)/repos/components/reposTable.tsx b/packages/web/src/app/(app)/repos/components/reposTable.tsx index cbbc51676..865f19402 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.tsx @@ -26,7 +26,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip import { NotificationDot } from "../../components/notificationDot" import { CodeHostType } from "@sourcebot/db" import { useHotkeys } from "react-hotkeys-hook" -import { indexRepo } from "@/features/workerApi/actions" +import { indexRepo } from "@/features/repos/actions" import { RepoActionsDropdown } from "./repoActionsDropdown" // @see: https://v0.app/chat/repo-indexing-status-uhjdDim8OUS @@ -352,7 +352,7 @@ export const ReposTable = ({ if (!isServiceError(response)) { const { jobId } = response; toast({ - description: `✅ Repository sync triggered successfully. Job ID: ${jobId}`, + description: `✅ Repository indexing scheduled. Job ID: ${jobId}`, }); router.refresh(); } else { diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx index 9277d81b7..0600a5c37 100644 --- a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx +++ b/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx @@ -27,7 +27,7 @@ import { useRouter } from "next/navigation" import { useToast } from "@/components/hooks/use-toast" import { DisplayDate } from "@/app/(app)/components/DisplayDate" import { LoadingButton } from "@/components/ui/loading-button" -import { syncConnection } from "@/features/workerApi/actions" +import { syncConnection } from "@/features/connections/actions" import { isServiceError } from "@/lib/utils" @@ -199,7 +199,7 @@ export const ConnectionJobsTable = ({ data, connectionId }: { data: ConnectionSy if (!isServiceError(response)) { const { jobId } = response; toast({ - description: `✅ Connection synced successfully. Job ID: ${jobId}`, + description: `✅ Connection sync scheduled. Job ID: ${jobId}`, }) router.refresh(); } else { diff --git a/packages/web/src/auth.ts b/packages/web/src/auth.ts index 8f1932c0c..b6f132b9a 100644 --- a/packages/web/src/auth.ts +++ b/packages/web/src/auth.ts @@ -4,7 +4,7 @@ import NextAuth, { DefaultSession, Session, User as AuthJsUser } from "next-auth import Credentials from "next-auth/providers/credentials" import EmailProvider from "next-auth/providers/nodemailer"; import { __unsafePrisma } from "@/prisma"; -import { createLogger, env, getSMTPConnectionURL } from "@sourcebot/shared"; +import { createLogger, doesIdpSupportPermissionSyncing, env, getSMTPConnectionURL } from "@sourcebot/shared"; import { User } from '@sourcebot/db'; import 'next-auth/jwt'; import type { Provider } from "next-auth/providers"; @@ -23,7 +23,7 @@ import { captureEvent } from '@/lib/posthog'; import { isEmailCodeLoginEnabled, isCredentialsLoginEnabled } from '@sourcebot/shared' import { onCreateUser } from './features/membership/onCreateUser'; import { setSentryUser } from './lib/sentryUser'; -import { requestAccountPermissionSync } from './features/workerApi/client.server'; +import { scheduleAndTriggerAccountPermissionSync } from './ee/features/permissionSync/accountPermissionSyncQueue.server'; export const runtime = 'nodejs'; const logger = createLogger('auth'); @@ -225,17 +225,24 @@ const nextAuthResult = NextAuth(async () => ({ }) }); + // Once the latest OAuth credentials and issuer URL are persisted, + // ensure a recurring permission-sync schedule exists and trigger a + // sync for new accounts or accounts recovering from a previous issue. if ( - updatedAccount.permissionSyncIssue !== null && - env.PERMISSION_SYNC_ENABLED === 'true' + env.PERMISSION_SYNC_ENABLED === 'true' && + doesIdpSupportPermissionSyncing(updatedAccount.providerType) && + ( + updatedAccount.permissionSyncedAt === null || + updatedAccount.permissionSyncIssue !== null + ) ) { try { if (await hasEntitlement('permission-syncing')) { - await requestAccountPermissionSync(updatedAccount.id); + await scheduleAndTriggerAccountPermissionSync(updatedAccount.id); } } catch (error) { const message = error instanceof Error ? error.message : String(error); - logger.error(`Failed to schedule permission sync after reauthentication for account ${updatedAccount.id}: ${message}`); + logger.error(`Failed to schedule permission sync after authentication for account ${updatedAccount.id}: ${message}`); } } } diff --git a/packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.test.ts b/packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.test.ts new file mode 100644 index 000000000..593b5d3b2 --- /dev/null +++ b/packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.test.ts @@ -0,0 +1,85 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + enqueue: vi.fn(), + getConfigSettings: vi.fn(), + removeJobScheduler: vi.fn(), + upsertJobScheduler: vi.fn(), +})); + +const accountPermissionSyncQueue = { + name: 'account-permission-sync', +}; + +vi.mock('server-only', () => ({})); +vi.mock('@/lib/bullmqClient', () => ({ + getBullMQClient: () => ({ + enqueue: mocks.enqueue, + removeJobScheduler: mocks.removeJobScheduler, + upsertJobScheduler: mocks.upsertJobScheduler, + }), +})); +vi.mock('@sourcebot/shared', () => ({ + ACCOUNT_PERMISSION_SYNC_QUEUE: accountPermissionSyncQueue, + env: { CONFIG_PATH: '/config.json' }, + getAccountPermissionSyncSchedulerId: (accountId: string) => + `account-permission-sync-v1-${accountId}`, + getConfigSettings: mocks.getConfigSettings, + JOB_PRIORITIES: { + INTERACTIVE: 1, + SCHEDULED: 10, + }, +})); + +const { + removeAccountPermissionSyncScheduler, + scheduleAndTriggerAccountPermissionSync, +} = await import('./accountPermissionSyncQueue.server'); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.getConfigSettings.mockResolvedValue({ + userDrivenPermissionSyncIntervalMs: 86_400_000, + }); + mocks.upsertJobScheduler.mockResolvedValue('scheduled-job'); + mocks.enqueue.mockResolvedValue('job-1'); + mocks.removeJobScheduler.mockResolvedValue(true); +}); + +describe('scheduleAndTriggerAccountPermissionSync', () => { + test('upserts the account scheduler before enqueueing an immediate sync', async () => { + await expect( + scheduleAndTriggerAccountPermissionSync('account-1'), + ).resolves.toEqual({ jobId: 'job-1' }); + + expect(mocks.getConfigSettings).toHaveBeenCalledWith('/config.json'); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + accountPermissionSyncQueue, + 'account-permission-sync-v1-account-1', + 86_400_000, + { accountId: 'account-1' }, + { priority: 10 }, + ); + expect(mocks.enqueue).toHaveBeenCalledWith( + accountPermissionSyncQueue, + { accountId: 'account-1' }, + { priority: 1 }, + ); + expect( + mocks.upsertJobScheduler.mock.invocationCallOrder[0], + ).toBeLessThan(mocks.enqueue.mock.invocationCallOrder[0]); + }); +}); + +describe('removeAccountPermissionSyncScheduler', () => { + test('removes the scheduler using the account scheduler ID', async () => { + await expect( + removeAccountPermissionSyncScheduler('account-1'), + ).resolves.toBe(true); + + expect(mocks.removeJobScheduler).toHaveBeenCalledWith( + accountPermissionSyncQueue, + 'account-permission-sync-v1-account-1', + ); + }); +}); diff --git a/packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.ts b/packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.ts new file mode 100644 index 000000000..94054fffe --- /dev/null +++ b/packages/web/src/ee/features/permissionSync/accountPermissionSyncQueue.server.ts @@ -0,0 +1,41 @@ +import 'server-only'; + +import { getBullMQClient } from '@/lib/bullmqClient'; +import { + ACCOUNT_PERMISSION_SYNC_QUEUE, + env, + getAccountPermissionSyncSchedulerId, + getConfigSettings, + JOB_PRIORITIES, +} from '@sourcebot/shared'; + +export const scheduleAndTriggerAccountPermissionSync = async ( + accountId: string, +): Promise<{ jobId: string }> => { + const settings = await getConfigSettings(env.CONFIG_PATH); + const client = getBullMQClient(); + const data = { accountId }; + + await client.upsertJobScheduler( + ACCOUNT_PERMISSION_SYNC_QUEUE, + getAccountPermissionSyncSchedulerId(accountId), + settings.userDrivenPermissionSyncIntervalMs, + data, + { priority: JOB_PRIORITIES.SCHEDULED }, + ); + const jobId = await client.enqueue( + ACCOUNT_PERMISSION_SYNC_QUEUE, + data, + { priority: JOB_PRIORITIES.INTERACTIVE }, + ); + + return { jobId }; +}; + +export const removeAccountPermissionSyncScheduler = ( + accountId: string, +): Promise => + getBullMQClient().removeJobScheduler( + ACCOUNT_PERMISSION_SYNC_QUEUE, + getAccountPermissionSyncSchedulerId(accountId), + ); diff --git a/packages/web/src/ee/features/sso/actions.test.ts b/packages/web/src/ee/features/sso/actions.test.ts new file mode 100644 index 000000000..4cec5a018 --- /dev/null +++ b/packages/web/src/ee/features/sso/actions.test.ts @@ -0,0 +1,156 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + authContext: undefined as unknown, + hasEntitlement: vi.fn(), + removeAccountPermissionSyncScheduler: vi.fn(), + scheduleAndTriggerAccountPermissionSync: vi.fn(), +})); + +vi.mock('@/middleware/sew', () => ({ + sew: (callback: () => unknown) => callback(), +})); +vi.mock('@/middleware/withAuth', () => ({ + withAuth: (callback: (context: unknown) => unknown) => + callback(mocks.authContext), +})); +vi.mock('@/middleware/withMinimumOrgRole', () => ({ + withMinimumOrgRole: ( + _role: unknown, + _minimumRole: unknown, + callback: () => unknown, + ) => callback(), +})); +vi.mock('@/ee/features/permissionSync/accountPermissionSyncQueue.server', () => ({ + removeAccountPermissionSyncScheduler: + mocks.removeAccountPermissionSyncScheduler, + scheduleAndTriggerAccountPermissionSync: + mocks.scheduleAndTriggerAccountPermissionSync, +})); +vi.mock('@/lib/entitlements', () => ({ + hasEntitlement: mocks.hasEntitlement, +})); +vi.mock('@/lib/serviceError', () => ({ + unexpectedError: (message: string) => ({ error: message }), +})); +vi.mock('@sourcebot/shared', () => ({ + createLogger: () => ({ info: vi.fn() }), + doesIdpSupportPermissionSyncing: () => true, + env: { PERMISSION_SYNC_ENABLED: 'true' }, + getIdentityProviderConfig: vi.fn(), + getIdentityProviderConfigs: vi.fn(), +})); +vi.mock('next/headers', () => ({ + cookies: vi.fn(), +})); + +const { + triggerAccountPermissionSync, + unlinkLinkedAccountProvider, +} = await import('./actions'); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.hasEntitlement.mockResolvedValue(true); + mocks.removeAccountPermissionSyncScheduler.mockResolvedValue(true); + mocks.scheduleAndTriggerAccountPermissionSync.mockResolvedValue({ + jobId: 'job-1', + }); +}); + +describe('triggerAccountPermissionSync', () => { + test('enqueues a sync for an eligible account owned by the user', async () => { + const findFirst = vi.fn().mockResolvedValue({ providerType: 'github' }); + mocks.authContext = { + prisma: { account: { findFirst } }, + role: 'MEMBER', + user: { id: 'user-1' }, + }; + + await expect( + triggerAccountPermissionSync('account-1'), + ).resolves.toEqual({ jobId: 'job-1' }); + + expect(findFirst).toHaveBeenCalledWith({ + where: { + id: 'account-1', + userId: 'user-1', + }, + select: { + providerType: true, + }, + }); + expect( + mocks.scheduleAndTriggerAccountPermissionSync, + ).toHaveBeenCalledWith('account-1'); + }); + + test('does not enqueue a sync for an account the user does not own', async () => { + mocks.authContext = { + prisma: { + account: { findFirst: vi.fn().mockResolvedValue(null) }, + }, + role: 'MEMBER', + user: { id: 'user-1' }, + }; + + await expect( + triggerAccountPermissionSync('account-2'), + ).resolves.toEqual({ + error: 'Account does not support permission syncing', + }); + expect( + mocks.scheduleAndTriggerAccountPermissionSync, + ).not.toHaveBeenCalled(); + }); +}); + +describe('unlinkLinkedAccountProvider', () => { + test('removes account schedulers before deleting the accounts', async () => { + const findMany = vi.fn().mockResolvedValue([ + { id: 'account-1' }, + { id: 'account-2' }, + ]); + const deleteMany = vi.fn().mockResolvedValue({ count: 2 }); + mocks.authContext = { + prisma: { account: { deleteMany, findMany } }, + role: 'MEMBER', + user: { id: 'user-1' }, + }; + + await expect( + unlinkLinkedAccountProvider('github'), + ).resolves.toEqual({ success: true, count: 2 }); + + const where = { providerId: 'github', userId: 'user-1' }; + expect(findMany).toHaveBeenCalledWith({ + where, + select: { id: true }, + }); + expect(mocks.removeAccountPermissionSyncScheduler).toHaveBeenCalledTimes(2); + expect(mocks.removeAccountPermissionSyncScheduler).toHaveBeenCalledWith('account-1'); + expect(mocks.removeAccountPermissionSyncScheduler).toHaveBeenCalledWith('account-2'); + expect(deleteMany).toHaveBeenCalledWith({ where }); + expect( + mocks.removeAccountPermissionSyncScheduler.mock.invocationCallOrder[1], + ).toBeLessThan(deleteMany.mock.invocationCallOrder[0]); + }); + + test('does not delete accounts when scheduler removal fails', async () => { + const findMany = vi.fn().mockResolvedValue([{ id: 'account-1' }]); + const deleteMany = vi.fn(); + mocks.authContext = { + prisma: { account: { deleteMany, findMany } }, + role: 'MEMBER', + user: { id: 'user-1' }, + }; + mocks.removeAccountPermissionSyncScheduler.mockRejectedValue( + new Error('Redis unavailable'), + ); + + await expect( + unlinkLinkedAccountProvider('github'), + ).rejects.toThrow('Redis unavailable'); + expect(deleteMany).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/web/src/ee/features/sso/actions.ts b/packages/web/src/ee/features/sso/actions.ts index f5734c71b..23cf6a10c 100644 --- a/packages/web/src/ee/features/sso/actions.ts +++ b/packages/web/src/ee/features/sso/actions.ts @@ -8,6 +8,8 @@ import { OrgRole, type AccountPermissionSyncIssue } from "@sourcebot/db"; import { hasEntitlement } from "@/lib/entitlements"; import { createLogger, doesIdpSupportPermissionSyncing, env, getIdentityProviderConfig, getIdentityProviderConfigs } from "@sourcebot/shared"; import { cookies } from "next/headers"; +import { removeAccountPermissionSyncScheduler, scheduleAndTriggerAccountPermissionSync } from "@/ee/features/permissionSync/accountPermissionSyncQueue.server"; +import { unexpectedError } from "@/lib/serviceError"; const logger = createLogger('web-ee-sso-actions'); @@ -91,17 +93,65 @@ export const getLinkedAccounts = async () => sew(() => ) ); +export const triggerAccountPermissionSync = async (accountId: string) => sew(() => + withAuth(({ prisma, role, user }) => + withMinimumOrgRole(role, OrgRole.MEMBER, async () => { + try { + if ( + env.PERMISSION_SYNC_ENABLED !== 'true' || + !await hasEntitlement('permission-syncing') + ) { + return unexpectedError('Permission syncing is not enabled'); + } + + const account = await prisma.account.findFirst({ + where: { + id: accountId, + userId: user.id, + }, + select: { + providerType: true, + }, + }); + if ( + !account || + !doesIdpSupportPermissionSyncing(account.providerType) + ) { + return unexpectedError('Account does not support permission syncing'); + } + + return await scheduleAndTriggerAccountPermissionSync(accountId); + } catch { + return unexpectedError('Failed to trigger account permission sync'); + } + }) + ) +); export const unlinkLinkedAccountProvider = async (providerId: string) => sew(() => withAuth(async ({ prisma, role, user }) => withMinimumOrgRole(role, OrgRole.MEMBER, async () => { - const result = await prisma.account.deleteMany({ - where: { - providerId, - userId: user.id, + const where = { + providerId, + userId: user.id, + }; + const accounts = await prisma.account.findMany({ + where, + select: { + id: true, }, }); + await Promise.all( + accounts.map(({ id }) => + removeAccountPermissionSyncScheduler(id), + ), + ); + + const result = await prisma.account.deleteMany({ + where, + }); + logger.info(`Unlinked account provider ${providerId} for user ${user.id}. Deleted ${result.count} account(s).`); return { success: true, count: result.count }; diff --git a/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.test.tsx b/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.test.tsx index 8e54b2c55..4f6ddb3f2 100644 --- a/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.test.tsx +++ b/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.test.tsx @@ -24,11 +24,8 @@ vi.mock('@/components/hooks/use-toast', () => ({ })); vi.mock('@/ee/features/sso/actions', () => ({ - unlinkLinkedAccountProvider: vi.fn(), -})); - -vi.mock('@/features/workerApi/actions', () => ({ triggerAccountPermissionSync: mocks.triggerAccountPermissionSync, + unlinkLinkedAccountProvider: vi.fn(), })); vi.mock('@/app/api/(client)/client', () => ({ diff --git a/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.tsx b/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.tsx index ceb3ef79a..d6b378fd8 100644 --- a/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.tsx +++ b/packages/web/src/ee/features/sso/components/linkedAccountProviderCard.tsx @@ -4,7 +4,7 @@ import { useEffect, useState } from "react"; import { cn, getAuthProviderInfo, unwrapServiceError } from "@/lib/utils"; import { AlertCircle, ArrowUpRight, ChevronDown, RefreshCw, Unlink } from "lucide-react"; import { ProviderIcon } from "./providerIcon"; -import { LinkedAccount } from "@/ee/features/sso/actions"; +import { LinkedAccount, triggerAccountPermissionSync, unlinkLinkedAccountProvider } from "@/ee/features/sso/actions"; import { LoadingButton } from "@/components/ui/loading-button"; import { @@ -13,8 +13,6 @@ import { DropdownMenuItem, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; -import { unlinkLinkedAccountProvider } from "@/ee/features/sso/actions"; -import { triggerAccountPermissionSync } from "@/features/workerApi/actions"; import { isServiceError } from "@/lib/utils"; import { useRouter } from "next/navigation"; import { useToast } from "@/components/hooks/use-toast"; diff --git a/packages/web/src/features/connections/actions.test.ts b/packages/web/src/features/connections/actions.test.ts new file mode 100644 index 000000000..ff9325229 --- /dev/null +++ b/packages/web/src/features/connections/actions.test.ts @@ -0,0 +1,92 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + authContext: undefined as unknown, + enqueue: vi.fn(), +})); + +const connectionQueue = { + name: 'connection-sync', +}; + +vi.mock('@/lib/bullmqClient', () => ({ + getBullMQClient: () => ({ enqueue: mocks.enqueue }), +})); +vi.mock('@/lib/serviceError', () => ({ + unexpectedError: (message: string) => ({ error: message }), +})); +vi.mock('@/middleware/sew', () => ({ + sew: (callback: () => unknown) => callback(), +})); +vi.mock('@/middleware/withAuth', () => ({ + withAuth: (callback: (context: unknown) => unknown) => + callback(mocks.authContext), +})); +vi.mock('@/middleware/withMinimumOrgRole', () => ({ + withMinimumOrgRole: ( + _role: unknown, + _minimumRole: unknown, + callback: () => unknown, + ) => callback(), +})); +vi.mock('@sourcebot/shared', () => ({ + CONNECTION_QUEUE: connectionQueue, + JOB_PRIORITIES: { INTERACTIVE: 1 }, +})); + +const { syncConnection } = await import('./actions'); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.enqueue.mockResolvedValue('job-1'); +}); + +const setAuthContext = (findFirst: ReturnType) => { + mocks.authContext = { + org: { id: 1 }, + prisma: { connection: { findFirst } }, + role: 'OWNER', + }; +}; + +describe('syncConnection', () => { + test('enqueues an interactive sync for an existing connection', async () => { + const findFirst = vi.fn().mockResolvedValue({ id: 42 }); + setAuthContext(findFirst); + + await expect(syncConnection(42)).resolves.toEqual({ jobId: 'job-1' }); + + expect(findFirst).toHaveBeenCalledWith({ + where: { + id: 42, + orgId: 1, + }, + select: { + id: true, + }, + }); + expect(mocks.enqueue).toHaveBeenCalledWith( + connectionQueue, + { connectionId: 42 }, + { priority: 1 }, + ); + }); + + test('does not enqueue a sync for a missing connection', async () => { + setAuthContext(vi.fn().mockResolvedValue(null)); + + await expect(syncConnection(42)).resolves.toEqual({ + error: 'Failed to sync connection', + }); + expect(mocks.enqueue).not.toHaveBeenCalled(); + }); + + test('returns a service error when Redis rejects the job', async () => { + setAuthContext(vi.fn().mockResolvedValue({ id: 42 })); + mocks.enqueue.mockRejectedValue(new Error('Redis unavailable')); + + await expect(syncConnection(42)).resolves.toEqual({ + error: 'Failed to sync connection', + }); + }); +}); diff --git a/packages/web/src/features/connections/actions.ts b/packages/web/src/features/connections/actions.ts new file mode 100644 index 000000000..d6563cf95 --- /dev/null +++ b/packages/web/src/features/connections/actions.ts @@ -0,0 +1,40 @@ +'use server'; + +import { getBullMQClient } from '@/lib/bullmqClient'; +import { unexpectedError } from '@/lib/serviceError'; +import { sew } from '@/middleware/sew'; +import { withAuth } from '@/middleware/withAuth'; +import { withMinimumOrgRole } from '@/middleware/withMinimumOrgRole'; +import { OrgRole } from '@sourcebot/db'; +import { CONNECTION_QUEUE, JOB_PRIORITIES } from '@sourcebot/shared'; + +export const syncConnection = async (connectionId: number) => sew(() => + withAuth(({ org, prisma, role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + try { + const connection = await prisma.connection.findFirst({ + where: { + id: connectionId, + orgId: org.id, + }, + select: { + id: true, + }, + }); + if (!connection) { + return unexpectedError('Failed to sync connection'); + } + + const jobId = await getBullMQClient().enqueue( + CONNECTION_QUEUE, + { connectionId: connection.id }, + { priority: JOB_PRIORITIES.INTERACTIVE }, + ); + + return { jobId }; + } catch { + return unexpectedError('Failed to sync connection'); + } + }) + ) +); diff --git a/packages/web/src/features/repos/actions.test.ts b/packages/web/src/features/repos/actions.test.ts new file mode 100644 index 000000000..c64d5c0ac --- /dev/null +++ b/packages/web/src/features/repos/actions.test.ts @@ -0,0 +1,92 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + authContext: undefined as unknown, + enqueue: vi.fn(), +})); + +const repoIndexQueue = { + name: 'repo-index', +}; + +vi.mock('@/lib/bullmqClient', () => ({ + getBullMQClient: () => ({ enqueue: mocks.enqueue }), +})); +vi.mock('@/lib/serviceError', () => ({ + unexpectedError: (message: string) => ({ error: message }), +})); +vi.mock('@/middleware/sew', () => ({ + sew: (callback: () => unknown) => callback(), +})); +vi.mock('@/middleware/withAuth', () => ({ + withAuth: (callback: (context: unknown) => unknown) => + callback(mocks.authContext), +})); +vi.mock('@/middleware/withMinimumOrgRole', () => ({ + withMinimumOrgRole: ( + _role: unknown, + _minimumRole: unknown, + callback: () => unknown, + ) => callback(), +})); +vi.mock('@sourcebot/shared', () => ({ + JOB_PRIORITIES: { INTERACTIVE: 1 }, + REPO_INDEX_QUEUE: repoIndexQueue, +})); + +const { indexRepo } = await import('./actions'); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.enqueue.mockResolvedValue('job-1'); +}); + +const setAuthContext = (findFirst: ReturnType) => { + mocks.authContext = { + org: { id: 1 }, + prisma: { repo: { findFirst } }, + role: 'OWNER', + }; +}; + +describe('indexRepo', () => { + test('enqueues an interactive index job for an existing repo', async () => { + const findFirst = vi.fn().mockResolvedValue({ id: 42 }); + setAuthContext(findFirst); + + await expect(indexRepo(42)).resolves.toEqual({ jobId: 'job-1' }); + + expect(findFirst).toHaveBeenCalledWith({ + where: { + id: 42, + orgId: 1, + }, + select: { + id: true, + }, + }); + expect(mocks.enqueue).toHaveBeenCalledWith( + repoIndexQueue, + { repoId: 42, type: 'INDEX' }, + { priority: 1 }, + ); + }); + + test('does not enqueue an index job for a missing repo', async () => { + setAuthContext(vi.fn().mockResolvedValue(null)); + + await expect(indexRepo(42)).resolves.toEqual({ + error: 'Failed to index repo', + }); + expect(mocks.enqueue).not.toHaveBeenCalled(); + }); + + test('returns a service error when Redis rejects the job', async () => { + setAuthContext(vi.fn().mockResolvedValue({ id: 42 })); + mocks.enqueue.mockRejectedValue(new Error('Redis unavailable')); + + await expect(indexRepo(42)).resolves.toEqual({ + error: 'Failed to index repo', + }); + }); +}); diff --git a/packages/web/src/features/repos/actions.ts b/packages/web/src/features/repos/actions.ts new file mode 100644 index 000000000..286fb1ab8 --- /dev/null +++ b/packages/web/src/features/repos/actions.ts @@ -0,0 +1,40 @@ +'use server'; + +import { getBullMQClient } from '@/lib/bullmqClient'; +import { unexpectedError } from '@/lib/serviceError'; +import { sew } from '@/middleware/sew'; +import { withAuth } from '@/middleware/withAuth'; +import { withMinimumOrgRole } from '@/middleware/withMinimumOrgRole'; +import { OrgRole } from '@sourcebot/db'; +import { JOB_PRIORITIES, REPO_INDEX_QUEUE } from '@sourcebot/shared'; + +export const indexRepo = async (repoId: number) => sew(() => + withAuth(({ org, prisma, role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + try { + const repo = await prisma.repo.findFirst({ + where: { + id: repoId, + orgId: org.id, + }, + select: { + id: true, + }, + }); + if (!repo) { + return unexpectedError('Failed to index repo'); + } + + const jobId = await getBullMQClient().enqueue( + REPO_INDEX_QUEUE, + { repoId: repo.id, type: 'INDEX' }, + { priority: JOB_PRIORITIES.INTERACTIVE }, + ); + + return { jobId }; + } catch { + return unexpectedError('Failed to index repo'); + } + }) + ) +); diff --git a/packages/web/src/features/workerApi/actions.ts b/packages/web/src/features/workerApi/actions.ts index 33c6c60d7..35ab05aab 100644 --- a/packages/web/src/features/workerApi/actions.ts +++ b/packages/web/src/features/workerApi/actions.ts @@ -2,77 +2,12 @@ import { sew } from "@/middleware/sew"; import { githubRateLimited, repositoryNotFound, unexpectedError } from "@/lib/serviceError"; -import { withAuth, withOptionalAuth } from "@/middleware/withAuth"; -import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; -import { OrgRole } from "@sourcebot/db"; +import { withOptionalAuth } from "@/middleware/withAuth"; import { env } from "@sourcebot/shared"; import z from "zod"; -import { requestAccountPermissionSync } from "./client.server"; const WORKER_API_URL = env.WORKER_API_URL; -export const syncConnection = async (connectionId: number) => sew(() => - withAuth(({ role }) => - withMinimumOrgRole(role, OrgRole.OWNER, async () => { - const response = await fetch(`${WORKER_API_URL}/api/sync-connection`, { - method: 'POST', - body: JSON.stringify({ - connectionId - }), - headers: { - 'Content-Type': 'application/json', - }, - }); - - if (!response.ok) { - return unexpectedError('Failed to sync connection'); - } - - const data = await response.json(); - const schema = z.object({ - jobId: z.string(), - }); - return schema.parse(data); - }) - ) -); - -export const indexRepo = async (repoId: number) => sew(() => - withAuth(({ role }) => - withMinimumOrgRole(role, OrgRole.OWNER, async () => { - const response = await fetch(`${WORKER_API_URL}/api/index-repo`, { - method: 'POST', - body: JSON.stringify({ repoId }), - headers: { - 'Content-Type': 'application/json', - }, - }); - - if (!response.ok) { - return unexpectedError('Failed to index repo'); - } - - const data = await response.json(); - const schema = z.object({ - jobId: z.string(), - }); - return schema.parse(data); - }) - ) -); - -export const triggerAccountPermissionSync = async (accountId: string) => sew(() => - withAuth(({ role }) => - withMinimumOrgRole(role, OrgRole.MEMBER, async () => { - try { - return await requestAccountPermissionSync(accountId); - } catch { - return unexpectedError('Failed to trigger account permission sync'); - } - }) - ) -); - export const addGithubRepo = async (owner: string, repo: string) => sew(() => withOptionalAuth(async () => { const response = await fetch(`${WORKER_API_URL}/api/experimental/add-github-repo`, { diff --git a/packages/web/src/features/workerApi/client.server.test.ts b/packages/web/src/features/workerApi/client.server.test.ts deleted file mode 100644 index 8096f79d9..000000000 --- a/packages/web/src/features/workerApi/client.server.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { beforeEach, describe, expect, test, vi } from 'vitest'; - -vi.mock('server-only', () => ({})); -vi.mock('@sourcebot/shared', () => ({ - env: { WORKER_API_URL: 'http://worker.example.com' }, -})); - -const { requestAccountPermissionSync } = await import('./client.server'); - -beforeEach(() => { - vi.unstubAllGlobals(); -}); - -describe('requestAccountPermissionSync', () => { - test('schedules an account permission sync with the worker', async () => { - const fetchMock = vi.fn().mockResolvedValue(new Response( - JSON.stringify({ jobId: 'job_1' }), - { status: 200 }, - )); - vi.stubGlobal('fetch', fetchMock); - - await expect(requestAccountPermissionSync('account_1')).resolves.toEqual({ jobId: 'job_1' }); - expect(fetchMock).toHaveBeenCalledWith( - 'http://worker.example.com/api/trigger-account-permission-sync', - expect.objectContaining({ - method: 'POST', - body: JSON.stringify({ accountId: 'account_1' }), - }), - ); - }); - - test('rejects when the worker does not accept the sync request', async () => { - vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(null, { status: 503 }))); - - await expect(requestAccountPermissionSync('account_1')) - .rejects.toThrow('Worker rejected account permission sync with HTTP 503.'); - }); -}); diff --git a/packages/web/src/features/workerApi/client.server.ts b/packages/web/src/features/workerApi/client.server.ts deleted file mode 100644 index 79b110d13..000000000 --- a/packages/web/src/features/workerApi/client.server.ts +++ /dev/null @@ -1,26 +0,0 @@ -import 'server-only'; - -import { env } from '@sourcebot/shared'; -import { z } from 'zod'; - -const accountPermissionSyncResponseSchema = z.object({ - jobId: z.string(), -}); -const WORKER_REQUEST_TIMEOUT_MS = 5000; - -export const requestAccountPermissionSync = async (accountId: string): Promise<{ jobId: string }> => { - const response = await fetch(`${env.WORKER_API_URL}/api/trigger-account-permission-sync`, { - method: 'POST', - body: JSON.stringify({ accountId }), - headers: { - 'Content-Type': 'application/json', - }, - signal: AbortSignal.timeout(WORKER_REQUEST_TIMEOUT_MS), - }); - - if (!response.ok) { - throw new Error(`Worker rejected account permission sync with HTTP ${response.status}.`); - } - - return accountPermissionSyncResponseSchema.parse(await response.json()); -}; diff --git a/packages/web/src/lib/encryptedPrismaAdapter.test.ts b/packages/web/src/lib/encryptedPrismaAdapter.test.ts new file mode 100644 index 000000000..286d61760 --- /dev/null +++ b/packages/web/src/lib/encryptedPrismaAdapter.test.ts @@ -0,0 +1,79 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + removeAccountPermissionSyncScheduler: vi.fn(), +})); + +vi.mock('@auth/prisma-adapter', () => ({ + PrismaAdapter: () => ({}), +})); +vi.mock('@sourcebot/shared', () => ({ + encryptOAuthToken: (value: unknown) => value, + getIdentityProviderConfig: vi.fn(), +})); +vi.mock('@/ee/features/permissionSync/accountPermissionSyncQueue.server', () => ({ + removeAccountPermissionSyncScheduler: + mocks.removeAccountPermissionSyncScheduler, +})); + +const { EncryptedPrismaAdapter } = await import('./encryptedPrismaAdapter'); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.removeAccountPermissionSyncScheduler.mockResolvedValue(true); +}); + +describe('EncryptedPrismaAdapter.unlinkAccount', () => { + test('removes the scheduler before deleting the account', async () => { + const findUnique = vi.fn().mockResolvedValue({ id: 'account-1' }); + const deleteAccount = vi.fn().mockResolvedValue({}); + const adapter = EncryptedPrismaAdapter({ + account: { + delete: deleteAccount, + findUnique, + }, + } as never); + + await adapter.unlinkAccount?.({ + provider: 'github', + providerAccountId: 'github-user-1', + }); + + const where = { + providerId_providerAccountId: { + providerId: 'github', + providerAccountId: 'github-user-1', + }, + }; + expect(findUnique).toHaveBeenCalledWith({ + where, + select: { id: true }, + }); + expect(mocks.removeAccountPermissionSyncScheduler).toHaveBeenCalledWith( + 'account-1', + ); + expect(deleteAccount).toHaveBeenCalledWith({ where }); + expect( + mocks.removeAccountPermissionSyncScheduler.mock.invocationCallOrder[0], + ).toBeLessThan(deleteAccount.mock.invocationCallOrder[0]); + }); + + test('does not delete the account when scheduler removal fails', async () => { + const deleteAccount = vi.fn(); + const adapter = EncryptedPrismaAdapter({ + account: { + delete: deleteAccount, + findUnique: vi.fn().mockResolvedValue({ id: 'account-1' }), + }, + } as never); + mocks.removeAccountPermissionSyncScheduler.mockRejectedValue( + new Error('Redis unavailable'), + ); + + await expect(adapter.unlinkAccount?.({ + provider: 'github', + providerAccountId: 'github-user-1', + })).rejects.toThrow('Redis unavailable'); + expect(deleteAccount).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/web/src/lib/encryptedPrismaAdapter.ts b/packages/web/src/lib/encryptedPrismaAdapter.ts index 70e145032..b9a813ff9 100644 --- a/packages/web/src/lib/encryptedPrismaAdapter.ts +++ b/packages/web/src/lib/encryptedPrismaAdapter.ts @@ -2,6 +2,7 @@ import { PrismaAdapter } from "@auth/prisma-adapter"; import type { Adapter, AdapterAccount, AdapterUser } from "next-auth/adapters"; import { PrismaClient } from "@sourcebot/db"; import { encryptOAuthToken, getIdentityProviderConfig } from "@sourcebot/shared"; +import { removeAccountPermissionSyncScheduler } from "@/ee/features/permissionSync/accountPermissionSyncQueue.server"; /** * Encrypts OAuth tokens in account data before database storage @@ -58,13 +59,25 @@ export function EncryptedPrismaAdapter(prisma: PrismaClient): Adapter { return (account?.user ?? null) as AdapterUser | null; }, async unlinkAccount({ provider, providerAccountId }) { - await prisma.account.delete({ - where: { - providerId_providerAccountId: { - providerId: provider, - providerAccountId, - }, + const where = { + providerId_providerAccountId: { + providerId: provider, + providerAccountId, }, + }; + const account = await prisma.account.findUnique({ + where, + select: { + id: true, + }, + }); + + if (account) { + await removeAccountPermissionSyncScheduler(account.id); + } + + await prisma.account.delete({ + where, }); }, }; diff --git a/packages/web/src/lib/redis.ts b/packages/web/src/lib/redis.ts index 0eb6e7f92..2f6a1020f 100644 --- a/packages/web/src/lib/redis.ts +++ b/packages/web/src/lib/redis.ts @@ -4,7 +4,13 @@ import { createRedisClient } from '@sourcebot/shared'; let redis: ReturnType | undefined; +const REDIS_REQUEST_TIMEOUT_MS = 5000; + export function getRedisClient() { - redis ??= createRedisClient(); + redis ??= createRedisClient({ + commandTimeout: REDIS_REQUEST_TIMEOUT_MS, + connectTimeout: REDIS_REQUEST_TIMEOUT_MS, + maxRetriesPerRequest: 1, + }); return redis; } From 511f78e4502d29833e49253ecfadac8f4383b6e4 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 10 Aug 2026 13:04:32 -0700 Subject: [PATCH 12/15] migrate attachment & audit log pruning to workload system --- .../src/attachmentPruneWorkload.test.ts | 168 ++++++++++++++++ .../backend/src/attachmentPruneWorkload.ts | 182 ++++++++++++++++++ packages/backend/src/attachmentPruner.ts | 156 --------------- .../src/ee/auditLogPruneWorkload.test.ts | 142 ++++++++++++++ .../backend/src/ee/auditLogPruneWorkload.ts | 80 ++++++++ packages/backend/src/ee/auditLogPruner.ts | 71 ------- packages/backend/src/index.ts | 13 ++ packages/backend/src/utils.ts | 25 --- packages/shared/src/index.server.ts | 2 + packages/shared/src/queue.ts | 14 ++ 10 files changed, 601 insertions(+), 252 deletions(-) create mode 100644 packages/backend/src/attachmentPruneWorkload.test.ts create mode 100644 packages/backend/src/attachmentPruneWorkload.ts delete mode 100644 packages/backend/src/attachmentPruner.ts create mode 100644 packages/backend/src/ee/auditLogPruneWorkload.test.ts create mode 100644 packages/backend/src/ee/auditLogPruneWorkload.ts delete mode 100644 packages/backend/src/ee/auditLogPruner.ts diff --git a/packages/backend/src/attachmentPruneWorkload.test.ts b/packages/backend/src/attachmentPruneWorkload.test.ts new file mode 100644 index 000000000..9fe516918 --- /dev/null +++ b/packages/backend/src/attachmentPruneWorkload.test.ts @@ -0,0 +1,168 @@ +import type { PrismaClient } from "@sourcebot/db"; +import type { StorageBackend } from "@sourcebot/shared"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { createAttachmentPruneWorkload } from "./attachmentPruneWorkload.js"; + +const mocks = { + updateMany: vi.fn(), + findMany: vi.fn(), + deleteMany: vi.fn(), + storageDelete: vi.fn(), +}; + +const db = { + attachment: { + updateMany: mocks.updateMany, + findMany: mocks.findMany, + deleteMany: mocks.deleteMany, + }, +} as unknown as PrismaClient; + +const storage = { + delete: mocks.storageDelete, +} as unknown as StorageBackend; + +const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + +const processWorkload = (ttlHours = 24) => + createAttachmentPruneWorkload({ db, storage, ttlHours }).process({ + data: {}, + jobId: "job-1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), + }); + +describe("attachmentPruneWorkload", () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.clearAllMocks(); + mocks.updateMany.mockResolvedValue({ count: 0 }); + mocks.findMany.mockResolvedValue([]); + mocks.deleteMany.mockResolvedValue({ count: 0 }); + mocks.storageDelete.mockResolvedValue(undefined); + }); + + test("declares an hourly scheduled workload", () => { + const workload = createAttachmentPruneWorkload({ + db, + storage, + ttlHours: 24, + }); + + expect(workload.queueSpec.name).toBe("attachment-prune"); + expect(workload.concurrency).toBe(1); + expect(workload.schedule).toEqual({ + interval: "1h", + data: {}, + options: { priority: 10 }, + }); + }); + + test("does not schedule or prune when the TTL is disabled", async () => { + const workload = createAttachmentPruneWorkload({ + db, + storage, + ttlHours: 0, + }); + + expect(workload.schedule).toBeUndefined(); + await expect(processWorkload(0)).resolves.toEqual({ + pendingClaimed: 0, + committedClaimed: 0, + reclaimed: 0, + }); + expect(mocks.updateMany).not.toHaveBeenCalled(); + expect(mocks.findMany).not.toHaveBeenCalled(); + }); + + test("claims expired orphans and reclaims their tombstones", async () => { + vi.spyOn(Date, "now").mockReturnValue( + new Date("2026-08-10T12:00:00.000Z").getTime(), + ); + mocks.updateMany + .mockResolvedValueOnce({ count: 2 }) + .mockResolvedValueOnce({ count: 1 }); + mocks.findMany.mockResolvedValue([ + { id: "attachment-1", storageKey: "key-1" }, + { id: "attachment-2", storageKey: "key-2" }, + ]); + mocks.deleteMany.mockResolvedValue({ count: 2 }); + + await expect(processWorkload()).resolves.toEqual({ + pendingClaimed: 2, + committedClaimed: 1, + reclaimed: 2, + }); + + const cutoff = new Date("2026-08-09T12:00:00.000Z"); + expect(mocks.updateMany).toHaveBeenNthCalledWith(1, { + where: { + status: "PENDING", + createdAt: { lt: cutoff }, + }, + data: { status: "DELETING" }, + }); + expect(mocks.updateMany).toHaveBeenNthCalledWith(2, { + where: { + status: "COMMITTED", + createdAt: { lt: cutoff }, + chats: { none: {} }, + }, + data: { status: "DELETING" }, + }); + expect(mocks.storageDelete).toHaveBeenCalledWith("key-1"); + expect(mocks.storageDelete).toHaveBeenCalledWith("key-2"); + expect(mocks.deleteMany).toHaveBeenCalledWith({ + where: { + id: { in: ["attachment-1", "attachment-2"] }, + status: "DELETING", + }, + }); + }); + + test("leaves tombstones whose bytes could not be deleted", async () => { + mocks.findMany.mockResolvedValue([ + { id: "attachment-1", storageKey: "key-1" }, + { id: "attachment-2", storageKey: "key-2" }, + ]); + mocks.storageDelete.mockImplementation(async (key: string) => { + if (key === "key-2") { + throw new Error("Storage unavailable"); + } + }); + mocks.deleteMany.mockResolvedValue({ count: 1 }); + + await expect(processWorkload()).resolves.toEqual({ + pendingClaimed: 0, + committedClaimed: 0, + reclaimed: 1, + }); + + expect(mocks.deleteMany).toHaveBeenCalledWith({ + where: { + id: { in: ["attachment-1"] }, + status: "DELETING", + }, + }); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining("attachment-2"), + ); + }); + + test("propagates database failures so BullMQ can retry", async () => { + const error = new Error("Database unavailable"); + mocks.updateMany.mockRejectedValueOnce(error); + + await expect(processWorkload()).rejects.toBe(error); + }); +}); diff --git a/packages/backend/src/attachmentPruneWorkload.ts b/packages/backend/src/attachmentPruneWorkload.ts new file mode 100644 index 000000000..ab600b5ab --- /dev/null +++ b/packages/backend/src/attachmentPruneWorkload.ts @@ -0,0 +1,182 @@ +import { AttachmentStatus, type PrismaClient } from "@sourcebot/db"; +import { + ATTACHMENT_PRUNE_QUEUE, + getStorageBackend, + JOB_PRIORITIES, + type StorageBackend, +} from "@sourcebot/shared"; +import type { Workload } from "./types.js"; + +const BATCH_SIZE = 1_000; +const ONE_HOUR_MS = 60 * 60 * 1000; + +interface Props { + db: PrismaClient; + ttlHours: number; + storage?: StorageBackend; +} + +interface AttachmentPruneResult { + pendingClaimed: number; + committedClaimed: number; + reclaimed: number; +} + +/** + * Reclaims orphaned attachment blobs using the `DELETING` tombstone protocol: + * an orphan is first atomically flipped to `DELETING`, then its bytes are + * deleted, and only then is the row removed. Because the row (the only durable + * handle to the bytes) outlives the byte delete, a failed byte delete is always + * retryable. + * + * Each run condemns two classes of orphan to `DELETING`, then reclaims all + * tombstones: + * + * 1. PENDING (uploaded-but-never-linked): produced when a user selects a file + * in the chat box but never sends the message. + * 2. COMMITTED with zero links: normally a committed blob is reclaimed inline + * by the chat-delete sweep in the web app, but this is the backstop for an + * interrupted sweep. + * + * @note Byte deletion goes through the shared `StorageBackend`, so the web app + * and this worker share one on-disk layout. + */ +export const createAttachmentPruneWorkload = ({ + db, + ttlHours, + storage = getStorageBackend(), +}: Props): Workload<"attachment-prune", AttachmentPruneResult> => ({ + queueSpec: ATTACHMENT_PRUNE_QUEUE, + concurrency: 1, + ...(ttlHours > 0 + ? { + schedule: { + interval: "1h", + data: {}, + options: { priority: JOB_PRIORITIES.SCHEDULED }, + }, + } + : {}), + process: async ({ logger }) => { + if (ttlHours <= 0) { + logger.debug("Attachment orphan pruning is disabled."); + return { + pendingClaimed: 0, + committedClaimed: 0, + reclaimed: 0, + }; + } + + const cutoff = new Date(Date.now() - ttlHours * ONE_HOUR_MS); + + // Each claim is atomic, so a PENDING blob committed by a concurrent + // send or a zero-link blob re-linked by a concurrent duplicate-chat + // loses the claim and is left intact. + const pendingClaimed = await db.attachment.updateMany({ + where: { + status: AttachmentStatus.PENDING, + createdAt: { lt: cutoff }, + }, + data: { status: AttachmentStatus.DELETING }, + }); + + const committedClaimed = await db.attachment.updateMany({ + where: { + status: AttachmentStatus.COMMITTED, + createdAt: { lt: cutoff }, + chats: { none: {} }, + }, + data: { status: AttachmentStatus.DELETING }, + }); + + const reclaimed = await reclaimTombstonedAttachments({ + db, + storage, + warn: (message) => logger.warn(message), + }); + + if ( + pendingClaimed.count > 0 || + committedClaimed.count > 0 || + reclaimed > 0 + ) { + logger.debug( + `Attachment prune: condemned ${pendingClaimed.count} PENDING + ` + + `${committedClaimed.count} COMMITTED orphan(s), reclaimed ${reclaimed} tombstone(s).`, + ); + } + + return { + pendingClaimed: pendingClaimed.count, + committedClaimed: committedClaimed.count, + reclaimed, + }; + }, +}); + +/** + * Deletes the bytes for every `DELETING` tombstone, then removes the row. A + * failed byte delete leaves the tombstone in place for the next scheduled run. + * Failed IDs are excluded from later batches in this run so a persistent + * storage failure cannot spin the loop. + */ +const reclaimTombstonedAttachments = async ({ + db, + storage, + warn, +}: { + db: PrismaClient; + storage: StorageBackend; + warn: (message: string) => void; +}): Promise => { + let totalReclaimed = 0; + const failedIds: string[] = []; + + while (true) { + const batch = await db.attachment.findMany({ + where: { + status: AttachmentStatus.DELETING, + id: { notIn: failedIds }, + }, + select: { id: true, storageKey: true }, + take: BATCH_SIZE, + }); + + if (batch.length === 0) { + break; + } + + const settled = await Promise.allSettled( + batch.map((attachment) => storage.delete(attachment.storageKey)), + ); + + const reclaimedIds: string[] = []; + batch.forEach((attachment, index) => { + const outcome = settled[index]; + if (outcome.status === "fulfilled") { + reclaimedIds.push(attachment.id); + } else { + warn( + `Failed to delete bytes for tombstoned attachment ${attachment.id}, will retry next run: ${outcome.reason}`, + ); + failedIds.push(attachment.id); + } + }); + + if (reclaimedIds.length > 0) { + const result = await db.attachment.deleteMany({ + where: { + id: { in: reclaimedIds }, + status: AttachmentStatus.DELETING, + }, + }); + totalReclaimed += result.count; + } + + if (batch.length < BATCH_SIZE) { + break; + } + } + + return totalReclaimed; +}; diff --git a/packages/backend/src/attachmentPruner.ts b/packages/backend/src/attachmentPruner.ts deleted file mode 100644 index ffeb534c3..000000000 --- a/packages/backend/src/attachmentPruner.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { AttachmentStatus, PrismaClient } from "@sourcebot/db"; -import { createLogger, env, getStorageBackend } from "@sourcebot/shared"; -import { setIntervalAsync } from "./utils.js"; - -const BATCH_SIZE = 1_000; -const ONE_HOUR_MS = 60 * 60 * 1000; - -const logger = createLogger('attachment-pruner'); - -/** - * Periodically reclaims orphaned attachment blobs older than the configured TTL, - * along with their stored bytes, using the `DELETING` tombstone protocol: an - * orphan is first atomically flipped to `DELETING`, then its bytes are deleted, - * and only then is the row removed. Because the row (the only durable handle to - * the bytes) outlives the byte delete, a failed byte delete is always retryable. - * - * Each tick condemns two classes of orphan to `DELETING`, then reclaims all - * tombstones: - * - * 1. PENDING (uploaded-but-never-linked): produced when a user selects a file - * in the chat box but never sends the message. - * 2. COMMITTED with zero links: normally a committed blob is reclaimed inline - * by the chat-delete sweep in the web app, but if that sweep is interrupted - * (process crash / DB error / failed byte delete) the blob is left tombstoned - * or unlinked. This is the backstop for that case. - * - * @note Byte deletion goes through the shared `StorageBackend`, so the web app - * and this worker share one on-disk layout. - */ -export class AttachmentPruner { - private interval?: NodeJS.Timeout; - private readonly storage = getStorageBackend(); - - constructor(private db: PrismaClient) {} - - startScheduler() { - const ttlHours = env.SOURCEBOT_CHAT_ATTACHMENT_ORPHAN_TTL_HOURS; - if (ttlHours <= 0) { - logger.info('SOURCEBOT_CHAT_ATTACHMENT_ORPHAN_TTL_HOURS is 0, attachment orphan pruning is disabled.'); - return; - } - - logger.debug(`Attachment pruner started. Reclaiming orphaned attachments older than ${ttlHours} hours.`); - - // Run immediately on startup, then every hour. The startup call isn't - // awaited, so log any failure here: this worker exits on - // unhandledRejection, and the recurring schedule will retry. - this.pruneOrphanedAttachments().catch((error) => { - logger.warn(`Initial attachment prune failed: ${error}`); - }); - this.interval = setIntervalAsync(() => this.pruneOrphanedAttachments(), ONE_HOUR_MS); - } - - async dispose() { - if (this.interval) { - clearInterval(this.interval); - this.interval = undefined; - } - } - - private async pruneOrphanedAttachments() { - const cutoff = new Date(Date.now() - env.SOURCEBOT_CHAT_ATTACHMENT_ORPHAN_TTL_HOURS * ONE_HOUR_MS); - - // Condemn orphans by flipping them to the DELETING tombstone. Each claim - // is atomic, so a PENDING blob committed by a concurrent send (its commit - // matches only PENDING rows) or a zero-link blob re-linked by a concurrent - // duplicate-chat loses the claim and is left intact. - // - // PENDING orphans: uploaded but the message was never sent. - const pendingClaimed = await this.db.attachment.updateMany({ - where: { - status: AttachmentStatus.PENDING, - createdAt: { lt: cutoff }, - }, - data: { status: AttachmentStatus.DELETING }, - }); - - // COMMITTED orphans: blobs left with zero links by an interrupted - // chat-delete sweep in the web app. - const committedClaimed = await this.db.attachment.updateMany({ - where: { - status: AttachmentStatus.COMMITTED, - createdAt: { lt: cutoff }, - chats: { none: {} }, - }, - data: { status: AttachmentStatus.DELETING }, - }); - - // Reclaim every tombstone: delete bytes, then the row. This also picks up - // tombstones left behind by the web app's inline reclaim (or a crashed - // earlier tick) whose byte delete failed. - const reclaimed = await this.reclaimTombstonedAttachments(); - - if (pendingClaimed.count > 0 || committedClaimed.count > 0 || reclaimed > 0) { - logger.debug( - `Attachment prune: condemned ${pendingClaimed.count} PENDING + ` + - `${committedClaimed.count} COMMITTED orphan(s), reclaimed ${reclaimed} tombstone(s).`, - ); - } - } - - /** - * Deletes the bytes for every `DELETING` tombstone, then removes the row. - * The row (the only durable handle to the bytes) is removed only after its - * bytes are confirmed gone, so a failed byte delete leaves the tombstone in - * place to be retried on the next tick — bytes can never be orphaned by a - * transient storage error. Rows whose byte delete fails this run are - * excluded from subsequent batches so a persistent failure can't spin the - * loop. - * - * @returns the number of tombstones fully reclaimed (bytes + row). - */ - private async reclaimTombstonedAttachments(): Promise { - let totalReclaimed = 0; - const failedIds: string[] = []; - - while (true) { - const batch = await this.db.attachment.findMany({ - where: { status: AttachmentStatus.DELETING, id: { notIn: failedIds } }, - select: { id: true, storageKey: true }, - take: BATCH_SIZE, - }); - - if (batch.length === 0) { - break; - } - - const settled = await Promise.allSettled( - batch.map((attachment) => this.storage.delete(attachment.storageKey))); - - const reclaimedIds: string[] = []; - batch.forEach((attachment, index) => { - const outcome = settled[index]; - if (outcome.status === 'fulfilled') { - reclaimedIds.push(attachment.id); - } else { - logger.warn(`Failed to delete bytes for tombstoned attachment ${attachment.id}, will retry next tick: ${outcome.reason}`); - failedIds.push(attachment.id); - } - }); - - if (reclaimedIds.length > 0) { - const result = await this.db.attachment.deleteMany({ - where: { id: { in: reclaimedIds }, status: AttachmentStatus.DELETING }, - }); - totalReclaimed += result.count; - } - - if (batch.length < BATCH_SIZE) { - break; - } - } - - return totalReclaimed; - } -} diff --git a/packages/backend/src/ee/auditLogPruneWorkload.test.ts b/packages/backend/src/ee/auditLogPruneWorkload.test.ts new file mode 100644 index 000000000..ef6d4e724 --- /dev/null +++ b/packages/backend/src/ee/auditLogPruneWorkload.test.ts @@ -0,0 +1,142 @@ +import type { PrismaClient } from "@sourcebot/db"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { createAuditLogPruneWorkload } from "./auditLogPruneWorkload.js"; + +const mocks = { + findMany: vi.fn(), + deleteMany: vi.fn(), +}; + +const db = { + audit: { + findMany: mocks.findMany, + deleteMany: mocks.deleteMany, + }, +} as unknown as PrismaClient; + +const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + +const processWorkload = ({ + enabled = true, + retentionDays = 180, +}: { + enabled?: boolean; + retentionDays?: number; +} = {}) => + createAuditLogPruneWorkload({ db, enabled, retentionDays }).process({ + data: {}, + jobId: "job-1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), + }); + +describe("auditLogPruneWorkload", () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.clearAllMocks(); + mocks.findMany.mockResolvedValue([]); + mocks.deleteMany.mockResolvedValue({ count: 0 }); + }); + + test("declares a daily scheduled workload", () => { + const workload = createAuditLogPruneWorkload({ + db, + enabled: true, + retentionDays: 180, + }); + + expect(workload.queueSpec.name).toBe("audit-log-prune"); + expect(workload.concurrency).toBe(1); + expect(workload.schedule).toEqual({ + interval: "1d", + data: {}, + options: { priority: 10 }, + }); + }); + + test.each([ + { enabled: false, retentionDays: 180 }, + { enabled: true, retentionDays: 0 }, + ])( + "does not schedule or prune when disabled: %o", + async ({ enabled, retentionDays }) => { + const workload = createAuditLogPruneWorkload({ + db, + enabled, + retentionDays, + }); + + expect(workload.schedule).toBeUndefined(); + await expect( + processWorkload({ enabled, retentionDays }), + ).resolves.toEqual({ deleted: 0 }); + expect(mocks.findMany).not.toHaveBeenCalled(); + expect(mocks.deleteMany).not.toHaveBeenCalled(); + }, + ); + + test("deletes audit logs older than the retention period", async () => { + vi.spyOn(Date, "now").mockReturnValue( + new Date("2026-08-10T12:00:00.000Z").getTime(), + ); + mocks.findMany.mockResolvedValue([ + { id: "audit-1" }, + { id: "audit-2" }, + ]); + mocks.deleteMany.mockResolvedValue({ count: 2 }); + + await expect(processWorkload()).resolves.toEqual({ deleted: 2 }); + + expect(mocks.findMany).toHaveBeenCalledWith({ + where: { + timestamp: { + lt: new Date("2026-02-11T12:00:00.000Z"), + }, + }, + select: { id: true }, + take: 10_000, + }); + expect(mocks.deleteMany).toHaveBeenCalledWith({ + where: { + id: { in: ["audit-1", "audit-2"] }, + }, + }); + expect(logger.debug).toHaveBeenCalledWith( + "Pruned 2 audit log record(s).", + ); + }); + + test("continues deleting full batches", async () => { + const firstBatch = Array.from({ length: 10_000 }, (_, index) => ({ + id: `audit-${index}`, + })); + mocks.findMany + .mockResolvedValueOnce(firstBatch) + .mockResolvedValueOnce([{ id: "audit-last" }]); + mocks.deleteMany + .mockResolvedValueOnce({ count: 10_000 }) + .mockResolvedValueOnce({ count: 1 }); + + await expect(processWorkload()).resolves.toEqual({ deleted: 10_001 }); + + expect(mocks.findMany).toHaveBeenCalledTimes(2); + expect(mocks.deleteMany).toHaveBeenCalledTimes(2); + }); + + test("propagates database failures so BullMQ can retry", async () => { + const error = new Error("Database unavailable"); + mocks.findMany.mockRejectedValueOnce(error); + + await expect(processWorkload()).rejects.toBe(error); + }); +}); diff --git a/packages/backend/src/ee/auditLogPruneWorkload.ts b/packages/backend/src/ee/auditLogPruneWorkload.ts new file mode 100644 index 000000000..eb6b69a91 --- /dev/null +++ b/packages/backend/src/ee/auditLogPruneWorkload.ts @@ -0,0 +1,80 @@ +import type { PrismaClient } from "@sourcebot/db"; +import { + AUDIT_LOG_PRUNE_QUEUE, + JOB_PRIORITIES, +} from "@sourcebot/shared"; +import type { Workload } from "../types.js"; + +const BATCH_SIZE = 10_000; +const ONE_DAY_MS = 24 * 60 * 60 * 1000; + +interface Props { + db: PrismaClient; + enabled: boolean; + retentionDays: number; +} + +interface AuditLogPruneResult { + deleted: number; +} + +export const createAuditLogPruneWorkload = ({ + db, + enabled, + retentionDays, +}: Props): Workload<"audit-log-prune", AuditLogPruneResult> => ({ + queueSpec: AUDIT_LOG_PRUNE_QUEUE, + concurrency: 1, + ...(enabled && retentionDays > 0 + ? { + schedule: { + interval: "1d", + data: {}, + options: { priority: JOB_PRIORITIES.SCHEDULED }, + }, + } + : {}), + process: async ({ logger }) => { + if (!enabled || retentionDays <= 0) { + logger.debug("Audit log pruning is disabled."); + return { deleted: 0 }; + } + + const cutoff = new Date(Date.now() - retentionDays * ONE_DAY_MS); + let totalDeleted = 0; + + logger.debug( + `Pruning audit logs older than ${cutoff.toISOString()}.`, + ); + + // Delete in batches to avoid long-running transactions. + while (true) { + const batch = await db.audit.findMany({ + where: { timestamp: { lt: cutoff } }, + select: { id: true }, + take: BATCH_SIZE, + }); + + if (batch.length === 0) { + break; + } + + const result = await db.audit.deleteMany({ + where: { id: { in: batch.map(({ id }) => id) } }, + }); + totalDeleted += result.count; + + if (batch.length < BATCH_SIZE) { + break; + } + } + + logger.debug( + totalDeleted > 0 + ? `Pruned ${totalDeleted} audit log record(s).` + : "No audit log records to prune.", + ); + + return { deleted: totalDeleted }; + }, +}); diff --git a/packages/backend/src/ee/auditLogPruner.ts b/packages/backend/src/ee/auditLogPruner.ts deleted file mode 100644 index 9222b91e1..000000000 --- a/packages/backend/src/ee/auditLogPruner.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { PrismaClient } from "@sourcebot/db"; -import { createLogger, env } from "@sourcebot/shared"; -import { setIntervalAsync } from "../utils.js"; - -const BATCH_SIZE = 10_000; -const ONE_DAY_MS = 24 * 60 * 60 * 1000; - -const logger = createLogger('audit-log-pruner'); - -export class AuditLogPruner { - private interval?: NodeJS.Timeout; - - constructor(private db: PrismaClient) {} - - startScheduler() { - if (env.SOURCEBOT_EE_AUDIT_LOGGING_ENABLED !== 'true') { - logger.info('Audit logging is disabled, skipping audit log pruner.'); - return; - } - - if (env.SOURCEBOT_EE_AUDIT_RETENTION_DAYS <= 0) { - logger.info('SOURCEBOT_EE_AUDIT_RETENTION_DAYS is 0, audit log pruning is disabled.'); - return; - } - - logger.debug(`Audit log pruner started. Retaining logs for ${env.SOURCEBOT_EE_AUDIT_RETENTION_DAYS} days.`); - - // Run immediately on startup, then every 24 hours - this.pruneOldAuditLogs(); - this.interval = setIntervalAsync(() => this.pruneOldAuditLogs(), ONE_DAY_MS); - } - - async dispose() { - if (this.interval) { - clearInterval(this.interval); - this.interval = undefined; - } - } - - private async pruneOldAuditLogs() { - const cutoff = new Date(Date.now() - env.SOURCEBOT_EE_AUDIT_RETENTION_DAYS * ONE_DAY_MS); - let totalDeleted = 0; - - logger.debug(`Pruning audit logs older than ${cutoff.toISOString()}...`); - - // Delete in batches to avoid long-running transactions - while (true) { - const batch = await this.db.audit.findMany({ - where: { timestamp: { lt: cutoff } }, - select: { id: true }, - take: BATCH_SIZE, - }); - - if (batch.length === 0) break; - - const result = await this.db.audit.deleteMany({ - where: { id: { in: batch.map(r => r.id) } }, - }); - - totalDeleted += result.count; - - if (batch.length < BATCH_SIZE) break; - } - - if (totalDeleted > 0) { - logger.debug(`Pruned ${totalDeleted} audit log records.`); - } else { - logger.debug('No audit log records to prune.'); - } - } -} diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 7d32466a5..0b19cb082 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -19,6 +19,8 @@ import { createAccountPermissionSyncWorkload } from "./ee/accountPermissionSyncW import { createRepoPermissionSyncWorkload } from "./ee/repoPermissionSyncWorkload.js"; import { reconcileJobSchedulersAtStartup } from "./reconcileJobSchedulersAtStartup.js"; import { hasEntitlement } from "./entitlements.js"; +import { createAttachmentPruneWorkload } from "./attachmentPruneWorkload.js"; +import { createAuditLogPruneWorkload } from "./ee/auditLogPruneWorkload.js"; const logger = createLogger('backend-entrypoint'); @@ -71,11 +73,22 @@ const repoPermissionSyncWorkload = createRepoPermissionSyncWorkload({ db: prisma, settings, }); +const attachmentPruneWorkload = createAttachmentPruneWorkload({ + db: prisma, + ttlHours: env.SOURCEBOT_CHAT_ATTACHMENT_ORPHAN_TTL_HOURS, +}); +const auditLogPruneWorkload = createAuditLogPruneWorkload({ + db: prisma, + enabled: env.SOURCEBOT_EE_AUDIT_LOGGING_ENABLED === "true", + retentionDays: env.SOURCEBOT_EE_AUDIT_RETENTION_DAYS, +}); jobManager.register(connectionWorkload); jobManager.register(repoIndexWorkload); jobManager.register(accountPermissionSyncWorkload); jobManager.register(repoPermissionSyncWorkload); +jobManager.register(attachmentPruneWorkload); +jobManager.register(auditLogPruneWorkload); const api = new Api(promClient, prisma, jobManager); diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index 7db5130be..5836bc04d 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -287,28 +287,3 @@ const createGitCloneUrlWithToken = (cloneUrl: string, credentials: { username?: } return url.toString(); } - - -// setInterval wrapper that ensures async callbacks are not executed concurrently. -// @see: https://mottaquikarim.github.io/dev/posts/setinterval-that-blocks-on-await/ -export const setIntervalAsync = (target: () => Promise, pollingIntervalMs: number): NodeJS.Timeout => { - const setIntervalWithPromise = Promise>( - target: T - ): (...args: Parameters) => Promise => { - return async function (...args: Parameters): Promise { - if ((target as any).isRunning) return; - - (target as any).isRunning = true; - try { - await target(...args); - } finally { - (target as any).isRunning = false; - } - }; - } - - return setInterval( - setIntervalWithPromise(target), - pollingIntervalMs - ); -} diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 9eb86c7a0..da6d7a131 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -102,6 +102,8 @@ export type { } from "./queue.js"; export { ACCOUNT_PERMISSION_SYNC_QUEUE, + ATTACHMENT_PRUNE_QUEUE, + AUDIT_LOG_PRUNE_QUEUE, CONNECTION_QUEUE, DEFAULT_JOB_OPTIONS, JOB_PRIORITIES, diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index e688a342f..58ef706ab 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -42,6 +42,8 @@ export type QueueName = keyof QueueRegistry; export type DataOf = QueueRegistry[TName]; interface QueueRegistry { + "attachment-prune": Record; + "audit-log-prune": Record; "connection-sync": { connectionId: number; }; @@ -57,6 +59,18 @@ interface QueueRegistry { }; } +export const ATTACHMENT_PRUNE_QUEUE: QueueSpec<"attachment-prune"> = { + name: "attachment-prune", + jobOptions: DEFAULT_JOB_OPTIONS, + dedupKey: () => "global", +}; + +export const AUDIT_LOG_PRUNE_QUEUE: QueueSpec<"audit-log-prune"> = { + name: "audit-log-prune", + jobOptions: DEFAULT_JOB_OPTIONS, + dedupKey: () => "global", +}; + export const CONNECTION_QUEUE: QueueSpec<"connection-sync"> = { name: "connection-sync", jobOptions: DEFAULT_JOB_OPTIONS, From 3ec18864b3969bee512e334deff9cb5f11a044bf Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 10 Aug 2026 14:16:50 -0700 Subject: [PATCH 13/15] prioritize initial repository indexing --- packages/backend/src/connectionWorkload.test.ts | 5 +++-- packages/backend/src/connectionWorkload.ts | 2 +- packages/shared/src/queue.ts | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionWorkload.test.ts index c50cd3cac..803ef8707 100644 --- a/packages/backend/src/connectionWorkload.test.ts +++ b/packages/backend/src/connectionWorkload.test.ts @@ -39,6 +39,7 @@ vi.mock("@sourcebot/shared", () => ({ }, }, JOB_PRIORITIES: { + INITIAL: 5, SCHEDULED: 10, }, env: { @@ -269,7 +270,7 @@ describe("connectionWorkload", () => { repoId: 4, type: "INDEX", }, - { priority: 10 }, + { priority: 5 }, ); expect(mocks.connectionUpdate).toHaveBeenCalledWith({ where: { id: 42 }, @@ -493,7 +494,7 @@ describe("connectionWorkload repo sync helpers", () => { repoId: 4, type: "INDEX", }, - { priority: 10 }, + { priority: 5 }, ); expect( mocks.upsertJobScheduler.mock.invocationCallOrder[1], diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts index 01b149444..7c0ca2137 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionWorkload.ts @@ -378,7 +378,7 @@ export const reconcileRepoIndexWork = async ({ repoId: id, type: "INDEX", }, - { priority: JOB_PRIORITIES.SCHEDULED }, + { priority: JOB_PRIORITIES.INITIAL }, ), ), ); diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 58ef706ab..ec7b0f4a5 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -23,6 +23,7 @@ export type JobEnqueueOptions = { export const JOB_PRIORITIES = { INTERACTIVE: 1, + INITIAL: 5, SCHEDULED: 10, } as const; From 19ccad52fb4c65bbf1118d692f476c4f035689d7 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 10 Aug 2026 14:17:49 -0700 Subject: [PATCH 14/15] update retry behaviour --- packages/shared/src/bullmqClient.test.ts | 18 +++++++++++++++--- packages/shared/src/bullmqClient.ts | 6 ++++++ packages/shared/src/queue.ts | 14 +++++++++++--- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts index 84e5efcb0..bd84c3316 100644 --- a/packages/shared/src/bullmqClient.test.ts +++ b/packages/shared/src/bullmqClient.test.ts @@ -53,8 +53,12 @@ describe("BullMQClient", () => { name: "connection-sync", data, opts: { - attempts: 2, - backoff: { type: "exponential", delay: 5000 }, + attempts: 4, + backoff: { + type: "exponential", + delay: 30_000, + jitter: 0.5, + }, removeOnComplete: { age: 1_209_600 }, removeOnFail: { age: 1_209_600 }, keepLogs: 500, @@ -75,7 +79,15 @@ describe("BullMQClient", () => { expect(mocks.add).toHaveBeenCalledWith( "connection-sync", { connectionId: 42 }, - expect.objectContaining({ priority: 1 }), + expect.objectContaining({ + priority: 1, + attempts: 4, + backoff: { + type: "exponential", + delay: 30_000, + jitter: 0.5, + }, + }), ); }); diff --git a/packages/shared/src/bullmqClient.ts b/packages/shared/src/bullmqClient.ts index bc960d76b..4853c433b 100644 --- a/packages/shared/src/bullmqClient.ts +++ b/packages/shared/src/bullmqClient.ts @@ -119,6 +119,9 @@ export class BullMQClient { backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs, + ...(spec.jobOptions.backoff.jitter !== undefined + ? { jitter: spec.jobOptions.backoff.jitter } + : {}), }, removeOnComplete: spec.jobOptions.keepJobs.completed, removeOnFail: spec.jobOptions.keepJobs.failed, @@ -163,6 +166,9 @@ export class BullMQClient { backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs, + ...(spec.jobOptions.backoff.jitter !== undefined + ? { jitter: spec.jobOptions.backoff.jitter } + : {}), }, removeOnComplete: spec.jobOptions.keepJobs.completed, removeOnFail: spec.jobOptions.keepJobs.failed, diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index ec7b0f4a5..8a3cef439 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -9,7 +9,11 @@ export interface QueueSpec { export type JobOptions = { attempts: number; - backoff: { type: "fixed" | "exponential"; delayMs: number }; + backoff: { + type: "fixed" | "exponential"; + delayMs: number; + jitter?: number; + }; keepJobs: { completed: KeepJobs; failed: KeepJobs; @@ -30,8 +34,12 @@ export const JOB_PRIORITIES = { const TWO_WEEKS_IN_SECONDS = 14 * 24 * 60 * 60; export const DEFAULT_JOB_OPTIONS: JobOptions = { - attempts: 2, - backoff: { type: "exponential", delayMs: 5000 }, + attempts: 4, + backoff: { + type: "exponential", + delayMs: 30_000, + jitter: 0.5, + }, keepJobs: { completed: { age: TWO_WEEKS_IN_SECONDS }, failed: { age: TWO_WEEKS_IN_SECONDS }, From 0b7992599c1d9f2bbe7e39b0113c8bc2d26e7303 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 10 Aug 2026 21:13:49 -0700 Subject: [PATCH 15/15] add locking --- CLAUDE.md | 20 ++ packages/backend/package.json | 1 + .../backend/src/connectionWorkload.test.ts | 49 ++++ packages/backend/src/connectionWorkload.ts | 58 +++-- .../ee/accountPermissionSyncWorkload.test.ts | 77 +++++- .../src/ee/accountPermissionSyncWorkload.ts | 114 ++++++--- .../src/ee/repoPermissionSyncWorkload.test.ts | 99 ++++++-- .../src/ee/repoPermissionSyncWorkload.ts | 120 ++++++--- packages/backend/src/executionLock.test.ts | 180 +++++++++++++ packages/backend/src/executionLock.ts | 222 ++++++++++++++++ packages/backend/src/jobManager.test.ts | 89 ++++++- packages/backend/src/jobManager.ts | 55 +++- .../backend/src/repoIndexWorkload.test.ts | 233 ++++++++++++++--- packages/backend/src/repoIndexWorkload.ts | 239 ++++++++++++++---- packages/backend/src/types.ts | 6 + packages/backend/src/types/redlock.d.ts | 57 +++++ .../migration.sql | 2 + .../migration.sql | 2 + .../migration.sql | 2 + .../migration.sql | 2 + packages/db/prisma/schema.prisma | 4 + packages/shared/src/queue.ts | 1 - yarn.lock | 12 +- 23 files changed, 1419 insertions(+), 225 deletions(-) create mode 100644 packages/backend/src/executionLock.test.ts create mode 100644 packages/backend/src/executionLock.ts create mode 100644 packages/backend/src/types/redlock.d.ts create mode 100644 packages/db/prisma/migrations/20260810000000_add_latest_repo_indexing_job_id/migration.sql create mode 100644 packages/db/prisma/migrations/20260811000000_add_latest_account_permission_sync_job_id/migration.sql create mode 100644 packages/db/prisma/migrations/20260811001000_add_latest_repo_permission_sync_job_id/migration.sql create mode 100644 packages/db/prisma/migrations/20260811002000_add_latest_connection_sync_job_id/migration.sql diff --git a/CLAUDE.md b/CLAUDE.md index 99c42fb08..fb2df6e8c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,6 +18,26 @@ To build a specific package: yarn workspace @sourcebot/ build ``` +## Backend Workloads + +Use the workload system in `packages/backend` for background work. Define the queue payload and default job behavior in the shared queue registry, implement a `Workload`, and register it with the `JobManager`. + +### Execution locks + +- Key an execution lock by the logical resource being mutated, not by the job ID. Workloads that mutate the same resource must use the exact same lock key. For example, repo indexing, repo cleanup, and repo permission syncing share the per-repo lock. +- An execution lock serializes work but does not deduplicate it. Multiple jobs for one resource may still be queued and will execute one at a time. +- The lock lease is extended automatically while work is running. The workload's `AbortSignal` is aborted if extension fails or the worker shuts down. +- Abortion is cooperative. Call `signal.throwIfAborted()` before side effects and after long-running or external operations so work stops promptly after losing the lock. The signal cannot cancel an operation that has already been submitted. +- `onStarted` runs after the execution lock is acquired and immediately before `process`. `onCompleted` and `onTerminalFailure` are BullMQ event hooks and run after the processor has returned and released the lock. + +### Lifecycle state + +- In `onStarted`, upsert the workload-specific job row as `IN_PROGRESS`. If the parent resource tracks a `latest...JobId`, update that pointer in the same database transaction. +- Completion and terminal-failure hooks must always update their own historical job row by job ID. Do not condition that update on the job still being latest. Every job row should record its actual outcome. +- `onTerminalFailure` only runs after the job exhausts all retry attempts. Intermediate failures are retried without marking the lifecycle row as terminally failed. +- If a completion or failure hook publishes state onto the parent resource, use a conditional `updateMany` keyed by both the resource ID and its `latest...JobId`. This prevents an older hook from overwriting state belonging to a newer job after the execution lock has been released. +- Parent-resource state written inside `process` is already serialized by the execution lock. It does not need a latest-job conditional merely because the resource tracks the latest job ID. + ## File Naming Files should use camelCase starting with a lowercase letter: diff --git a/packages/backend/package.json b/packages/backend/package.json index bc1fc5e03..50c8cf9d6 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -55,6 +55,7 @@ "p-limit": "^7.2.0", "posthog-node": "^5.24.15", "prom-client": "^15.1.3", + "redlock": "5.0.0-beta.2", "simple-git": "^3.36.0", "zod": "^3.25.76" } diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionWorkload.test.ts index 803ef8707..5997b8e4b 100644 --- a/packages/backend/src/connectionWorkload.test.ts +++ b/packages/backend/src/connectionWorkload.test.ts @@ -83,6 +83,19 @@ import { } from "./connectionWorkload.js"; import { REPO_PERMISSION_SYNC_WHERE } from "./ee/permissionSyncEligibility.js"; +const transactionClient = { + connection: { + update: mocks.connectionUpdate, + }, + connectionSyncJob: { + upsert: mocks.connectionSyncJobUpsert, + }, +}; +const transaction = vi.fn( + (callback: (tx: typeof transactionClient) => Promise) => + callback(transactionClient), +); + const db = { connection: { findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, @@ -99,6 +112,7 @@ const db = { repoToConnection: { deleteMany: mocks.repoToConnectionDeleteMany, }, + $transaction: transaction, } as unknown as PrismaClient; const jobManager = { @@ -156,6 +170,32 @@ describe("connectionWorkload", () => { expect(connectionWorkload.onTerminalFailure).toBeTypeOf("function"); }); + test("uses a distinct execution lock for each connection", () => { + expect(connectionWorkload.executionLock).toBeDefined(); + expect( + connectionWorkload.executionLock?.resource({ connectionId: 42 }), + ).toBe("sourcebot:lock:connection:42"); + expect( + connectionWorkload.executionLock?.resource({ connectionId: 43 }), + ).toBe("sourcebot:lock:connection:43"); + expect(connectionWorkload.executionLock?.durationMs).toBe(60_000); + }); + + test("does not start syncing when execution has already been aborted", async () => { + const controller = new AbortController(); + controller.abort(new Error("Connection execution lock was lost")); + + await expect( + connectionWorkload.process({ + ...lifecycleContext, + signal: controller.signal, + updateProgress: vi.fn(), + trigger: vi.fn(), + }), + ).rejects.toThrow("Connection execution lock was lost"); + expect(mocks.connectionFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + test("marks the connection sync job as in progress when started", async () => { await connectionWorkload.onStarted?.(lifecycleContext); @@ -176,6 +216,15 @@ describe("connectionWorkload", () => { warningMessages: [], }, }); + expect(mocks.connectionUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestSyncJobId: "job-1", + }, + }); + expect(transaction).toHaveBeenCalledOnce(); }); test("marks the connection sync job as completed", async () => { diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts index 7c0ca2137..31eb59e7a 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionWorkload.ts @@ -21,6 +21,8 @@ import { import type { RepoData } from "./repoCompileUtils.js"; import { JobManager, ProcessContext, Settings, Workload } from "./types.js"; +const CONNECTION_SYNC_LOCK_DURATION_MS = 60_000; + interface Props { db: PrismaClient; jobManager: JobManager; @@ -41,6 +43,11 @@ export const createConnectionWorkload = ({ }: Props): Workload<"connection-sync", ConnectionSyncResult> => ({ queueSpec: CONNECTION_QUEUE, concurrency: settings.maxConnectionSyncJobConcurrency, + executionLock: { + resource: ({ connectionId }) => + `sourcebot:lock:connection:${connectionId}`, + durationMs: CONNECTION_SYNC_LOCK_DURATION_MS, + }, process: async ({ data: { connectionId }, logger, @@ -48,11 +55,13 @@ export const createConnectionWorkload = ({ jobId, trigger, }) => { + signal.throwIfAborted(); const connection = await db.connection.findUniqueOrThrow({ where: { id: connectionId, }, }); + signal.throwIfAborted(); const { orgId } = connection; logger.info(`Syncing connection ${connectionId}`, { @@ -66,6 +75,7 @@ export const createConnectionWorkload = ({ signal, }); + signal.throwIfAborted(); await db.connectionSyncJob.update({ where: { id: jobId, @@ -80,6 +90,7 @@ export const createConnectionWorkload = ({ repositoryCount: repoData.length, }); + signal.throwIfAborted(); const repoChanges = await persistConnectionRepositories({ db, connectionId, @@ -87,6 +98,7 @@ export const createConnectionWorkload = ({ discoveredRepos: repoData, }); + signal.throwIfAborted(); await reconcileRepoIndexWork({ jobManager, trigger, @@ -96,6 +108,7 @@ export const createConnectionWorkload = ({ intervalMs: settings.reindexIntervalMs, }); + signal.throwIfAborted(); await reconcileRepoPermissionSyncWork({ db, jobManager, @@ -114,6 +127,7 @@ export const createConnectionWorkload = ({ }, ); + signal.throwIfAborted(); await db.connection.update({ where: { id: connectionId, @@ -125,6 +139,7 @@ export const createConnectionWorkload = ({ // After a connection has synced, we need to re-sync the org's search contexts as // there may be new repos that match the search context's include/exclude patterns. + signal.throwIfAborted(); try { const config = await loadConfig(env.CONFIG_PATH); @@ -139,6 +154,7 @@ export const createConnectionWorkload = ({ ); Sentry.captureException(error); } + signal.throwIfAborted(); logger.info(`Connection ${connectionId} sync finished`, { connectionId, @@ -150,22 +166,32 @@ export const createConnectionWorkload = ({ }; }, onStarted: async ({ data: { connectionId }, jobId }) => { - await db.connectionSyncJob.upsert({ - where: { - id: jobId, - }, - update: { - status: ConnectionSyncJobStatus.IN_PROGRESS, - completedAt: null, - errorMessage: null, - warningMessages: [], - }, - create: { - id: jobId, - connectionId, - status: ConnectionSyncJobStatus.IN_PROGRESS, - warningMessages: [], - }, + await db.$transaction(async (tx) => { + await tx.connectionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: ConnectionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + warningMessages: [], + }, + create: { + id: jobId, + connectionId, + status: ConnectionSyncJobStatus.IN_PROGRESS, + warningMessages: [], + }, + }); + await tx.connection.update({ + where: { + id: connectionId, + }, + data: { + latestSyncJobId: jobId, + }, + }); }); }, onCompleted: async ({ jobId }) => { diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts index 4633dda27..aedd1e590 100644 --- a/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts @@ -91,19 +91,38 @@ const account = { }; const accountFindUniqueOrThrow = vi.fn().mockResolvedValue(account); const accountUpdate = vi.fn().mockResolvedValue(account); +const accountUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); const repoFindMany = vi.fn().mockResolvedValue([]); const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); const permissionDeleteMany = vi.fn().mockResolvedValue({ count: 95 }); const permissionSyncJobUpsert = vi.fn(); const permissionSyncJobUpdate = vi.fn().mockResolvedValue({ account }); -const transaction = vi.fn((queries: Array>) => - Promise.all(queries), +const transactionClient = { + account: { + update: accountUpdate, + updateMany: accountUpdateMany, + }, + accountPermissionSyncJob: { + upsert: permissionSyncJobUpsert, + update: permissionSyncJobUpdate, + }, +}; +const transaction = vi.fn( + ( + queriesOrCallback: + | Array> + | ((tx: typeof transactionClient) => Promise), + ) => + typeof queriesOrCallback === "function" + ? queriesOrCallback(transactionClient) + : Promise.all(queriesOrCallback), ); const db = { account: { findUniqueOrThrow: accountFindUniqueOrThrow, update: accountUpdate, + updateMany: accountUpdateMany, }, accountToRepoPermission: { createMany: permissionCreateMany, @@ -170,6 +189,7 @@ beforeEach(() => { permissionCreateMany.mockResolvedValue({ count: 0 }); permissionDeleteMany.mockResolvedValue({ count: 95 }); permissionSyncJobUpdate.mockResolvedValue({ account }); + accountUpdateMany.mockResolvedValue({ count: 1 }); }); describe("classifyPermissionSyncFailure", () => { @@ -261,6 +281,33 @@ describe("accountPermissionSyncWorkload", () => { expect(workload.onTerminalFailure).toBeTypeOf("function"); }); + test("uses a distinct execution lock for each account", () => { + const workload = createWorkload(); + + expect(workload.executionLock).toBeDefined(); + expect( + workload.executionLock?.resource({ accountId: "account_1" }), + ).toBe("sourcebot:lock:account:account_1"); + expect( + workload.executionLock?.resource({ accountId: "account_2" }), + ).toBe("sourcebot:lock:account:account_2"); + expect(workload.executionLock?.durationMs).toBe(60_000); + }); + + test("does not start syncing when execution has already been aborted", async () => { + const controller = new AbortController(); + controller.abort(new Error("Account execution lock was lost")); + + await expect( + createWorkload().process({ + ...processContext, + signal: controller.signal, + }), + ).rejects.toThrow("Account execution lock was lost"); + expect(mocks.hasEntitlement).not.toHaveBeenCalled(); + expect(accountFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + test("syncs the requested account", async () => { const workload = createWorkload(); @@ -374,9 +421,14 @@ describe("accountPermissionSyncWorkload", () => { status: "IN_PROGRESS", }, }); + expect(accountUpdate).toHaveBeenCalledWith({ + where: { id: "account_1" }, + data: { latestPermissionSyncJobId: "job_1" }, + }); + expect(transaction).toHaveBeenCalledOnce(); }); - test("marks a job completed and clears the account issue", async () => { + test("marks a job completed and clears the account issue when it is still latest", async () => { await createWorkload().onCompleted?.(lifecycleContext, undefined); expect(permissionSyncJobUpdate).toHaveBeenCalledWith({ @@ -385,13 +437,6 @@ describe("accountPermissionSyncWorkload", () => { status: "COMPLETED", completedAt: expect.any(Date), errorMessage: null, - account: { - update: { - permissionSyncedAt: expect.any(Date), - permissionSyncIssue: null, - permissionSyncIssueAt: null, - }, - }, }, select: { account: { @@ -399,6 +444,18 @@ describe("accountPermissionSyncWorkload", () => { }, }, }); + expect(accountUpdateMany).toHaveBeenCalledWith({ + where: { + id: "account_1", + latestPermissionSyncJobId: "job_1", + }, + data: { + permissionSyncedAt: expect.any(Date), + permissionSyncIssue: null, + permissionSyncIssueAt: null, + }, + }); + expect(transaction).toHaveBeenCalledOnce(); }); test("marks a job failed after terminal failure", async () => { diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.ts index af5a3b721..a6bcc5573 100644 --- a/packages/backend/src/ee/accountPermissionSyncWorkload.ts +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.ts @@ -48,6 +48,8 @@ type ProviderConfig = Extract< { provider: TProvider } >; +const ACCOUNT_PERMISSION_SYNC_LOCK_DURATION_MS = 60_000; + interface ProviderPermissionSyncProps { db: PrismaClient; account: AccountWithUser; @@ -135,13 +137,24 @@ export const createAccountPermissionSyncWorkload = ({ return { queueSpec: ACCOUNT_PERMISSION_SYNC_QUEUE, concurrency: settings.maxAccountPermissionSyncJobConcurrency, - process: async ({ data: { accountId }, logger: jobLogger }) => { + executionLock: { + resource: ({ accountId }) => + `sourcebot:lock:account:${accountId}`, + durationMs: ACCOUNT_PERMISSION_SYNC_LOCK_DURATION_MS, + }, + process: async ({ + data: { accountId }, + logger: jobLogger, + signal, + }) => { + signal.throwIfAborted(); if (!(await hasEntitlement("permission-syncing"))) { throw new Error( "Permission syncing entitlement is not currently available.", ); } + signal.throwIfAborted(); const account = await db.account.findUniqueOrThrow({ where: { id: accountId, @@ -150,6 +163,7 @@ export const createAccountPermissionSyncWorkload = ({ user: true, }, }); + signal.throwIfAborted(); jobLogger.debug( `Syncing permissions for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}...`, @@ -158,10 +172,12 @@ export const createAccountPermissionSyncWorkload = ({ try { // Ensure the OAuth token is fresh, refreshing it if it is expired or near expiry. const accessToken = await ensureFreshAccountToken(account, db); + signal.throwIfAborted(); const idpConfig = await getIdentityProviderConfig( account.providerId, ); + signal.throwIfAborted(); if (!idpConfig) { throw new Error( "Unable to find IDP config in config.json.", @@ -175,6 +191,7 @@ export const createAccountPermissionSyncWorkload = ({ config: idpConfig, }); + signal.throwIfAborted(); await db.$transaction([ db.account.update({ where: { @@ -195,7 +212,9 @@ export const createAccountPermissionSyncWorkload = ({ skipDuplicates: true, }), ]); + signal.throwIfAborted(); } catch (error) { + signal.throwIfAborted(); // Clear cached permissions only for classified permanent failures. // Ambiguous HTTP errors and transient upstream failures preserve the // last successful permission state. @@ -204,6 +223,7 @@ export const createAccountPermissionSyncWorkload = ({ if (cleanupDecision.action === "clear_permissions") { const details = PERMISSION_CLEANUP_DETAILS[cleanupDecision.reason]; + signal.throwIfAborted(); const [{ count }] = await db.$transaction([ db.accountToRepoPermission.deleteMany({ where: { accountId: account.id }, @@ -216,6 +236,7 @@ export const createAccountPermissionSyncWorkload = ({ }, }), ]); + signal.throwIfAborted(); const message = error instanceof Error ? error.message : String(error); jobLogger.warn( @@ -226,46 +247,67 @@ export const createAccountPermissionSyncWorkload = ({ } }, onStarted: async ({ data: { accountId }, jobId }) => { - await db.accountPermissionSyncJob.upsert({ - where: { - id: jobId, - }, - update: { - status: AccountPermissionSyncJobStatus.IN_PROGRESS, - completedAt: null, - errorMessage: null, - }, - create: { - id: jobId, - accountId, - status: AccountPermissionSyncJobStatus.IN_PROGRESS, - }, + await db.$transaction(async (tx) => { + await tx.accountPermissionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: AccountPermissionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + accountId, + status: AccountPermissionSyncJobStatus.IN_PROGRESS, + }, + }); + await tx.account.update({ + where: { + id: accountId, + }, + data: { + latestPermissionSyncJobId: jobId, + }, + }); }); }, - onCompleted: async ({ jobId, logger: jobLogger }) => { - const { account } = await db.accountPermissionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: AccountPermissionSyncJobStatus.COMPLETED, - completedAt: new Date(), - errorMessage: null, - account: { - update: { - permissionSyncedAt: new Date(), - permissionSyncIssue: null, - permissionSyncIssueAt: null, - }, + onCompleted: async ({ + data: { accountId }, + jobId, + logger: jobLogger, + }) => { + const account = await db.$transaction(async (tx) => { + const { account } = await tx.accountPermissionSyncJob.update({ + where: { + id: jobId, }, - }, - select: { - account: { - include: { - user: true, + data: { + status: AccountPermissionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + }, + select: { + account: { + include: { + user: true, + }, }, }, - }, + }); + await tx.account.updateMany({ + where: { + id: accountId, + latestPermissionSyncJobId: jobId, + }, + data: { + permissionSyncedAt: new Date(), + permissionSyncIssue: null, + permissionSyncIssueAt: null, + }, + }); + return account; }); jobLogger.debug( diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts index 96796e65d..26ebfe595 100644 --- a/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts @@ -77,18 +77,37 @@ const repo = { }; const repoFindUniqueOrThrow = vi.fn().mockResolvedValue(repo); const repoUpdate = vi.fn().mockResolvedValue(repo); +const repoUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); const accountFindMany = vi.fn().mockResolvedValue([]); const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); const permissionSyncJobUpsert = vi.fn(); -const permissionSyncJobUpdate = vi.fn().mockResolvedValue({ repo }); -const transaction = vi.fn((queries: Array>) => - Promise.all(queries), +const permissionSyncJobUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); +const transactionClient = { + repo: { + update: repoUpdate, + updateMany: repoUpdateMany, + }, + repoPermissionSyncJob: { + upsert: permissionSyncJobUpsert, + updateMany: permissionSyncJobUpdateMany, + }, +}; +const transaction = vi.fn( + ( + queriesOrCallback: + | Array> + | ((tx: typeof transactionClient) => Promise), + ) => + typeof queriesOrCallback === "function" + ? queriesOrCallback(transactionClient) + : Promise.all(queriesOrCallback), ); const db = { repo: { findUniqueOrThrow: repoFindUniqueOrThrow, update: repoUpdate, + updateMany: repoUpdateMany, }, account: { findMany: accountFindMany, @@ -98,7 +117,7 @@ const db = { }, repoPermissionSyncJob: { upsert: permissionSyncJobUpsert, - update: permissionSyncJobUpdate, + updateMany: permissionSyncJobUpdateMany, }, $transaction: transaction, } as unknown as PrismaClient; @@ -147,9 +166,10 @@ beforeEach(() => { mocks.createOctokitFromToken.mockReset().mockResolvedValue({ octokit: {} }); mocks.getRepoCollaborators.mockReset().mockResolvedValue([]); repoFindUniqueOrThrow.mockResolvedValue(repo); - permissionSyncJobUpdate.mockResolvedValue({ repo }); + permissionSyncJobUpdateMany.mockResolvedValue({ count: 1 }); accountFindMany.mockResolvedValue([]); repoUpdate.mockResolvedValue(repo); + repoUpdateMany.mockResolvedValue({ count: 1 }); permissionCreateMany.mockResolvedValue({ count: 0 }); }); @@ -164,6 +184,30 @@ describe("repoPermissionSyncWorkload", () => { expect(workload.onTerminalFailure).toBeTypeOf("function"); }); + test("shares the repository execution lock with indexing and cleanup", () => { + const workload = createWorkload(); + + expect(workload.executionLock).toBeDefined(); + expect(workload.executionLock?.resource({ repoId: 42 })).toBe( + "sourcebot:lock:repo:42", + ); + expect(workload.executionLock?.durationMs).toBe(60_000); + }); + + test("does not start syncing when execution has already been aborted", async () => { + const controller = new AbortController(); + controller.abort(new Error("Repository execution lock was lost")); + + await expect( + createWorkload().process({ + ...processContext, + signal: controller.signal, + }), + ).rejects.toThrow("Repository execution lock was lost"); + expect(mocks.hasEntitlement).not.toHaveBeenCalled(); + expect(repoFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + test("syncs the requested repo with its connections", async () => { await createWorkload().process(processContext); @@ -315,27 +359,47 @@ describe("repoPermissionSyncWorkload", () => { status: "IN_PROGRESS", }, }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { id: 42 }, + data: { latestPermissionSyncJobId: "job_1" }, + }); + expect(transaction).toHaveBeenCalledOnce(); }); - test("marks a job completed and updates the repo sync timestamp", async () => { - await createWorkload().onCompleted?.(lifecycleContext, undefined); + test("marks a job completed and updates the repo sync timestamp when it is still latest", async () => { + await createWorkload().onCompleted?.(lifecycleContext, { + repoName: "sourcebot-dev/sourcebot", + }); - expect(permissionSyncJobUpdate).toHaveBeenCalledWith({ + expect(permissionSyncJobUpdateMany).toHaveBeenCalledWith({ where: { id: "job_1" }, data: { status: "COMPLETED", completedAt: expect.any(Date), errorMessage: null, - repo: { - update: { - permissionSyncedAt: expect.any(Date), - }, - }, }, - select: { - repo: true, + }); + expect(repoUpdateMany).toHaveBeenCalledWith({ + where: { + id: 42, + latestPermissionSyncJobId: "job_1", + }, + data: { + permissionSyncedAt: expect.any(Date), }, }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("does not fail completion after the repo has been deleted", async () => { + permissionSyncJobUpdateMany.mockResolvedValue({ count: 0 }); + repoUpdateMany.mockResolvedValue({ count: 0 }); + + await expect( + createWorkload().onCompleted?.(lifecycleContext, { + repoName: "sourcebot-dev/sourcebot", + }), + ).resolves.toBeUndefined(); }); test("marks a job failed after terminal failure", async () => { @@ -343,16 +407,13 @@ describe("repoPermissionSyncWorkload", () => { await createWorkload().onTerminalFailure?.(lifecycleContext, error); - expect(permissionSyncJobUpdate).toHaveBeenCalledWith({ + expect(permissionSyncJobUpdateMany).toHaveBeenCalledWith({ where: { id: "job_1" }, data: { status: "FAILED", completedAt: expect.any(Date), errorMessage: "Upstream unavailable", }, - select: { - repo: true, - }, }); expect(mocks.captureException).toHaveBeenCalledWith(error, { tags: { diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.ts index 7fd1e59df..be736057d 100644 --- a/packages/backend/src/ee/repoPermissionSyncWorkload.ts +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.ts @@ -39,19 +39,36 @@ interface RepoPermissionSyncWorkloadDependencies { settings: Settings; } +interface RepoPermissionSyncResult { + repoName: string; +} + +const REPO_PERMISSION_SYNC_LOCK_DURATION_MS = 60_000; + export const createRepoPermissionSyncWorkload = ({ db, settings, -}: RepoPermissionSyncWorkloadDependencies): Workload<"repo-permission-sync"> => ({ +}: RepoPermissionSyncWorkloadDependencies): Workload< + "repo-permission-sync", + RepoPermissionSyncResult +> => ({ queueSpec: REPO_PERMISSION_SYNC_QUEUE, concurrency: settings.maxRepoPermissionSyncJobConcurrency, - process: async ({ data: { repoId }, logger }) => { + // This lock is shared with repoIndexWorkload so indexing, cleanup, and + // permission syncing are serialized for the same repository. + executionLock: { + resource: ({ repoId }) => `sourcebot:lock:repo:${repoId}`, + durationMs: REPO_PERMISSION_SYNC_LOCK_DURATION_MS, + }, + process: async ({ data: { repoId }, logger, signal }) => { + signal.throwIfAborted(); if (!(await hasEntitlement("permission-syncing"))) { throw new Error( "Permission syncing entitlement is not currently available.", ); } + signal.throwIfAborted(); const repo = await db.repo.findUniqueOrThrow({ where: { id: repoId, @@ -64,11 +81,13 @@ export const createRepoPermissionSyncWorkload = ({ }, }, }); + signal.throwIfAborted(); const id = repo.id; logger.debug(`Syncing permissions for repo ${repo.displayName}...`); const credentials = await getAuthCredentialsForRepo(repo, logger); + signal.throwIfAborted(); if (!credentials) { throw new Error(`No credentials found for repo ${id}`); } @@ -81,6 +100,7 @@ export const createRepoPermissionSyncWorkload = ({ logger: logger, }); + signal.throwIfAborted(); await db.$transaction([ db.repo.update({ where: { @@ -107,49 +127,68 @@ export const createRepoPermissionSyncWorkload = ({ skipDuplicates: true, }), ]); + signal.throwIfAborted(); + + return { + repoName: repo.displayName ?? repo.name, + }; }, onStarted: async ({ data: { repoId }, jobId }) => { - await db.repoPermissionSyncJob.upsert({ - where: { - id: jobId, - }, - update: { - status: RepoPermissionSyncJobStatus.IN_PROGRESS, - completedAt: null, - errorMessage: null, - }, - create: { - id: jobId, - repoId, - status: RepoPermissionSyncJobStatus.IN_PROGRESS, - }, + await db.$transaction(async (tx) => { + await tx.repoPermissionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: RepoPermissionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + repoId, + status: RepoPermissionSyncJobStatus.IN_PROGRESS, + }, + }); + await tx.repo.update({ + where: { + id: repoId, + }, + data: { + latestPermissionSyncJobId: jobId, + }, + }); }); }, - onCompleted: async ({ jobId, logger }) => { - const { repo } = await db.repoPermissionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: RepoPermissionSyncJobStatus.COMPLETED, - completedAt: new Date(), - errorMessage: null, - repo: { - update: { - permissionSyncedAt: new Date(), - }, + onCompleted: async ( + { data: { repoId }, jobId, logger }, + { repoName }, + ) => { + await db.$transaction(async (tx) => { + await tx.repoPermissionSyncJob.updateMany({ + where: { + id: jobId, }, - }, - select: { - repo: true, - }, + data: { + status: RepoPermissionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + }, + }); + await tx.repo.updateMany({ + where: { + id: repoId, + latestPermissionSyncJobId: jobId, + }, + data: { + permissionSyncedAt: new Date(), + }, + }); }); - logger.debug( - `Permissions synced for repo ${repo.displayName ?? repo.name}`, - ); + logger.debug(`Permissions synced for repo ${repoName}`); }, - onTerminalFailure: async ({ jobId, logger }, error) => { + onTerminalFailure: async ({ data: { repoId }, jobId, logger }, error) => { Sentry.captureException(error, { tags: { jobId, @@ -157,7 +196,7 @@ export const createRepoPermissionSyncWorkload = ({ }, }); - const { repo } = await db.repoPermissionSyncJob.update({ + await db.repoPermissionSyncJob.updateMany({ where: { id: jobId, }, @@ -166,13 +205,10 @@ export const createRepoPermissionSyncWorkload = ({ completedAt: new Date(), errorMessage: error.message, }, - select: { - repo: true, - }, }); logger.error( - `Repo permission sync job failed for repo ${repo.displayName ?? repo.name}: ${error.message}`, + `Repo permission sync job failed for repo ${repoId}: ${error.message}`, ); }, }); diff --git a/packages/backend/src/executionLock.test.ts b/packages/backend/src/executionLock.test.ts new file mode 100644 index 000000000..15ed9ae82 --- /dev/null +++ b/packages/backend/src/executionLock.test.ts @@ -0,0 +1,180 @@ +import type { Redis } from "ioredis"; +import Redlock, { + ExecutionError, + ResourceLockedError, + type ExecutionStats, + type RedlockAbortSignal, +} from "redlock"; +import { describe, expect, test, vi } from "vitest"; +import { RedlockExecutionLockRunner } from "./executionLock.js"; + +const LOCK_DURATION_MS = 60_000; + +const executionError = (attemptError: Error): ExecutionError => { + const client = {} as Redis; + const stats: ExecutionStats = { + membershipSize: 1, + quorumSize: 1, + votesFor: new Set(), + votesAgainst: new Map([[client, attemptError]]), + }; + + return new ExecutionError("Unable to acquire lock", [ + Promise.resolve(stats), + ]); +}; + +const contentionError = (): ExecutionError => + executionError(new ResourceLockedError("Resource is locked")); + +const lockSignal = (controller = new AbortController()) => + controller.signal as RedlockAbortSignal; + +describe("RedlockExecutionLockRunner", () => { + test("waits and retries after lock contention", async () => { + const using = vi + .fn() + .mockRejectedValueOnce(contentionError()) + .mockImplementationOnce( + async (_resources, _duration, _settings, routine) => + routine(lockSignal()), + ); + const runner = new RedlockExecutionLockRunner( + { using } as unknown as Redlock, + { retryDelayMs: 0, retryJitterMs: 0 }, + ); + + await expect( + runner.using( + "resource:1", + LOCK_DURATION_MS, + new AbortController().signal, + async () => "complete", + ), + ).resolves.toBe("complete"); + + expect(using).toHaveBeenCalledTimes(2); + expect(using).toHaveBeenLastCalledWith( + ["resource:1"], + LOCK_DURATION_MS, + { retryCount: 0 }, + expect.any(Function), + ); + }); + + test("does not retry an error thrown after lock acquisition", async () => { + const error = contentionError(); + const using = vi.fn(async (_resources, _duration, _settings, routine) => + routine(lockSignal()), + ); + const runner = new RedlockExecutionLockRunner( + { using } as unknown as Redlock, + { retryDelayMs: 0, retryJitterMs: 0 }, + ); + + await expect( + runner.using( + "resource:1", + LOCK_DURATION_MS, + new AbortController().signal, + async () => { + throw error; + }, + ), + ).rejects.toBe(error); + expect(using).toHaveBeenCalledOnce(); + }); + + test("immediately propagates non-contention acquisition errors", async () => { + const error = executionError(new Error("Redis unavailable")); + const using = vi.fn().mockRejectedValue(error); + const runner = new RedlockExecutionLockRunner( + { using } as unknown as Redlock, + { retryDelayMs: 0, retryJitterMs: 0 }, + ); + + await expect( + runner.using( + "resource:1", + LOCK_DURATION_MS, + new AbortController().signal, + async () => "unreachable", + ), + ).rejects.toBe(error); + expect(using).toHaveBeenCalledOnce(); + }); + + test("stops waiting for a contended lock during shutdown", async () => { + const using = vi.fn().mockRejectedValue(contentionError()); + const runner = new RedlockExecutionLockRunner( + { using } as unknown as Redlock, + { retryDelayMs: 60_000, retryJitterMs: 0 }, + ); + const shutdownController = new AbortController(); + + const result = runner.using( + "resource:1", + LOCK_DURATION_MS, + shutdownController.signal, + async () => "unreachable", + ); + await vi.waitFor(() => expect(using).toHaveBeenCalledOnce()); + + shutdownController.abort(); + + await expect(result).rejects.toBe(shutdownController.signal.reason); + expect(using).toHaveBeenCalledOnce(); + }); + + test("passes lease loss to the routine and rejects a successful return", async () => { + const lockController = new AbortController(); + const signal = lockSignal(lockController); + const leaseError = new Error("Unable to extend lock"); + signal.error = leaseError; + const using = vi.fn(async (_resources, _duration, _settings, routine) => + routine(signal), + ); + const runner = new RedlockExecutionLockRunner({ + using, + } as unknown as Redlock); + let workloadSignal: AbortSignal | undefined; + + const result = runner.using( + "resource:1", + LOCK_DURATION_MS, + new AbortController().signal, + async (combinedSignal) => { + workloadSignal = combinedSignal; + await new Promise((resolve) => { + combinedSignal.addEventListener("abort", () => resolve(), { + once: true, + }); + }); + return "must not succeed"; + }, + ); + await vi.waitFor(() => expect(workloadSignal).toBeDefined()); + + lockController.abort(); + + await expect(result).rejects.toBe(leaseError); + expect(workloadSignal?.aborted).toBe(true); + expect(workloadSignal?.reason).toBe(leaseError); + }); + + test("validates resource names and lock durations", async () => { + const using = vi.fn(); + const runner = new RedlockExecutionLockRunner({ + using, + } as unknown as Redlock); + const signal = new AbortController().signal; + + await expect( + runner.using("", LOCK_DURATION_MS, signal, async () => undefined), + ).rejects.toThrow("resource must not be empty"); + await expect( + runner.using("resource:1", 20_099, signal, async () => undefined), + ).rejects.toThrow("greater than or equal to 20100ms"); + expect(using).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/backend/src/executionLock.ts b/packages/backend/src/executionLock.ts new file mode 100644 index 000000000..1ccd3d284 --- /dev/null +++ b/packages/backend/src/executionLock.ts @@ -0,0 +1,222 @@ +/// + +import { createLogger } from "@sourcebot/shared"; +import type { Redis } from "ioredis"; +import Redlock, { + ExecutionError, + ResourceLockedError, + type RedlockAbortSignal, +} from "redlock"; + +const LOG_TAG = "execution-lock"; +const logger = createLogger(LOG_TAG); + +const REDLOCK_RETRY_DELAY_MS = 250; +const REDLOCK_RETRY_JITTER_MS = 100; +const REDLOCK_AUTOMATIC_EXTENSION_THRESHOLD_MS = 20_000; + +const REDLOCK_SETTINGS = { + driftFactor: 0.01, + retryCount: 0, + retryDelay: REDLOCK_RETRY_DELAY_MS, + retryJitter: REDLOCK_RETRY_JITTER_MS, + automaticExtensionThreshold: REDLOCK_AUTOMATIC_EXTENSION_THRESHOLD_MS, +} as const; + +type RedlockUsingClient = Pick; + +export interface ExecutionLockRunner { + using( + resource: string, + durationMs: number, + shutdownSignal: AbortSignal, + routine: (signal: AbortSignal) => Promise, + ): Promise; +} + +interface RedlockExecutionLockRunnerOptions { + retryDelayMs?: number; + retryJitterMs?: number; + random?: () => number; +} + +const abortReason = (signal: AbortSignal): Error => { + if (signal.reason instanceof Error) { + return signal.reason; + } + + return new Error( + signal.reason === undefined + ? "The operation was aborted" + : String(signal.reason), + ); +}; + +const abortableDelay = async ( + durationMs: number, + signal: AbortSignal, +): Promise => { + if (signal.aborted) { + throw abortReason(signal); + } + + await new Promise((resolve, reject) => { + const onAbort = () => { + clearTimeout(timeout); + reject(abortReason(signal)); + }; + const timeout = setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, durationMs); + + signal.addEventListener("abort", onAbort, { once: true }); + }); +}; + +const combineAbortSignals = ( + shutdownSignal: AbortSignal, + lockSignal: RedlockAbortSignal, +): { signal: AbortSignal; dispose: () => void } => { + const controller = new AbortController(); + const abortFromShutdown = () => { + controller.abort(abortReason(shutdownSignal)); + }; + const abortFromLock = () => { + controller.abort(lockSignal.error ?? abortReason(lockSignal)); + }; + + shutdownSignal.addEventListener("abort", abortFromShutdown, { + once: true, + }); + lockSignal.addEventListener("abort", abortFromLock, { once: true }); + + if (lockSignal.aborted) { + abortFromLock(); + } else if (shutdownSignal.aborted) { + abortFromShutdown(); + } + + return { + signal: controller.signal, + dispose: () => { + shutdownSignal.removeEventListener("abort", abortFromShutdown); + lockSignal.removeEventListener("abort", abortFromLock); + }, + }; +}; + +const isLockContention = async (error: unknown): Promise => { + if (!(error instanceof ExecutionError) || error.attempts.length === 0) { + return false; + } + + const attempts = await Promise.all(error.attempts); + return attempts.every( + ({ votesAgainst }) => + votesAgainst.size > 0 && + [...votesAgainst.values()].every( + (attemptError) => attemptError instanceof ResourceLockedError, + ), + ); +}; + +export class RedlockExecutionLockRunner implements ExecutionLockRunner { + private readonly retryDelayMs: number; + private readonly retryJitterMs: number; + private readonly random: () => number; + + constructor( + private readonly redlock: RedlockUsingClient, + options: RedlockExecutionLockRunnerOptions = {}, + ) { + this.retryDelayMs = options.retryDelayMs ?? REDLOCK_RETRY_DELAY_MS; + this.retryJitterMs = options.retryJitterMs ?? REDLOCK_RETRY_JITTER_MS; + this.random = options.random ?? Math.random; + } + + async using( + resource: string, + durationMs: number, + shutdownSignal: AbortSignal, + routine: (signal: AbortSignal) => Promise, + ): Promise { + if (resource.length === 0) { + throw new Error("Execution lock resource must not be empty"); + } + if ( + !Number.isInteger(durationMs) || + durationMs < REDLOCK_AUTOMATIC_EXTENSION_THRESHOLD_MS + 100 + ) { + throw new Error( + `Execution lock duration must be an integer greater than or equal to ${REDLOCK_AUTOMATIC_EXTENSION_THRESHOLD_MS + 100}ms`, + ); + } + + while (true) { + if (shutdownSignal.aborted) { + throw abortReason(shutdownSignal); + } + + let acquired = false; + try { + return await this.redlock.using( + [resource], + durationMs, + { retryCount: 0 }, + async (lockSignal) => { + acquired = true; + const combined = combineAbortSignals( + shutdownSignal, + lockSignal, + ); + + try { + if (combined.signal.aborted) { + throw abortReason(combined.signal); + } + + const result = await routine(combined.signal); + + if (combined.signal.aborted) { + throw abortReason(combined.signal); + } + + return result; + } finally { + combined.dispose(); + } + }, + ); + } catch (error) { + if (acquired || !(await isLockContention(error))) { + throw error; + } + + const delayMs = Math.max( + 0, + this.retryDelayMs + + Math.floor( + (this.random() * 2 - 1) * this.retryJitterMs, + ), + ); + await abortableDelay(delayMs, shutdownSignal); + } + } + } +} + +export const createExecutionLockRunner = ( + redis: Redis, +): ExecutionLockRunner => { + const redlock = new Redlock([redis], REDLOCK_SETTINGS); + redlock.on("error", (error: unknown) => { + if (error instanceof ResourceLockedError) { + return; + } + + logger.error("Unexpected Redlock error", error); + }); + + return new RedlockExecutionLockRunner(redlock); +}; diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts index f2cc64b9d..e2259d2bd 100644 --- a/packages/backend/src/jobManager.test.ts +++ b/packages/backend/src/jobManager.test.ts @@ -17,6 +17,7 @@ const mocks = vi.hoisted(() => { removeJobScheduler: vi.fn(), producerClose: vi.fn(), workerClose: vi.fn(), + executionLockUsing: vi.fn(), jobLogger, createBullMQJobLogger: vi.fn(() => jobLogger), workers: [] as Array<{ @@ -61,6 +62,12 @@ vi.mock("@sentry/node", () => ({ captureException: vi.fn(), })); +vi.mock("./executionLock.js", () => ({ + createExecutionLockRunner: vi.fn(() => ({ + using: mocks.executionLockUsing, + })), +})); + vi.mock("bullmq", () => ({ Worker: class { private readonly record: (typeof mocks.workers)[number]; @@ -125,6 +132,10 @@ describe("BullMQJobManager lifecycle", () => { mocks.removeJobScheduler.mockResolvedValue(true); mocks.workerClose.mockResolvedValue(undefined); mocks.producerClose.mockResolvedValue(undefined); + mocks.executionLockUsing.mockImplementation( + async (_resource, _durationMs, _shutdownSignal, routine) => + routine(new AbortController().signal), + ); }); test("delegates enqueueing to BullMQClient and returns its job id", async () => { @@ -132,18 +143,14 @@ describe("BullMQJobManager lifecycle", () => { const workload = createWorkload(); manager.register(workload); - const result = await manager.trigger( - "connection-sync", - data, - { priority: 1 }, - ); + const result = await manager.trigger("connection-sync", data, { + priority: 1, + }); expect(result).toBe("job-1"); - expect(mocks.enqueue).toHaveBeenCalledWith( - workload.queueSpec, - data, - { priority: 1 }, - ); + expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data, { + priority: 1, + }); }); test("manages schedulers through the registered workload", async () => { @@ -269,6 +276,68 @@ describe("BullMQJobManager lifecycle", () => { }); }); + test("runs onStarted and processing while the execution lock is held", async () => { + const calls: string[] = []; + const workloadSignal = new AbortController().signal; + mocks.executionLockUsing.mockImplementation( + async (_resource, _durationMs, _shutdownSignal, routine) => { + calls.push("lock-acquired"); + const result = await routine(workloadSignal); + calls.push("lock-released"); + return result; + }, + ); + const workload = createWorkload({ + executionLock: { + resource: ({ connectionId }) => + `sourcebot:lock:connection:${connectionId}`, + durationMs: 60_000, + }, + onStarted: vi.fn(async () => { + calls.push("started"); + }), + process: vi.fn(async ({ signal }) => { + expect(signal).toBe(workloadSignal); + calls.push("processed"); + return { repoCount: 3 }; + }), + }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + await expect( + mocks.workers[0].processor({ ...job, attemptsMade: 0 }), + ).resolves.toEqual({ repoCount: 3 }); + + expect(calls).toEqual([ + "lock-acquired", + "started", + "processed", + "lock-released", + ]); + expect(mocks.executionLockUsing).toHaveBeenCalledWith( + "sourcebot:lock:connection:42", + 60_000, + expect.any(AbortSignal), + expect.any(Function), + ); + expect(mocks.jobLogger.debug).toHaveBeenCalledWith( + "Acquired workload execution lock", + { + resource: "sourcebot:lock:connection:42", + lockWaitMs: expect.any(Number), + }, + ); + expect(mocks.jobLogger.debug).toHaveBeenCalledWith( + "Finished work protected by execution lock", + { + resource: "sourcebot:lock:connection:42", + lockHeldMs: expect.any(Number), + }, + ); + }); + test("provides the structured job logger to the workload processor", async () => { const process = vi.fn( async (context: ProcessContext<"connection-sync">) => { diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index 980080a2b..683a0fb38 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -13,6 +13,8 @@ import { import { Job, Queue, Worker } from "bullmq"; import { Redis } from "ioredis"; import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; +import { createExecutionLockRunner } from "./executionLock.js"; +import type { ExecutionLockRunner } from "./executionLock.js"; import { JobLifecycleContext, Workload } from "./types.js"; import type { JobManager } from "./types.js"; import { prisma } from "./prisma.js"; @@ -28,9 +30,11 @@ export class BullMQJobManager implements JobManager { private readonly workers = new Map(); private readonly bullmqClient: BullMQClient; private readonly abortController = new AbortController(); + private readonly executionLockRunner: ExecutionLockRunner; constructor(private readonly connection: Redis) { this.bullmqClient = new BullMQClient(connection); + this.executionLockRunner = createExecutionLockRunner(connection); } register(workload: Workload): void { @@ -130,7 +134,13 @@ export class BullMQJobManager implements JobManager { private async startWorkload( workload: Workload, ): Promise { - const { queueSpec: spec, concurrency, rateLimit, schedule } = workload; + const { + queueSpec: spec, + concurrency, + executionLock, + rateLimit, + schedule, + } = workload; const worker = new Worker( spec.name, @@ -143,17 +153,52 @@ export class BullMQJobManager implements JobManager { jobLogger, ); - try { + const process = async (signal: AbortSignal) => { await workload.onStarted?.(lifecycleContext); - const result = await workload.process({ + return workload.process({ ...lifecycleContext, - signal: this.abortController.signal, + signal, updateProgress: (progress) => job.updateProgress(progress), trigger: (target, data, options) => this.trigger(target, data, options), }); - return result; + }; + + try { + if (executionLock) { + const resource = executionLock.resource(job.data); + const waitStartedAt = Date.now(); + return await this.executionLockRunner.using( + resource, + executionLock.durationMs, + this.abortController.signal, + async (signal) => { + const acquiredAt = Date.now(); + jobLogger.debug( + "Acquired workload execution lock", + { + resource, + lockWaitMs: acquiredAt - waitStartedAt, + }, + ); + + try { + return await process(signal); + } finally { + jobLogger.debug( + "Finished work protected by execution lock", + { + resource, + lockHeldMs: Date.now() - acquiredAt, + }, + ); + } + }, + ); + } + + return await process(this.abortController.signal); } catch (error) { jobLogger.error( `Workload "${spec.name}" attempt failed`, diff --git a/packages/backend/src/repoIndexWorkload.test.ts b/packages/backend/src/repoIndexWorkload.test.ts index 023a71e07..3bffa2b87 100644 --- a/packages/backend/src/repoIndexWorkload.test.ts +++ b/packages/backend/src/repoIndexWorkload.test.ts @@ -2,23 +2,47 @@ import type { PrismaClient } from "@sourcebot/db"; import { beforeEach, describe, expect, test, vi } from "vitest"; import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; +const fsMocks = vi.hoisted(() => ({ + existsSync: vi.fn(), + readdir: vi.fn(), + rm: vi.fn(), +})); + +vi.mock("fs", () => ({ + existsSync: fsMocks.existsSync, +})); + +vi.mock("fs/promises", () => ({ + readdir: fsMocks.readdir, + rm: fsMocks.rm, +})); + +const repoFindUnique = vi.fn(); +const repoDeleteMany = vi.fn(); const repoIndexingJobUpsert = vi.fn(); -const repoIndexingJobUpdate = vi.fn(); +const repoIndexingJobUpdateMany = vi.fn(); const repoUpdate = vi.fn(); +const repoUpdateMany = vi.fn(); + const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => callback({ repoIndexingJob: { upsert: repoIndexingJobUpsert, - update: repoIndexingJobUpdate, + updateMany: repoIndexingJobUpdateMany, }, repo: { + findUnique: repoFindUnique, update: repoUpdate, + updateMany: repoUpdateMany, }, }), ); const db = { $transaction: transaction, + repo: { + deleteMany: repoDeleteMany, + }, } as unknown as PrismaClient; const workload = createRepoIndexWorkload({ @@ -47,20 +71,69 @@ const lifecycleContext = { logger: lifecycleLogger, }; -describe("repoIndexWorkload lifecycle", () => { +const processContext = { + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), +}; + +const eligibleRepo = { + id: 42, + name: "github.com/acme/repo", + cloneUrl: "https://github.com/acme/repo.git", + external_codeHostType: "github", + orgId: 1, + indexedAt: null, + isAutoCleanupDisabled: false, + connections: [], +}; + +describe("repoIndexWorkload", () => { beforeEach(() => { vi.clearAllMocks(); + fsMocks.existsSync.mockReturnValue(false); + fsMocks.readdir.mockResolvedValue([]); + fsMocks.rm.mockResolvedValue(undefined); + repoFindUnique.mockResolvedValue(eligibleRepo); + repoDeleteMany.mockResolvedValue({ count: 1 }); + repoIndexingJobUpsert.mockResolvedValue(undefined); + repoIndexingJobUpdateMany.mockResolvedValue({ count: 1 }); + repoUpdate.mockResolvedValue(undefined); + repoUpdateMany.mockResolvedValue({ count: 1 }); }); - test("declares database-backed lifecycle hooks", () => { - expect(workload.onStarted).toBeTypeOf("function"); + test("uses the same repository execution lock for INDEX and CLEANUP", () => { + expect(workload.executionLock).toBeDefined(); + expect( + workload.executionLock?.resource({ repoId: 42, type: "INDEX" }), + ).toBe("sourcebot:lock:repo:42"); + expect( + workload.executionLock?.resource({ repoId: 42, type: "CLEANUP" }), + ).toBe("sourcebot:lock:repo:42"); + expect(workload.executionLock?.durationMs).toBe(60_000); + expect(workload.queueSpec.dedupKey).toBeUndefined(); + expect(workload.onStarted).toBeUndefined(); expect(workload.onCompleted).toBeTypeOf("function"); expect(workload.onTerminalFailure).toBeTypeOf("function"); }); - test("marks the repo indexing job and repo as in progress when started", async () => { - await workload.onStarted?.(lifecycleContext); + test("validates state and marks an eligible job in progress inside process", async () => { + await workload.process({ + ...processContext, + data: { repoId: 42, type: "CLEANUP" }, + }); + expect(repoFindUnique).toHaveBeenCalledWith({ + where: { id: 42 }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); expect(repoIndexingJobUpsert).toHaveBeenCalledWith({ where: { id: "job-1", @@ -73,7 +146,7 @@ describe("repoIndexWorkload lifecycle", () => { create: { id: "job-1", repoId: 42, - type: "INDEX", + type: "CLEANUP", status: "IN_PROGRESS", }, }); @@ -82,15 +155,130 @@ describe("repoIndexWorkload lifecycle", () => { id: 42, }, data: { + latestIndexingJobId: "job-1", latestIndexingJobStatus: "IN_PROGRESS", }, }); + expect(repoDeleteMany).toHaveBeenCalledWith({ + where: { + id: 42, + isAutoCleanupDisabled: false, + connections: { + none: {}, + }, + }, + }); + }); + + test("cleanup removes only shards belonging to the exact repository id", async () => { + fsMocks.readdir.mockResolvedValue([ + "1_42_v16.00000.zoekt", + "1_420_v16.00000.zoekt", + ]); + + await workload.process({ + ...processContext, + data: { repoId: 42, type: "CLEANUP" }, + }); + + expect(fsMocks.rm).toHaveBeenCalledWith( + expect.stringContaining("1_42_v16.00000.zoekt"), + { force: true }, + ); + expect(fsMocks.rm).not.toHaveBeenCalledWith( + expect.stringContaining("1_420_v16.00000.zoekt"), + expect.anything(), + ); + }); + + test("skips an INDEX job when the repository no longer exists", async () => { + repoFindUnique.mockResolvedValue(null); + + await workload.process(processContext); + + expect(repoIndexingJobUpsert).not.toHaveBeenCalled(); + expect(repoDeleteMany).not.toHaveBeenCalled(); + expect(fsMocks.readdir).not.toHaveBeenCalled(); + expect(lifecycleLogger.info).toHaveBeenCalledWith( + "Skipping INDEX job for repo 42: repository no longer exists", + ); + }); + + test("finishes orphaned filesystem cleanup when a CLEANUP retry finds no repo", async () => { + repoFindUnique.mockResolvedValue(null); + fsMocks.existsSync.mockReturnValue(true); + fsMocks.readdir.mockResolvedValue([ + "1_42_v16.00000.zoekt", + "1_99_v16.00000.zoekt", + ]); + + await workload.process({ + ...processContext, + data: { repoId: 42, type: "CLEANUP" }, + }); + + expect(repoIndexingJobUpsert).not.toHaveBeenCalled(); + expect(fsMocks.rm).toHaveBeenCalledWith( + expect.stringMatching(/repos\/42$/), + { recursive: true, force: true }, + ); + expect(fsMocks.rm).toHaveBeenCalledWith( + expect.stringContaining("1_42_v16.00000.zoekt"), + { force: true }, + ); + expect(fsMocks.rm).not.toHaveBeenCalledWith( + expect.stringContaining("1_99_v16.00000.zoekt"), + expect.anything(), + ); + }); + + test.each([ + { + name: "automatic cleanup is disabled", + repo: { ...eligibleRepo, isAutoCleanupDisabled: true }, + reason: "automatic cleanup is disabled", + }, + { + name: "the repository was reattached", + repo: { ...eligibleRepo, connections: [{}] }, + reason: "repository has been reattached to a connection", + }, + ])("skips CLEANUP when $name", async ({ repo, reason }) => { + repoFindUnique.mockResolvedValue(repo); + + await workload.process({ + ...processContext, + data: { repoId: 42, type: "CLEANUP" }, + }); + + expect(repoIndexingJobUpsert).not.toHaveBeenCalled(); + expect(repoDeleteMany).not.toHaveBeenCalled(); + expect(fsMocks.readdir).not.toHaveBeenCalled(); + expect(lifecycleLogger.info).toHaveBeenCalledWith( + `Skipping CLEANUP job for repo 42: ${reason}`, + ); + }); + + test("revalidates cleanup eligibility when atomically deleting the repo", async () => { + repoDeleteMany.mockResolvedValue({ count: 0 }); + + await workload.process({ + ...processContext, + data: { repoId: 42, type: "CLEANUP" }, + }); + + expect(repoIndexingJobUpsert).toHaveBeenCalled(); + expect(repoDeleteMany).toHaveBeenCalled(); + expect(fsMocks.readdir).not.toHaveBeenCalled(); + expect(lifecycleLogger.info).toHaveBeenCalledWith( + "Skipping CLEANUP job for repo 42: repository is no longer eligible for cleanup", + ); }); - test("marks the repo indexing job and repo as completed", async () => { + test("marks a completed job and fences the repository summary by job id", async () => { await workload.onCompleted?.(lifecycleContext, undefined); - expect(repoIndexingJobUpdate).toHaveBeenCalledWith({ + expect(repoIndexingJobUpdateMany).toHaveBeenCalledWith({ where: { id: "job-1", }, @@ -100,9 +288,10 @@ describe("repoIndexWorkload lifecycle", () => { errorMessage: null, }, }); - expect(repoUpdate).toHaveBeenCalledWith({ + expect(repoUpdateMany).toHaveBeenCalledWith({ where: { id: 42, + latestIndexingJobId: "job-1", }, data: { latestIndexingJobStatus: "COMPLETED", @@ -110,28 +299,13 @@ describe("repoIndexWorkload lifecycle", () => { }); }); - test("does not update a completed cleanup job after its repo cascades the job row", async () => { - await workload.onCompleted?.( - { - ...lifecycleContext, - data: { - repoId: 42, - type: "CLEANUP", - }, - }, - undefined, - ); - - expect(transaction).not.toHaveBeenCalled(); - }); - - test("marks the repo indexing job and repo as failed after terminal failure", async () => { + test("marks a terminal failure and fences the repository summary by job id", async () => { await workload.onTerminalFailure?.( lifecycleContext, new Error("Unable to clone repository"), ); - expect(repoIndexingJobUpdate).toHaveBeenCalledWith({ + expect(repoIndexingJobUpdateMany).toHaveBeenCalledWith({ where: { id: "job-1", }, @@ -141,9 +315,10 @@ describe("repoIndexWorkload lifecycle", () => { errorMessage: "Unable to clone repository", }, }); - expect(repoUpdate).toHaveBeenCalledWith({ + expect(repoUpdateMany).toHaveBeenCalledWith({ where: { id: 42, + latestIndexingJobId: "job-1", }, data: { latestIndexingJobStatus: "FAILED", diff --git a/packages/backend/src/repoIndexWorkload.ts b/packages/backend/src/repoIndexWorkload.ts index 7e21d9493..cd697e279 100644 --- a/packages/backend/src/repoIndexWorkload.ts +++ b/packages/backend/src/repoIndexWorkload.ts @@ -7,11 +7,12 @@ import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js'; import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getLatestCommitTimestamp, getLocalDefaultBranch, getTags, isPathAValidGitRepoRoot, isRepoEmpty, unsetGitConfig, upsertGitConfig, writeCommitGraph } from './git.js'; import { captureEvent } from './posthog.js'; import { RepoWithConnections, Settings, Workload } from "./types.js"; -import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, getShardPrefix, measure } from './utils.js'; +import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, measure } from './utils.js'; import { cleanupTempShards, indexGitRepository } from './zoekt.js'; const LOG_TAG = 'repo-index-workload'; const logger = createLogger(LOG_TAG); +const REPO_INDEX_LOCK_DURATION_MS = 60_000; interface Props { db: PrismaClient; @@ -24,28 +25,65 @@ export const createRepoIndexWorkload = ({ }: Props): Workload<'repo-index'> => ({ queueSpec: REPO_INDEX_QUEUE, concurrency: settings.maxRepoIndexingJobConcurrency, - process: async ({ data, logger: jobLogger, signal }) => { - const repo = await db.repo.findUniqueOrThrow({ - where: { id: data.repoId }, - include: { - connections: { - include: { - connection: true, - }, - }, - }, + // This lock is shared with repoPermissionSyncWorkload so indexing, cleanup, + // and permission syncing are serialized for the same repository. + executionLock: { + resource: ({ repoId }) => `sourcebot:lock:repo:${repoId}`, + durationMs: REPO_INDEX_LOCK_DURATION_MS, + }, + process: async ({ data, jobId, logger: jobLogger, signal }) => { + signal.throwIfAborted(); + const start = await prepareRepoIndexJob({ + db, + repoId: data.repoId, + type: data.type, + jobId, }); + if (start.action === "skip") { + jobLogger.info( + `Skipping ${data.type} job for repo ${data.repoId}: ${start.reason}`, + ); + + if (data.type === "CLEANUP" && start.repoMissing) { + signal.throwIfAborted(); + await cleanupOrphanedRepoResourcesForRepoId( + data.repoId, + jobLogger, + ); + } + return; + } + + signal.throwIfAborted(); + const { repo } = start; jobLogger.debug(`Running ${data.type} job for repo ${repo.name} (id: ${repo.id})`); - if (data.type === 'CLEANUP') { - await cleanupRepository(repo, jobLogger); - await db.repo.delete({ - where: { id: repo.id }, + if (data.type === "CLEANUP") { + signal.throwIfAborted(); + const { count } = await db.repo.deleteMany({ + where: { + id: repo.id, + isAutoCleanupDisabled: false, + connections: { + none: {}, + }, + }, }); + + if (count === 0) { + jobLogger.info( + `Skipping CLEANUP job for repo ${repo.id}: repository is no longer eligible for cleanup`, + ); + return; + } + + signal.throwIfAborted(); + await cleanupRepository(repo, jobLogger); } else { const isFirstIndex = repo.indexedAt === null; const revisions = await indexRepository(db, settings, repo, jobLogger, signal); + signal.throwIfAborted(); const { path: repoPath } = getRepoPath(repo); const isEmpty = await isRepoEmpty({ path: repoPath }); const commitHash = isEmpty ? undefined : await getCommitHashForRefName({ @@ -59,6 +97,7 @@ export const createRepoIndexWorkload = ({ select: { metadata: true }, }); + signal.throwIfAborted(); await db.repo.update({ where: { id: repo.id }, data: { @@ -81,43 +120,9 @@ export const createRepoIndexWorkload = ({ } } }, - onStarted: async ({ data: { repoId, type }, jobId }) => { + onCompleted: async ({ data: { repoId }, jobId }) => { await db.$transaction(async (tx) => { - await tx.repoIndexingJob.upsert({ - where: { - id: jobId, - }, - update: { - status: RepoIndexingJobStatus.IN_PROGRESS, - completedAt: null, - errorMessage: null, - }, - create: { - id: jobId, - repoId, - type: RepoIndexingJobType[type], - status: RepoIndexingJobStatus.IN_PROGRESS, - }, - }); - await tx.repo.update({ - where: { - id: repoId, - }, - data: { - latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, - }, - }); - }); - }, - onCompleted: async ({ data: { repoId, type }, jobId }) => { - // A successful cleanup deletes the Repo in `process`, which cascades to its - // RepoIndexingJob records. There is no row left to mark as completed. - if (type === RepoIndexingJobType.CLEANUP) { - return; - } - - await db.$transaction(async (tx) => { - await tx.repoIndexingJob.update({ + await tx.repoIndexingJob.updateMany({ where: { id: jobId, }, @@ -127,9 +132,10 @@ export const createRepoIndexWorkload = ({ errorMessage: null, }, }); - await tx.repo.update({ + await tx.repo.updateMany({ where: { id: repoId, + latestIndexingJobId: jobId, }, data: { latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, @@ -139,7 +145,7 @@ export const createRepoIndexWorkload = ({ }, onTerminalFailure: async ({ data: { repoId }, jobId }, error) => { await db.$transaction(async (tx) => { - await tx.repoIndexingJob.update({ + await tx.repoIndexingJob.updateMany({ where: { id: jobId, }, @@ -149,9 +155,10 @@ export const createRepoIndexWorkload = ({ errorMessage: error.message, }, }); - await tx.repo.update({ + await tx.repo.updateMany({ where: { id: repoId, + latestIndexingJobId: jobId, }, data: { latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, @@ -161,6 +168,96 @@ export const createRepoIndexWorkload = ({ }, }); +type RepoIndexStartDecision = + | { + action: "run"; + repo: RepoWithConnections; + } + | { + action: "skip"; + reason: string; + repoMissing: boolean; + }; + +const prepareRepoIndexJob = async ({ + db, + repoId, + type, + jobId, +}: { + db: PrismaClient; + repoId: number; + type: "INDEX" | "CLEANUP"; + jobId: string; +}): Promise => + db.$transaction(async (tx) => { + const repo = await tx.repo.findUnique({ + where: { id: repoId }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + + if (!repo) { + return { + action: "skip", + reason: "repository no longer exists", + repoMissing: true, + }; + } + + if (type === "CLEANUP" && repo.isAutoCleanupDisabled) { + return { + action: "skip", + reason: "automatic cleanup is disabled", + repoMissing: false, + }; + } + + if (type === "CLEANUP" && repo.connections.length > 0) { + return { + action: "skip", + reason: "repository has been reattached to a connection", + repoMissing: false, + }; + } + + await tx.repoIndexingJob.upsert({ + where: { + id: jobId, + }, + update: { + status: RepoIndexingJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + repoId, + type: RepoIndexingJobType[type], + status: RepoIndexingJobStatus.IN_PROGRESS, + }, + }); + await tx.repo.update({ + where: { + id: repoId, + }, + data: { + latestIndexingJobId: jobId, + latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, + }, + }); + + return { + action: "run", + repo, + }; + }); + const indexRepository = async ( db: PrismaClient, settings: Settings, @@ -257,6 +354,7 @@ const indexRepository = async ( // for its full history (either freshly written during clone, or backfilled above // during fetch). if (!isReadOnly && !metadata.commitGraphChangedPathsBackfilledAt) { + signal.throwIfAborted(); await db.repo.update({ where: { id: repo.id }, data: { @@ -333,10 +431,16 @@ const indexRepository = async ( logger.debug(`Indexing ${repo.name} (id: ${repo.id})...`); try { + signal.throwIfAborted(); const { durationMs } = await measure(() => indexGitRepository(repo, settings, revisions, signal)); + signal.throwIfAborted(); const indexDuration_s = durationMs / 1000; logger.debug(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`); } catch (error) { + if (signal.aborted) { + throw error; + } + // Clean up any temporary shard files left behind by the failed indexing operation. // Zoekt creates .tmp files during indexing which can accumulate if indexing fails repeatedly. logger.warn(`Indexing failed for ${repo.name} (id: ${repo.id}), cleaning up temp shard files...`); @@ -354,8 +458,7 @@ const cleanupRepository = async (repo: Repo, logger: JobLogSink) => { await rm(repoPath, { recursive: true, force: true }); } - const shardPrefix = getShardPrefix(repo.orgId, repo.id); - const files = (await readdir(INDEX_CACHE_DIR)).filter(file => file.startsWith(shardPrefix)); + const files = (await readdir(INDEX_CACHE_DIR)).filter(file => getRepoIdFromShardFileName(file) === repo.id); for (const file of files) { const filePath = `${INDEX_CACHE_DIR}/${file}`; logger.debug(`Deleting shard file ${filePath}`); @@ -363,6 +466,30 @@ const cleanupRepository = async (repo: Repo, logger: JobLogSink) => { } }; +const cleanupOrphanedRepoResourcesForRepoId = async ( + repoId: number, + logger: JobLogSink, +) => { + const repoPath = `${REPOS_CACHE_DIR}/${repoId}`; + if (existsSync(repoPath)) { + logger.debug(`Deleting orphaned repo directory ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + + if (!existsSync(INDEX_CACHE_DIR)) { + return; + } + + const shardFiles = (await readdir(INDEX_CACHE_DIR)).filter( + (file) => getRepoIdFromShardFileName(file) === repoId, + ); + for (const file of shardFiles) { + const filePath = `${INDEX_CACHE_DIR}/${file}`; + logger.debug(`Deleting orphaned shard file ${filePath}`); + await rm(filePath, { force: true }); + } +}; + // Scans the repos and index directories on disk and removes any entries // that have no corresponding Repo record in the database. This handles // edge cases where the DB and disk resources are out of sync. diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index f5537f52a..1e2e6f6fd 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -60,9 +60,15 @@ export interface ProcessContext ): Promise; } +export interface WorkloadExecutionLock { + resource(data: DataOf): string; + durationMs: number; +} + export interface Workload { queueSpec: QueueSpec; concurrency: number; + executionLock?: WorkloadExecutionLock; schedule?: { interval: Schedule; data: DataOf; diff --git a/packages/backend/src/types/redlock.d.ts b/packages/backend/src/types/redlock.d.ts new file mode 100644 index 000000000..b03e91181 --- /dev/null +++ b/packages/backend/src/types/redlock.d.ts @@ -0,0 +1,57 @@ +// redlock@5.0.0-beta.2 bundles this declaration, but its package exports do not +// expose it to TypeScript's Node16 module resolution. Keep this narrow surface in +// sync with the upstream declaration until the package publishes a fixed export. +declare module "redlock" { + import { EventEmitter } from "events"; + import { Cluster, Redis } from "ioredis"; + + type Client = Redis | Cluster; + + export interface ExecutionStats { + readonly membershipSize: number; + readonly quorumSize: number; + readonly votesFor: Set; + readonly votesAgainst: Map; + } + + export interface Settings { + readonly driftFactor: number; + readonly retryCount: number; + readonly retryDelay: number; + readonly retryJitter: number; + readonly automaticExtensionThreshold: number; + } + + export class ResourceLockedError extends Error { + constructor(message: string); + } + + export class ExecutionError extends Error { + readonly attempts: ReadonlyArray>; + constructor( + message: string, + attempts: ReadonlyArray>, + ); + } + + export type RedlockAbortSignal = AbortSignal & { + error?: Error; + }; + + export default class Redlock extends EventEmitter { + constructor(clients: Iterable, settings?: Partial); + + using( + resources: string[], + duration: number, + settings: Partial, + routine: (signal: RedlockAbortSignal) => Promise, + ): Promise; + + using( + resources: string[], + duration: number, + routine: (signal: RedlockAbortSignal) => Promise, + ): Promise; + } +} diff --git a/packages/db/prisma/migrations/20260810000000_add_latest_repo_indexing_job_id/migration.sql b/packages/db/prisma/migrations/20260810000000_add_latest_repo_indexing_job_id/migration.sql new file mode 100644 index 000000000..b0c4a67a5 --- /dev/null +++ b/packages/db/prisma/migrations/20260810000000_add_latest_repo_indexing_job_id/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Repo" ADD COLUMN "latestIndexingJobId" TEXT; diff --git a/packages/db/prisma/migrations/20260811000000_add_latest_account_permission_sync_job_id/migration.sql b/packages/db/prisma/migrations/20260811000000_add_latest_account_permission_sync_job_id/migration.sql new file mode 100644 index 000000000..dbd96beb5 --- /dev/null +++ b/packages/db/prisma/migrations/20260811000000_add_latest_account_permission_sync_job_id/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Account" ADD COLUMN "latestPermissionSyncJobId" TEXT; diff --git a/packages/db/prisma/migrations/20260811001000_add_latest_repo_permission_sync_job_id/migration.sql b/packages/db/prisma/migrations/20260811001000_add_latest_repo_permission_sync_job_id/migration.sql new file mode 100644 index 000000000..35f303b95 --- /dev/null +++ b/packages/db/prisma/migrations/20260811001000_add_latest_repo_permission_sync_job_id/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Repo" ADD COLUMN "latestPermissionSyncJobId" TEXT; diff --git a/packages/db/prisma/migrations/20260811002000_add_latest_connection_sync_job_id/migration.sql b/packages/db/prisma/migrations/20260811002000_add_latest_connection_sync_job_id/migration.sql new file mode 100644 index 000000000..62a56fcad --- /dev/null +++ b/packages/db/prisma/migrations/20260811002000_add_latest_connection_sync_job_id/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Connection" ADD COLUMN "latestSyncJobId" TEXT; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index e43f2887b..250b7e2eb 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -73,10 +73,12 @@ model Repo { permittedAccounts AccountToRepoPermission[] permissionSyncJobs RepoPermissionSyncJob[] permissionSyncedAt DateTime? /// When the permissions were last synced successfully. + latestPermissionSyncJobId String? /// The permission sync job allowed to publish the latest repo state. jobs RepoIndexingJob[] indexedAt DateTime? /// When the repo was last indexed successfully. indexedCommitHash String? /// The commit hash of the last indexed commit (on HEAD). + latestIndexingJobId String? /// The job allowed to publish the latest indexing status. latestIndexingJobStatus RepoIndexingJobStatus? /// The status of the latest indexing job. pushedAt DateTime? /// The timestamp of the most recent commit across all branches. @@ -184,6 +186,7 @@ model Connection { syncJobs ConnectionSyncJob[] /// When the connection was last synced successfully. syncedAt DateTime? + latestSyncJobId String? /// The most recently started connection sync job. /// Controls whether repository permissions are enforced for this connection. /// When `PERMISSION_SYNC_ENABLED` is false, this setting has no effect. @@ -611,6 +614,7 @@ model Account { permissionSyncJobs AccountPermissionSyncJob[] permissionSyncedAt DateTime? + latestPermissionSyncJobId String? /// The permission sync job allowed to publish the latest account state. /// Set when permission syncing fails closed and user action is required. /// Cleared after a subsequent permission sync completes successfully. diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 8a3cef439..993fc9b6c 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -89,7 +89,6 @@ export const CONNECTION_QUEUE: QueueSpec<"connection-sync"> = { export const REPO_INDEX_QUEUE: QueueSpec<"repo-index"> = { name: "repo-index", jobOptions: DEFAULT_JOB_OPTIONS, - dedupKey: (data) => `repo:${data.repoId}`, }; export const ACCOUNT_PERMISSION_SYNC_QUEUE: QueueSpec<"account-permission-sync"> = diff --git a/yarn.lock b/yarn.lock index 607319c90..543d2ce43 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9048,6 +9048,7 @@ __metadata: p-limit: "npm:^7.2.0" posthog-node: "npm:^5.24.15" prom-client: "npm:^15.1.3" + redlock: "npm:5.0.0-beta.2" simple-git: "npm:^3.36.0" tsc-watch: "npm:^6.2.0" tsx: "npm:^4.21.0" @@ -18683,7 +18684,7 @@ __metadata: languageName: node linkType: hard -"node-abort-controller@npm:3.1.1": +"node-abort-controller@npm:3.1.1, node-abort-controller@npm:^3.0.1": version: 3.1.1 resolution: "node-abort-controller@npm:3.1.1" checksum: 10c0/f7ad0e7a8e33809d4f3a0d1d65036a711c39e9d23e0319d80ebe076b9a3b4432b4d6b86a7fab65521de3f6872ffed36fc35d1327487c48eb88c517803403eda3 @@ -20705,6 +20706,15 @@ __metadata: languageName: node linkType: hard +"redlock@npm:5.0.0-beta.2": + version: 5.0.0-beta.2 + resolution: "redlock@npm:5.0.0-beta.2" + dependencies: + node-abort-controller: "npm:^3.0.1" + checksum: 10c0/6664a1b7807ec0ceb223d8f50bf087ac9a6df6356329c47b4374c3a7e4cbcb24a3045d45735dbc74422fc399657baf533e24c0d6528c141c9dd14701880d4fe4 + languageName: node + linkType: hard + "reflect.getprototypeof@npm:^1.0.6, reflect.getprototypeof@npm:^1.0.9": version: 1.0.10 resolution: "reflect.getprototypeof@npm:1.0.10"