wip

Author: brendan-kellam

docs/snippets/schemas/v3/index.schema.mdx

@@ -43,8 +43,7 @@
         "maxConnectionSyncJobConcurrency": {
           "type": "number",
           "description": "The number of connection sync jobs to run concurrently. Defaults to 2.",
-          "minimum": 1,
-          "deprecated": true
+          "minimum": 1
         },
         "maxRepoIndexingJobConcurrency": {
           "type": "number",
@@ -230,8 +229,7 @@
         "maxConnectionSyncJobConcurrency": {
           "type": "number",
           "description": "The number of connection sync jobs to run concurrently. Defaults to 2.",
-          "minimum": 1,
-          "deprecated": true
+          "minimum": 1
         },
         "maxRepoIndexingJobConcurrency": {
           "type": "number",

packages/backend/src/connectionWorkload.ts

@@ -1,14 +1,22 @@
-import { Workload } from "./types.js";
-import { prisma } from "./prisma.js";
+import { Settings, Workload } from "./types.js";
 import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type";
 import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js";
 import { CONNECTION_QUEUE, env, loadConfig } from "@sourcebot/shared";
 import { syncSearchContexts } from "./ee/syncSearchContexts.js";
 import * as Sentry from "@sentry/node";
+import { PrismaClient } from "@sourcebot/db";
 
-export const connectionWorkload: Workload<'connection'> = {
+interface Props {
+    db: PrismaClient,
+    settings: Settings;
+}
+
+export const createConnectionWorkload = ({
+    db,
+    settings
+}: Props): Workload<'connection'> => ({
     queueSpec: CONNECTION_QUEUE,
-    concurrency: 2,
+    concurrency: settings.maxConnectionSyncJobConcurrency,
     process: async ({
         data: {
             connectionId,
@@ -21,7 +29,7 @@ export const connectionWorkload: Workload<'connection'> = {
             connectionId,
             orgId,
         });
-        const connection = await prisma.connection.findUniqueOrThrow({
+        const connection = await db.connection.findUniqueOrThrow({
             where: {
                 id: connectionId
             }
@@ -55,7 +63,7 @@ export const connectionWorkload: Workload<'connection'> = {
         // captured by the connection's config (e.g., it was deleted, marked archived, etc.), it won't
         // appear in the repoData array above, and so the RepoToConnection record won't be re-created.
         // Repos that have no RepoToConnection records are considered orphaned and can be deleted.
-        await prisma.$transaction(async (tx) => {
+        await db.$transaction(async (tx) => {
             const deleteStart = performance.now();
             await tx.connection.update({
                 where: {
@@ -104,7 +112,7 @@ export const connectionWorkload: Workload<'connection'> = {
             });
         }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS });
 
-        await prisma.connection.update({
+        await db.connection.update({
             where: {
                 id: connectionId,
             },
@@ -131,7 +139,7 @@ export const connectionWorkload: Workload<'connection'> = {
             connectionId,
         });
     }
-}
+})
 
 const discoverConnectionRepositories = async ({
     config,

packages/backend/src/index.ts

@@ -13,7 +13,7 @@ import { prisma } from "./prisma.js";
 import { PromClient } from './promClient.js';
 import { createReconciliationWorkload } from "./reconciliationWorkload.js";
 import { redis } from "./redis.js";
-import { connectionWorkload } from "./connectionWorkload.js";
+import { connectionWorkload, createConnectionWorkload } from "./connectionWorkload.js";
 import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js";
 import { Api } from "./api.js";
 
@@ -51,6 +51,10 @@ const reconciliationWorkload = createReconciliationWorkload({
     db: prisma,
     settings,
 });
+const connectionWorkload = createConnectionWorkload({
+    db: prisma,
+    settings,
+});
 const repoIndexWorkload = createRepoIndexWorkload({
     db: prisma,
     settings,

packages/backend/src/repoIndexWorkload.ts

@@ -13,15 +13,15 @@ import { cleanupTempShards, indexGitRepository } from './zoekt.js';
 const LOG_TAG = 'repo-index-workload';
 const logger = createLogger(LOG_TAG);
 
-interface RepoIndexWorkloadDependencies {
+interface Props {
     db: PrismaClient;
     settings: Settings;
 }
 
 export const createRepoIndexWorkload = ({
     db,
     settings,
-}: RepoIndexWorkloadDependencies): Workload<'repo-index'> => ({
+}: Props): Workload<'repo-index'> => ({
     queueSpec: REPO_INDEX_QUEUE,
     concurrency: settings.maxRepoIndexingJobConcurrency,
     process: async ({ data, logger: jobLogger, signal }) => {

packages/schemas/src/v3/index.schema.ts

@@ -42,8 +42,7 @@ const schema = {
         "maxConnectionSyncJobConcurrency": {
           "type": "number",
           "description": "The number of connection sync jobs to run concurrently. Defaults to 2.",
-          "minimum": 1,
-          "deprecated": true
+          "minimum": 1
         },
         "maxRepoIndexingJobConcurrency": {
           "type": "number",
@@ -229,8 +228,7 @@ const schema = {
         "maxConnectionSyncJobConcurrency": {
           "type": "number",
           "description": "The number of connection sync jobs to run concurrently. Defaults to 2.",
-          "minimum": 1,
-          "deprecated": true
+          "minimum": 1
         },
         "maxRepoIndexingJobConcurrency": {
           "type": "number",

packages/schemas/src/v3/index.type.ts

@@ -110,7 +110,6 @@ export interface Settings {
    */
   reindexRepoPollingIntervalMs?: number;
   /**
-   * @deprecated
    * The number of connection sync jobs to run concurrently. Defaults to 2.
    */
   maxConnectionSyncJobConcurrency?: number;

schemas/v3/index.json

@@ -41,8 +41,7 @@
                 "maxConnectionSyncJobConcurrency": {
                     "type": "number",
                     "description": "The number of connection sync jobs to run concurrently. Defaults to 2.",
-                    "minimum": 1,
-                    "deprecated": true
+                    "minimum": 1
                 },
                 "maxRepoIndexingJobConcurrency": {
                     "type": "number",