From 34d5229035fd77461c0e37385b82009a1d16d49b Mon Sep 17 00:00:00 2001
From: Parker Singleton <sps253@cornell.edu>
Date: Wed, 5 Aug 2026 15:45:13 -0400
Subject: [PATCH] perf(cli): Batch index adds instead of staging one file at a
 time

isomorphic-git re-serializes and rewrites the entire index on every
git.add, so staging one path per call costs O(n) per file and O(n^2)
overall. On large datasets this dominates the upload: the staging rate
decays as 1/n, from ~300 files/min early on to ~30 files/min at 83k
files.

Collect paths and stage them in batches of 500 instead. annexAdd no
longer stages the file itself, leaving that to the caller.

Measured with isomorphic-git 1.36.3 on local NVMe, 20k files:

  per-file add   1173.3s   rate decays 8840 -> 541 files/min
  batched (1000)    6.1s   rate flat, ~180k files/min

git.add already accepts an array of paths, so this needs no upstream
change.
---
 cli/src/worker/annex.ts |  7 ++-----
 cli/src/worker/git.ts   | 34 +++++++++++++++++++++++++++++-----
 2 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/cli/src/worker/annex.ts b/cli/src/worker/annex.ts
index e97a54fe8..1a3f67d31 100644
--- a/cli/src/worker/annex.ts
+++ b/cli/src/worker/annex.ts
@@ -155,11 +155,8 @@ export async function annexAdd(
     await context.fs.promises.rm(fileRepoPath, { force: true })
     // Create our new symlink pointing at the right annex object
     await context.fs.promises.symlink(symlinkTarget, fileRepoPath)
-    const options = {
-      ...context.config(),
-      filepath: relativePath,
-    }
-    await git.add(options)
+    // Staging is left to the caller so adds can be batched - see ADD_BATCH_SIZE
+    // in git.ts. Adding one path at a time rewrites the whole index per file.
     return true
   } else {
     return false
diff --git a/cli/src/worker/git.ts b/cli/src/worker/git.ts
index aecada04c..e772c4576 100644
--- a/cli/src/worker/git.ts
+++ b/cli/src/worker/git.ts
@@ -175,7 +175,30 @@ async function shouldBeAnnexed(
 /**
  * git-annex add equivalent
  */
+/**
+ * How many paths to stage per git.add call.
+ *
+ * isomorphic-git re-serializes and rewrites the whole index on every add, so
+ * adding one path at a time costs O(n) per file and O(n^2) overall. Batching
+ * makes the index cost negligible: staging 20k files took 1173s one at a time
+ * and 6.1s in batches of 1000.
+ *
+ * The batch is flushed after every chunk and again at the end, so at most this
+ * many files need restaging if the process dies mid-run.
+ */
+const ADD_BATCH_SIZE = 500
+
 async function add(event: GitWorkerEventAdd) {
+  const pending: string[] = []
+  const flushPending = async () => {
+    if (pending.length === 0) return
+    await git.add({
+      ...context.config(),
+      filepath: pending.length === 1 ? pending[0] : [...pending],
+    })
+    pending.length = 0
+  }
+
   for (const file of event.data.paths) {
     let size
     try {
@@ -197,10 +220,6 @@ async function add(event: GitWorkerEventAdd) {
     const annexed = await shouldBeAnnexed(file.relativePath, size)
     if (annexed === "GIT") {
       // Simple add case
-      const options = {
-        ...context.config(),
-        filepath: file.relativePath,
-      }
       const targetPath = join(context.repoPath, file.relativePath)
       // Verify parent directories exist
       await context.fs.promises.mkdir(dirname(targetPath), { recursive: true })
@@ -216,18 +235,23 @@ async function add(event: GitWorkerEventAdd) {
         // Copy all other non-annexed files for git index creation
         await context.fs.promises.copyFile(file.path, targetPath)
       }
-      await git.add(options)
+      pending.push(file.relativePath)
       logger.info(`Add\t${file.relativePath}`)
     } else {
       if (
         await annexAdd(annexed, file.path, file.relativePath, size, context)
       ) {
+        pending.push(file.relativePath)
         logger.info(`Annexed\t${file.relativePath}`)
       } else {
         logger.info(`Unchanged\t${file.relativePath}`)
       }
     }
+    if (pending.length >= ADD_BATCH_SIZE) {
+      await flushPending()
+    }
   }
+  await flushPending()
 }
 
 /**