Fix resumable MCP backfill fold progress

DEVELOPER_GUIDE.md

@@ -112,7 +112,11 @@ without mutating the DB or stamping FoldReady. The MCP `backfill_fold` tool
 accepts the same preview as `dry_run: true`, and also accepts `force: true` to
 rewrite all folded keys when an operator needs to recover from suspicious fold
 metadata or row state even though the stored version/fingerprint appears
-current.
+current. Non-dry-run row rewrites are resumable after interruption: completed
+row updates remain durable, and the final FoldReady metadata is stamped only
+after verification succeeds. MCP responses include `progress.rows_done`,
+`progress.rows_total`, and `progress.fraction` so clients can report and retry
+long backfills.
 
 ## Filesystem Permissions
 

README.md

@@ -74,7 +74,7 @@ After the first command, use these cues and follow-up commands:
 | Intentional rebuilds | Interactive terminals ask before deleting the DB. Scripts and CI must pass `--yes` or `--force`. |
 | Long-lived DB compaction | Run `cdidx optimize` or `cdidx index <projectPath> --optimize` to compact FTS5 segments immediately. Incremental refreshes also optimize opportunistically. |
 | Pathological generated files | `--max-symbols-per-file <n>` skips indexing file content, symbols, and references when one file emits too many symbols, leaving a `symbol_count_exceeded` issue for audit. |
-| Maintenance rollback | Run `cdidx db checkpoint <name>` before risky DB maintenance and `cdidx db restore <name>` to roll back. `backfill-fold` creates an automatic checkpoint unless `--no-checkpoint` is passed. |
+| Maintenance rollback | Run `cdidx db checkpoint <name>` before risky DB maintenance and `cdidx db restore <name>` to roll back. `backfill-fold` creates an automatic checkpoint unless `--no-checkpoint` is passed, and interrupted folded-key rewrites resume from remaining rows. |
 | Permission or I/O scan errors | `cdidx` records the scan error, continues other directories, and writes `.cdidx/scan-checkpoint.json` so same-HEAD retries can skip completed directories. |
 
 Output controls:
@@ -339,7 +339,7 @@ extractor fixture を確認できます。詳細は
 | 意図的な再構築 | interactive terminal では既存 DB 削除前に確認を求めます。script / CI では `--yes` または `--force` が必要です。 |
 | 長期間使っている DB の compact | `cdidx optimize` または `cdidx index <projectPath> --optimize` で FTS5 segment をすぐに compact できます。差分更新中も必要に応じて自動 optimize します。 |
 | 病的な generated file | 1 ファイルが過剰な symbol を出す場合、`--max-symbols-per-file <n>` は file content / symbols / references を保存せず、監査用の `symbol_count_exceeded` issue を残します。 |
-| 保守作業の rollback | risky な DB 保守の前に `cdidx db checkpoint <name>`、戻す場合は `cdidx db restore <name>` を使います。`backfill-fold` は `--no-checkpoint` を渡さない限り自動 checkpoint を作成します。 |
+| 保守作業の rollback | risky な DB 保守の前に `cdidx db checkpoint <name>`、戻す場合は `cdidx db restore <name>` を使います。`backfill-fold` は `--no-checkpoint` を渡さない限り自動 checkpoint を作成し、中断された folded-key rewrite は残り行から再開します。 |
 | 権限や I/O の scan error | `cdidx` は scan error を記録し、他のディレクトリの走査を続けます。同じ HEAD の再実行では `.cdidx/scan-checkpoint.json` により成功済みディレクトリを読み飛ばせます。 |
 
 出力を整える option:

changelog.d/unreleased/1461.fixed.md

@@ -0,0 +1,18 @@
+---
+category: fixed
+issues:
+  - 1461
+affected:
+  - src/CodeIndex/Database/DbWriter.cs
+  - src/CodeIndex/Mcp/McpToolHandlers.cs
+  - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs
+  - tests/CodeIndex.Tests/McpServerTests.cs
+---
+
+## English
+
+- **MCP `backfill_fold` can report progress and resume interrupted row rewrites (#1461)** — folded-key row updates are durable before the final FoldReady stamp, and MCP responses now include `progress.rows_done`, `progress.rows_total`, and `progress.fraction`.
+
+## 日本語
+
+- **MCP `backfill_fold` が進捗を返し、中断された行 rewrite を再開できるようになりました (#1461)** — folded-key 行更新は最終 FoldReady stamp の前に永続化され、MCP 応答に `progress.rows_done`、`progress.rows_total`、`progress.fraction` を含めます。

src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs

@@ -195,13 +195,13 @@ internal static int RunBackfillFold(
                 if (!options.NoCheckpoint)
                     DbCommandRunner.CreateAutomaticCheckpoint(options.DbPath);
 
-                using var transaction = writer.BeginTransaction();
                 (symbols, symbolReferences) = writer.BackfillFoldedColumns(
                     rewriteAll,
                     backfillCancellation.Token);
-                // MarkFoldReady re-verifies in the same transaction so rows, metadata, and
-                // FoldReady stamp commit or roll back together.
-                // 同一 transaction 内で再検証し、行・metadata・FoldReady stamp を原子的に扱う。
+                // Row rewrites commit before the final FoldReady stamp so interrupted
+                // backfills can resume from the remaining rows.
+                // 行更新は FoldReady stamp より前に永続化し、中断後に残り行から再開できるようにする。
+                using var transaction = writer.BeginTransaction();
                 verified = writer.MarkFoldReady();
                 if (!verified)
                 {

src/CodeIndex/Database/DbWriter.cs

@@ -15,6 +15,10 @@ namespace CodeIndex.Database;
 /// </summary>
 public class DbWriter
 {
+    private const string FoldBackfillPhaseMetaKey = "fold_backfill_phase";
+    private const string FoldBackfillLastSymbolIdMetaKey = "fold_backfill_last_symbol_id";
+    private const string FoldBackfillLastReferenceIdMetaKey = "fold_backfill_last_reference_id";
+
     public const string FtsIncrementalWritesSinceOptimizeMetaKey = "fts_incremental_writes_since_optimize";
     public const string FtsLastOptimizedAtMetaKey = "fts_last_optimized_at";
     public const int DefaultFtsOptimizeIncrementalWriteThreshold = 25;
@@ -3287,28 +3291,47 @@ private bool SymbolExtractorVersionMatchesCurrent(string? lang)
         CancellationToken cancellationToken = default)
     {
         cancellationToken.ThrowIfCancellationRequested();
-        using var txn = !IsInTransaction() ? BeginTransaction() : null;
+        var foldBackfillPhase = rewriteAll ? GetMetaString(FoldBackfillPhaseMetaKey) : null;
         var symbols = BackfillSymbolFoldedRows(rewriteAll, cancellationToken);
+        if (rewriteAll && foldBackfillPhase != "references")
+        {
+            SetMeta(FoldBackfillPhaseMetaKey, "references");
+            SetMeta(FoldBackfillLastReferenceIdMetaKey, "0");
+        }
+
         var symbolReferences = BackfillReferenceFoldedRows(rewriteAll, cancellationToken);
         cancellationToken.ThrowIfCancellationRequested();
-        txn?.Commit();
+        if (rewriteAll)
+            ClearFoldBackfillCheckpoint();
+
         return (symbols, symbolReferences);
     }
 
     public (int Symbols, int SymbolReferences) CountBackfillFoldedColumns(bool rewriteAll = false)
     {
+        var phase = rewriteAll ? GetMetaString(FoldBackfillPhaseMetaKey) : null;
+        var lastSymbolId = rewriteAll ? GetFoldBackfillCheckpoint(FoldBackfillLastSymbolIdMetaKey) : 0;
+        var lastReferenceId = rewriteAll ? GetFoldBackfillCheckpoint(FoldBackfillLastReferenceIdMetaKey) : 0;
+
         using var symbols = _conn.CreateCommand();
-        symbols.CommandText = rewriteAll
-            ? "SELECT COUNT(*) FROM symbols WHERE name IS NOT NULL"
+        symbols.CommandText = rewriteAll && phase != "references"
+            ? "SELECT COUNT(*) FROM symbols WHERE name IS NOT NULL AND id > @lastSymbolId"
+            : rewriteAll
+            ? "SELECT 0"
             : "SELECT COUNT(*) FROM symbols WHERE name IS NOT NULL AND name_folded IS NULL";
+        symbols.Parameters.AddWithValue("@lastSymbolId", lastSymbolId);
 
         using var references = _conn.CreateCommand();
         references.CommandText = rewriteAll
-            ? "SELECT COUNT(*) FROM symbol_references WHERE symbol_name IS NOT NULL OR container_name IS NOT NULL"
+            ? @"SELECT COUNT(*)
+                FROM symbol_references
+                WHERE id > @lastReferenceId
+                  AND (symbol_name IS NOT NULL OR container_name IS NOT NULL)"
             : @"SELECT COUNT(*)
                 FROM symbol_references
                 WHERE (symbol_name IS NOT NULL AND symbol_name_folded IS NULL)
                    OR (container_name IS NOT NULL AND container_name_folded IS NULL)";
+        references.Parameters.AddWithValue("@lastReferenceId", phase == "references" ? lastReferenceId : 0);
 
         return (ToInt32Count(symbols.ExecuteScalar()), ToInt32Count(references.ExecuteScalar()));
     }
@@ -3321,12 +3344,18 @@ private static int ToInt32Count(object? value)
 
     private int BackfillSymbolFoldedRows(bool rewriteAll, CancellationToken cancellationToken)
     {
+        var phase = rewriteAll ? GetMetaString(FoldBackfillPhaseMetaKey) : null;
+        if (phase == "references")
+            return 0;
+
+        var lastSymbolId = rewriteAll ? GetFoldBackfillCheckpoint(FoldBackfillLastSymbolIdMetaKey) : 0;
         var rows = new List<(long Id, string Name)>();
         using (var cmd = _conn.CreateCommand())
         {
             cmd.CommandText = rewriteAll
-                ? "SELECT id, name FROM symbols WHERE name IS NOT NULL"
+                ? "SELECT id, name FROM symbols WHERE name IS NOT NULL AND id > @lastSymbolId ORDER BY id"
                 : "SELECT id, name FROM symbols WHERE name IS NOT NULL AND name_folded IS NULL";
+            cmd.Parameters.AddWithValue("@lastSymbolId", lastSymbolId);
             using var reader = cmd.ExecuteTrackedReader();
             while (reader.TrackedRead())
             {
@@ -3350,6 +3379,8 @@ private int BackfillSymbolFoldedRows(bool rewriteAll, CancellationToken cancella
             pFolded.Value = (object?)NameFold.Fold(row.Name) ?? DBNull.Value;
             pId.Value = row.Id;
             update.ExecuteNonQuery();
+            if (rewriteAll)
+                SetMeta(FoldBackfillLastSymbolIdMetaKey, row.Id.ToString(System.Globalization.CultureInfo.InvariantCulture));
             FoldBackfillRowUpdatedForTesting?.Invoke();
         }
 
@@ -3358,15 +3389,21 @@ private int BackfillSymbolFoldedRows(bool rewriteAll, CancellationToken cancella
 
     private int BackfillReferenceFoldedRows(bool rewriteAll, CancellationToken cancellationToken)
     {
+        var lastReferenceId = rewriteAll ? GetFoldBackfillCheckpoint(FoldBackfillLastReferenceIdMetaKey) : 0;
         var rows = new List<(long Id, string? SymbolName, string? ContainerName)>();
         using (var cmd = _conn.CreateCommand())
         {
             cmd.CommandText = rewriteAll
-                ? "SELECT id, symbol_name, container_name FROM symbol_references WHERE symbol_name IS NOT NULL OR container_name IS NOT NULL"
+                ? @"SELECT id, symbol_name, container_name
+                    FROM symbol_references
+                    WHERE id > @lastReferenceId
+                      AND (symbol_name IS NOT NULL OR container_name IS NOT NULL)
+                    ORDER BY id"
                 : @"SELECT id, symbol_name, container_name
                     FROM symbol_references
                     WHERE (symbol_name IS NOT NULL AND symbol_name_folded IS NULL)
                        OR (container_name IS NOT NULL AND container_name_folded IS NULL)";
+            cmd.Parameters.AddWithValue("@lastReferenceId", lastReferenceId);
             using var reader = cmd.ExecuteTrackedReader();
             while (reader.TrackedRead())
             {
@@ -3398,12 +3435,29 @@ FROM symbol_references
             pContainerNameFolded.Value = (object?)NameFold.Fold(row.ContainerName) ?? DBNull.Value;
             pId.Value = row.Id;
             update.ExecuteNonQuery();
+            if (rewriteAll)
+                SetMeta(FoldBackfillLastReferenceIdMetaKey, row.Id.ToString(System.Globalization.CultureInfo.InvariantCulture));
             FoldBackfillRowUpdatedForTesting?.Invoke();
         }
 
         return rows.Count;
     }
 
+    private long GetFoldBackfillCheckpoint(string key)
+    {
+        var value = GetMetaString(key);
+        return long.TryParse(value, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var parsed)
+            ? parsed
+            : 0;
+    }
+
+    private void ClearFoldBackfillCheckpoint()
+    {
+        SetMeta(FoldBackfillPhaseMetaKey, null);
+        SetMeta(FoldBackfillLastSymbolIdMetaKey, null);
+        SetMeta(FoldBackfillLastReferenceIdMetaKey, null);
+    }
+
     private static object FoldedNameDbValue(string? name, Dictionary<string, string?> cache)
     {
         if (name == null)

src/CodeIndex/Mcp/McpToolHandlers.cs

@@ -3817,22 +3817,26 @@ private JsonNode ExecuteBackfillFold(JsonNode? id, JsonNode? args, JsonNode? pro
                 || !foldMetadataCurrentBefore;
             var symbols = 0;
             var symbolReferences = 0;
+            var totalSymbols = 0;
+            var totalSymbolReferences = 0;
             var verified = false;
             var userVersionAfter = userVersionBefore;
 
+            (totalSymbols, totalSymbolReferences) = writer.CountBackfillFoldedColumns(rewriteAll);
             if (dryRun)
             {
-                (symbols, symbolReferences) = writer.CountBackfillFoldedColumns(rewriteAll);
+                symbols = totalSymbols;
+                symbolReferences = totalSymbolReferences;
             }
             else
             {
                 EmitProgressNotification(progressToken, 0, null, "Backfilling folded-name keys.");
-                using var transaction = writer.BeginTransaction();
                 (symbols, symbolReferences) = writer.BackfillFoldedColumns(rewriteAll);
-                EmitProgressNotification(progressToken, symbols + symbolReferences, null, "Verifying folded-name keys.");
-                // Verify and stamp in the same transaction as the row rewrite so crash recovery
-                // never leaves current fold metadata without a matching FoldReady stamp.
-                // 行の再生成と同じ transaction で検証・stamp し、metadata だけが先に残らないようにする。
+                EmitProgressNotification(progressToken, symbols + symbolReferences, totalSymbols + totalSymbolReferences, "Verifying folded-name keys.");
+                // Row rewrites are intentionally committed before the final FoldReady stamp so
+                // interrupted MCP backfills can resume from the remaining rows.
+                // 行更新は FoldReady stamp より前に永続化し、中断後に残り行から再開できるようにする。
+                using var transaction = writer.BeginTransaction();
                 verified = writer.MarkFoldReady();
                 if (!verified)
                     return CreateToolErrorResponse(id, "Folded-name backfill verification failed: some rows still have NULL folded values. Re-run backfill_fold.");
@@ -3867,6 +3871,7 @@ private JsonNode ExecuteBackfillFold(JsonNode? id, JsonNode? args, JsonNode? pro
                 ["fold_key_version_after"] = dryRun ? storedFoldVersion : currentFoldVersion,
                 ["fold_key_fingerprint_before"] = storedFoldFingerprint,
                 ["fold_key_fingerprint_after"] = dryRun ? storedFoldFingerprint : currentFoldFingerprint,
+                ["progress"] = BuildBackfillProgressJson(symbols + symbolReferences, totalSymbols + totalSymbolReferences),
             };
 
             var summary = dryRun
@@ -3882,6 +3887,17 @@ private JsonNode ExecuteBackfillFold(JsonNode? id, JsonNode? args, JsonNode? pro
         }
     }
 
+    private static JsonObject BuildBackfillProgressJson(int rowsDone, int rowsTotal)
+    {
+        var fraction = rowsTotal <= 0 ? 1.0 : Math.Min(1.0, rowsDone / (double)rowsTotal);
+        return new JsonObject
+        {
+            ["rows_done"] = rowsDone,
+            ["rows_total"] = rowsTotal,
+            ["fraction"] = fraction,
+        };
+    }
+
     /// <summary>
     /// Maximum length for suggestion description text.
     /// 提案説明テキストの最大長。