From df706ac9592bc34a00a19b8f8144a8bb1a80ec4b Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 30 Jul 2026 12:26:35 +0900 Subject: [PATCH 1/2] Fix #4889 skip checkpoints for no-op backfills --- DEVELOPER_GUIDE.md | 8 +- TESTING_GUIDE.md | 2 + changelog.d/unreleased/4889.fixed.md | 17 ++ src/CodeIndex/Cli/CliFlagSchema.cs | 3 +- src/CodeIndex/Cli/ConsoleUi.cs | 2 +- .../Cli/IndexCommandRunner.Maintenance.cs | 118 +++++++--- src/CodeIndex/Cli/IndexCommandRunner.cs | 1 + src/CodeIndex/Cli/JsonOutputContracts.cs | 3 + .../Database/DbWriter.FoldBackfill.cs | 6 + tests/CodeIndex.Tests/ConsoleUiTests.cs | 2 +- .../IndexCommandRunnerTests.cs | 212 +++++++++++++++++- 11 files changed, 330 insertions(+), 44 deletions(-) create mode 100644 changelog.d/unreleased/4889.fixed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 47ec758c5..a0aade5dd 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -50,9 +50,9 @@ Development contracts: | Read-only database queries | `cdidx status --db /artifacts/codeindex.db --read-only --json`; `cdidx search AuthService --db /artifacts/codeindex.db --immutable` | Query commands accept `--read-only` (alias `--immutable`) to open an existing CodeIndex database through SQLite's immutable read-only URI mode. Use this for CI artifacts, mounted caches, and sandboxes where creating or updating `codeindex.db-wal` / `codeindex.db-shm` sidecars is not allowed. | | Mutating commands | `index`, `backfill-fold`, `optimize`, `vacuum` | These require writable storage and reject read-only database opens. | | Reusable index artifact | `cdidx export codeindex.cdidx.zip`; `cdidx import codeindex.cdidx.zip --db `; `cdidx import codeindex.cdidx.zip --dry-run --json` | Run export after indexing and upload the archive. Export refuses an existing destination unless `--overwrite` is explicit, publishes atomically from an owner-only temporary file, and verifies POSIX mode `0600`. Successful export JSON adds final archive byte size and SHA-256 plus the complete immutable manifest while retaining the prior result fields. Consumers import before query commands, or use `--dry-run` / `--check` to validate the archive without replacing the destination DB. Use `--prune-paths` when the archive comes from another checkout and the restored DB should advertise the import target project root; imports targeting `.../.cdidx/codeindex.db` use the sibling project directory, while other DB paths fall back to the process current directory. The archive contains only `manifest.json` plus `codeindex.db`; import validates ZIP entry names through `ZipArchiveSafetyPolicy` and rejects absolute, parent-directory, backslash, NUL, non-canonical, duplicate, and extra entries before extraction. The manifest carries bounded summary/readiness metadata including row counts, readiness bits, writer/indexed-head metadata, schema contract stamps, and unknown-extension summary when available. Import validates manifest format, manifest `user_version`, `database_sha256`, present summary counts, and the embedded SQLite file as a CodeIndex database before replacing the destination DB. Import rejects archive `codeindex.db` entries whose compressed or uncompressed metadata exceeds 8 GiB, and the extraction stream is also capped at 8 GiB. | -| Maintenance checkpoint and managed rollback | `cdidx db checkpoint [--dry-run]`; `cdidx db checkpoints --list|--delete |--prune --keep [--dry-run]`; `cdidx db restore [--dry-run] [--no-backup]`; `cdidx db restore-backups --list|--prune --keep |--restore [--dry-run] [--no-backup]` | Checkpoint snapshots `codeindex.db` plus existing WAL/SHM sidecars before risky maintenance. Import and both restore forms create a consistent, verified managed SQLite rollback snapshot before replacing an existing DB unless `--no-backup` is explicit. Managed directories use `.restore-backup-/` and contain a bounded manifest plus one standalone database payload; the manifest records SHA-256, byte count, supported `user_version`, provenance, and an optional source identifier, but no local absolute source path. `restore-backups --list` exposes the ID and provenance while retaining legacy directory metadata; existing prune retention remains compatible. `restore-backups --restore ` revalidates the directory boundary, manifest, payload hash, schema, and combined staging/rollback free space, then performs an atomic replacement with transient rollback-on-failure. Its `--dry-run` reports every validation and planned backup without mutation. Checkpoint delete/prune and restore-backup prune require an explicit mutation action, and checkpoint prune skips all deletion if its bounded 1,000-directory scan is truncated. Checkpoints live under `.checkpoints//`. `backfill-fold` creates an automatic checkpoint before row mutation unless `--no-checkpoint` is passed. | +| Maintenance checkpoint and managed rollback | `cdidx db checkpoint [--dry-run]`; `cdidx db checkpoints --list|--delete |--prune --keep [--dry-run]`; `cdidx db restore [--dry-run] [--no-backup]`; `cdidx db restore-backups --list|--prune --keep |--restore [--dry-run] [--no-backup]` | Checkpoint snapshots `codeindex.db` plus existing WAL/SHM sidecars before risky maintenance. Import and both restore forms create a consistent, verified managed SQLite rollback snapshot before replacing an existing DB unless `--no-backup` is explicit. Managed directories use `.restore-backup-/` and contain a bounded manifest plus one standalone database payload; the manifest records SHA-256, byte count, supported `user_version`, provenance, and an optional source identifier, but no local absolute source path. `restore-backups --list` exposes the ID and provenance while retaining legacy directory metadata; existing prune retention remains compatible. `restore-backups --restore ` revalidates the directory boundary, manifest, payload hash, schema, and combined staging/rollback free space, then performs an atomic replacement with transient rollback-on-failure. Its `--dry-run` reports every validation and planned backup without mutation. Checkpoint delete/prune and restore-backup prune require an explicit mutation action, and checkpoint prune skips all deletion if its bounded 1,000-directory scan is truncated. Checkpoints live under `.checkpoints//`. `backfill-fold` preflights folded rows and readiness under the index lock, creates an automatic checkpoint only when a mutation is required, accepts `--checkpoint` to force a snapshot for an already-complete DB, and accepts `--no-checkpoint` to skip mutation protection explicitly. JSON reports `checkpoint_skipped` plus `checkpoint_skipped_reason` (`already_complete`, `dry_run`, or `disabled_by_option`), and human output reports the same decision. | | Binary compatibility | [COMPATIBILITY.md](COMPATIBILITY.md) | Database compatibility across `cdidx` binary upgrades and downgrades is documented there. Keep that policy updated whenever readiness bits, `codeindex_meta` contract stamps, or rebuild requirements change. | -| Fold backfill preview and recovery | `backfill-fold --dry-run`; MCP `backfill_fold` with `dry_run: true` or `force: true` | Dry-run previews folded-key rows without mutating the DB or stamping FoldReady. MCP accepts the same preview and can force rewriting all folded keys when an operator needs to recover from suspicious fold metadata or row state even though the stored version/fingerprint appears current. Non-dry-run row rewrites are resumable after interruption: completed row updates remain durable, and final FoldReady metadata is stamped only after verification succeeds. MCP responses include `progress.rows_done`, `progress.rows_total`, and `progress.fraction` so clients can report and retry long backfills. | +| Fold backfill preview and recovery | `backfill-fold --dry-run`; `backfill-fold --checkpoint`; MCP `backfill_fold` with `dry_run: true` or `force: true` | Dry-run previews folded-key rows without mutating the DB or stamping FoldReady. `--checkpoint` explicitly preserves a snapshot even when CLI preflight finds no mutation, while the default completed no-op leaves no checkpoint artifacts. MCP accepts the same preview and can force rewriting all folded keys when an operator needs to recover from suspicious fold metadata or row state even though the stored version/fingerprint appears current. Non-dry-run row rewrites are resumable after interruption: completed row updates remain durable, and final FoldReady metadata is stamped only after verification succeeds. MCP responses include `progress.rows_done`, `progress.rows_total`, and `progress.fraction` so clients can report and retry long backfills. | ## Filesystem Permissions @@ -3394,9 +3394,9 @@ net9 CI lane に合わせる場合は `FRAMEWORK=net9.0 make test` を使いま | read-only database query | `cdidx status --db /artifacts/codeindex.db --read-only --json`; `cdidx search AuthService --db /artifacts/codeindex.db --immutable` | query コマンドは `--read-only`（alias: `--immutable`）を受け付け、既存の CodeIndex database を SQLite の immutable read-only URI mode で開けます。CI artifact、mounted cache、`codeindex.db-wal` / `codeindex.db-shm` sidecar を作成・更新できない sandbox で使います。 | | 変更系コマンド | `index`、`backfill-fold`、`optimize`、`vacuum` | 書き込み可能な storage を必要とし、read-only database open を拒否します。 | | 再利用可能な index artifact | `cdidx export codeindex.cdidx.zip`; `cdidx import codeindex.cdidx.zip --db `; `cdidx import codeindex.cdidx.zip --dry-run --json` | CI job では index 後に export して archive を upload します。export は `--overwrite` を明示しない限り既存 destination を拒否し、owner-only temporary file から atomic に publish して POSIX mode `0600` を検証します。export 成功時の JSON は従来 field を維持し、最終 archive の byte 数と SHA-256、完全で immutable な manifest を追加します。利用側は query コマンドの前に import でき、`--dry-run` / `--check` で destination DB を置き換えず archive を検証できます。別 checkout 由来の archive を import 先 project root として扱いたい場合は `--prune-paths` を使います。`.../.cdidx/codeindex.db` を import 先にした場合は sibling の project directory を使い、それ以外の DB path では process current directory に fallback します。archive は `manifest.json` と `codeindex.db` だけを含みます。import は ZIP entry 名を `ZipArchiveSafetyPolicy` で検証し、absolute path、parent-directory segment、backslash、NUL、non-canonical name、duplicate entry、extra entry を extraction 前に拒否します。manifest は row count、readiness bit、writer / indexed-head metadata、schema contract stamp、利用可能な unknown-extension summary などの bounded summary/readiness metadata を持ちます。import は manifest format、manifest `user_version`、`database_sha256`、存在する summary count、embedded SQLite file が CodeIndex database であることを検証してから destination DB を置き換えます。archive の `codeindex.db` entry は compressed / uncompressed metadata と extraction stream の双方で 8 GiB を上限に拒否されます。 | -| maintenance checkpoint と managed rollback | `cdidx db checkpoint [--dry-run]`; `cdidx db checkpoints --list|--delete |--prune --keep [--dry-run]`; `cdidx db restore [--dry-run] [--no-backup]`; `cdidx db restore-backups --list|--prune --keep |--restore [--dry-run] [--no-backup]` | 危険な maintenance の前に `codeindex.db` と既存 WAL/SHM sidecar の checkpoint を作成できます。import と2種類の restore は、既存 DB を置き換える前に consistent かつ検証済みの managed SQLite rollback snapshot を既定で作成し、`--no-backup` を明示した場合だけ省略します。managed directory は `.restore-backup-/` で、bounded manifest と standalone database payload 1個を含みます。manifest は SHA-256、byte 数、対応する `user_version`、provenance、任意の source identifier を記録しますが、local absolute source path は記録しません。`restore-backups --list` は従来 directory metadata との互換性を維持しつつ ID と provenance を表示し、既存 prune retention もそのまま利用できます。`restore-backups --restore ` は directory 境界、manifest、payload hash、schema、staging と rollback を合わせた free space を再検証してから、失敗時の transient rollback を伴う atomic replacement を実行します。`--dry-run` は変更せず、すべての検証と作成予定 backup を報告します。checkpoint の delete / prune と restore-backup の prune は明示的な変更 action を必要とし、checkpoint prune の bounded scan が truncated の場合は削除をすべて skip します。checkpoint は `.checkpoints//` に置かれ、`backfill-fold` は `--no-checkpoint` がなければ row mutation 前に automatic checkpoint を作ります。 | +| maintenance checkpoint と managed rollback | `cdidx db checkpoint [--dry-run]`; `cdidx db checkpoints --list|--delete |--prune --keep [--dry-run]`; `cdidx db restore [--dry-run] [--no-backup]`; `cdidx db restore-backups --list|--prune --keep |--restore [--dry-run] [--no-backup]` | 危険な maintenance の前に `codeindex.db` と既存 WAL/SHM sidecar の checkpoint を作成できます。import と2種類の restore は、既存 DB を置き換える前に consistent かつ検証済みの managed SQLite rollback snapshot を既定で作成し、`--no-backup` を明示した場合だけ省略します。managed directory は `.restore-backup-/` で、bounded manifest と standalone database payload 1個を含みます。manifest は SHA-256、byte 数、対応する `user_version`、provenance、任意の source identifier を記録しますが、local absolute source path は記録しません。`restore-backups --list` は従来 directory metadata との互換性を維持しつつ ID と provenance を表示し、既存 prune retention もそのまま利用できます。`restore-backups --restore ` は directory 境界、manifest、payload hash、schema、staging と rollback を合わせた free space を再検証してから、失敗時の transient rollback を伴う atomic replacement を実行します。`--dry-run` は変更せず、すべての検証と作成予定 backup を報告します。checkpoint の delete / prune と restore-backup の prune は明示的な変更 action を必要とし、checkpoint prune の bounded scan が truncated の場合は削除をすべて skip します。checkpoint は `.checkpoints//` に置かれます。`backfill-fold` は index lock 内で folded row と readiness を事前確認し、mutation が必要な場合だけ automatic checkpoint を作ります。完了済みDBでもsnapshotを明示的に残すには `--checkpoint`、mutation protectionを明示的に省略するには `--no-checkpoint` を使います。JSON は `checkpoint_skipped` と `checkpoint_skipped_reason`（`already_complete`、`dry_run`、`disabled_by_option`）を返し、human output も同じ判断を表示します。 | | binary compatibility | [COMPATIBILITY.md](COMPATIBILITY.md) | `cdidx` binary の upgrade / downgrade をまたぐ database compatibility を記載します。readiness bit、`codeindex_meta` contract stamp、rebuild requirement を変える場合は、この policy も更新してください。 | -| Fold backfill の preview / recovery | `backfill-fold --dry-run`; MCP `backfill_fold` の `dry_run: true` または `force: true` | dry-run は DB を変更せず FoldReady stamp も書かずに、rewrite 対象の folded-key row をプレビューします。MCP も同じ preview を受け付け、stored version / fingerprint が current に見える場合でも suspicious な fold metadata や row state を復旧するため `force: true` を受け付けます。non-dry-run rewrite は中断後に resume でき、完了済み row update は durable に残り、最終 FoldReady metadata は verification 成功後にだけ stamp されます。MCP response は `progress.rows_done`、`progress.rows_total`、`progress.fraction` を含みます。 | +| Fold backfill の preview / recovery | `backfill-fold --dry-run`; `backfill-fold --checkpoint`; MCP `backfill_fold` の `dry_run: true` または `force: true` | dry-run は DB を変更せず FoldReady stamp も書かずに、rewrite 対象の folded-key row をプレビューします。CLI preflight でmutation不要と判断された場合でもsnapshotを明示的に保存するには `--checkpoint` を使います。既定の完了済みno-opはcheckpoint artifactを作りません。MCP も同じ preview を受け付け、stored version / fingerprint が current に見える場合でも suspicious な fold metadata や row state を復旧するため `force: true` を受け付けます。non-dry-run rewrite は中断後に resume でき、完了済み row update は durable に残り、最終 FoldReady metadata は verification 成功後にだけ stamp されます。MCP response は `progress.rows_done`、`progress.rows_total`、`progress.fraction` を含みます。 | ## ファイルシステム権限 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index c14191ddd..27050f0dc 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -364,6 +364,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding - `IndexCommandRunnerTests.RunOptimizeFts_DryRunPreviewsWithoutWritingThenOptimizeMutates_Issue4577`, `RunOptimizeFts_LockHeld_ReportsDbLocked`, and `RunOptimizeFts_ReadOnlyUri_ReturnsDbNotWritable` share seeded databases to verify that optimize dry-run reports sizes, readiness, lock state, recommendations, and planned work while preserving the source bytes and creating no lock artifacts; the same fixtures retain the writable optimize and lock/read-only URI mutation guards. `ConsoleUiTests.PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags` keeps the optimize dry-run flag visible in full CLI usage. +- Issue #4889 backfill-checkpoint coverage keeps the mutation preflight, index lock, and artifact decision in one contract. Reuse the completed fixture for default no-op, forced `--checkpoint`, `--no-checkpoint` mutation, human output, and a zero-row pending graph refresh; keep WAL/SHM open during the forced snapshot so Windows sharing semantics and the complete DB/WAL/SHM/manifest file set are exercised. Separate dry-run, invalid-DB, and held-lock assertions must prove that no checkpoint directory is created, while a real pending rewrite or pending graph refresh must retain the automatic checkpoint. - Issue #4856 maintenance-error coverage spans `CommandErrorWriterTests`, `QueryCommandRunnerTests`, `IndexCommandRunnerTests`, and `DbCommandRunnerTests`. Keep the SQLite primary-code matrix (`5`, `6`, `8`, `11`, `26`), caller-spelled relative paths, whole-value Unix/Windows/file-URI redaction, missing and invalid-header files, directory/inaccessible preflight classification, integrity corruption rows, lock-holder metadata, the `index --optimize` human preamble, valid JSON, and category-specific recovery hints aligned with classifier version `1`. - `IndexCommandRunnerTests.Run_DryRunWithRebuildAndMemoryTrace_SkipsConfirmationAndPreservesWorkspace_Issue4580` @@ -1303,6 +1304,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - `IndexCommandRunnerTests.RunOptimizeFts_DryRunPreviewsWithoutWritingThenOptimizeMutates_Issue4577`、`RunOptimizeFts_LockHeld_ReportsDbLocked`、`RunOptimizeFts_ReadOnlyUri_ReturnsDbNotWritable` seed 済み database を共有し、optimize dry-run が size、readiness、lock state、推奨、planned work を報告しつつ source byte を保持し、lock artifact を作成しないことを検証する。同じ fixture で、書き込み版 optimize と lock/read-only URI の mutation guard も維持する。 `ConsoleUiTests.PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags` は、full CLI usage に optimize dry-run flag が表示され続けることを固定します。 +- Issue #4889 の backfill-checkpoint coverage は、mutation preflight、index lock、artifact 作成判断を1つの契約として固定します。完了済み fixture を既定no-op、明示 `--checkpoint`、`--no-checkpoint` mutation、human output、zero-rowのpending graph refreshで再利用し、明示snapshot中は WAL/SHM をopenしたままにしてWindowsの共有 semanticsとDB/WAL/SHM/manifest一式を検証してください。dry-run、無効DB、lock保持中のassertionではcheckpoint directoryが作られないことを個別に証明し、実際にpending rewriteまたはpending graph refreshがある場合はautomatic checkpointを維持してください。 - Issue #4856 の maintenance-error coverage は `CommandErrorWriterTests`、`QueryCommandRunnerTests`、`IndexCommandRunnerTests`、`DbCommandRunnerTests` にまたがります。 SQLite primary-code matrix (`5`、`6`、`8`、`11`、`26`)、呼び出し側表記を維持する relative path、Unix / Windows / file URI の値全体 redaction、missing / invalid-header file、directory / inaccessible preflight classification、integrity corruption row、lock-holder metadata、`index --optimize` の human preamble、有効な JSON、category 別 recovery hint を classifier version `1` と同期させてください。 - `IndexCommandRunnerTests.Run_DryRunWithRebuildAndMemoryTrace_SkipsConfirmationAndPreservesWorkspace_Issue4580` diff --git a/changelog.d/unreleased/4889.fixed.md b/changelog.d/unreleased/4889.fixed.md new file mode 100644 index 000000000..2199cf212 --- /dev/null +++ b/changelog.d/unreleased/4889.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 4889 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - DEVELOPER_GUIDE.md +--- + +## English + +- **Completed fold backfills no longer create automatic checkpoint artifacts (#4889)** — `backfill-fold` now preflights pending rows and readiness under the index lock, creates an automatic DB/WAL/SHM/manifest checkpoint only before a real mutation, reports bounded checkpoint-skip reasons in JSON and human output, and accepts `--checkpoint` when an operator explicitly wants a snapshot of an already-complete database. + +## 日本語 + +- **完了済みの fold backfill が automatic checkpoint artifact を作成しなくなりました (#4889)** — `backfill-fold` は index lock 内で pending row と readiness を事前確認し、実際の mutation 前にだけ DB/WAL/SHM/manifest checkpoint を自動作成します。JSON と human output には上限付きの checkpoint skip reason を表示し、完了済みDBでもoperatorが明示的にsnapshotを残したい場合は `--checkpoint` を利用できます。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 356d94e6d..a2233f931 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -625,7 +625,8 @@ private static IReadOnlyList BuildAll() }, new() { Name = "--show-paths", Description = "Show resolved local paths in maintenance diagnostics, recovery commands, or status --config output", PrimaryCommands = Set(RecoveryPathCommands.Concat(["index", "backfill-fold", "optimize", "vacuum", "db", "status"]).ToArray()) }, new() { Name = "--dry-run-path-limit", ValuePlaceholder = "", Description = "Dry run only: candidate path processing limit before truncated lower-bound estimates", PrimaryCommands = Set("index") }, - new() { Name = "--no-checkpoint", Description = "Skip the automatic DB checkpoint before maintenance", PrimaryCommands = Set("backfill-fold") }, + new() { Name = "--checkpoint", Description = "Create a DB checkpoint even when backfill preflight is already complete", PrimaryCommands = Set("backfill-fold") }, + new() { Name = "--no-checkpoint", Description = "Skip the automatic DB checkpoint before a required backfill mutation", PrimaryCommands = Set("backfill-fold") }, new() { Name = "--force", diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index cd171b80a..6bd9a6eed 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -77,7 +77,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = [ ("index", "cdidx index [--db ] [--rebuild [--yes]] [--optimize [--show-paths]] [--symbols-only] [--verbose] [--dry-run [--dry-run-path-limit ]] [--force] [--quiet] [--json] [--allow-partial] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--max-references-per-file ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ] [--watch-pending-path-limit ]]"), ("hooks", "cdidx hooks [--project ] [--force] [--dry-run] [--json]"), - ("backfill-fold", "cdidx backfill-fold [--db ] [--dry-run] [--no-checkpoint] [--show-paths] [--json]"), + ("backfill-fold", "cdidx backfill-fold [--db ] [--dry-run] [--checkpoint|--no-checkpoint] [--show-paths] [--json]"), ("optimize", "cdidx optimize [--db ] [--dry-run] [--show-paths] [--json]"), ("vacuum", "cdidx vacuum [--db ] [--dry-run] [--show-paths] [--json]"), ("index-commits", "cdidx index --commits [commit-ref ...] [--db ] [--verbose] [--dry-run [--dry-run-path-limit ]] [--json] [--allow-partial] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs b/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs index be0d4ad58..c30bef031 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs @@ -11,7 +11,7 @@ public static partial class IndexCommandRunner private static readonly string[] AcceptedBackfillFoldFlags = [ "--db", "--json", "--dry-run", "--help", - "--no-checkpoint", "--show-paths", + "--checkpoint", "--no-checkpoint", "--show-paths", ]; private static readonly (string Name, string[] Columns)[] OptimizeObjectDefinitions = [ @@ -564,6 +564,12 @@ internal static int RunBackfillFold( try { + var normalizedDbPath = Path.GetFullPath(DbPathResolver.NormalizeDbPath(options.DbPath)); + using var indexLock = options.DryRun + ? null + : IndexLock.Acquire( + IndexLock.GetLockPath(normalizedDbPath), + Path.GetDirectoryName(normalizedDbPath) ?? Environment.CurrentDirectory); using var db = new DbContext( options.DryRun ? DbOpenIntent.QueryOnly : DbOpenIntent.Migration, options.DbPath); @@ -617,45 +623,73 @@ internal static int RunBackfillFold( var symbolReferences = 0; var verified = false; var userVersionAfter = userVersionBefore; + var pendingRows = writer.CountBackfillFoldedColumns(rewriteAll); + var graphRefreshPending = writer.IsFoldBackfillGraphRefreshPending(); + var mutationRequired = pendingRows.Symbols > 0 + || pendingRows.SymbolReferences > 0 + || !foldReadyBefore + || csharpSymbolNameContractUpgradeRequired + || graphRefreshPending; + var wasAlreadyComplete = foldReadyBefore + && !rewriteAll + && !csharpSymbolNameContractUpgradeRequired + && !graphRefreshPending + && pendingRows.Symbols == 0 + && pendingRows.SymbolReferences == 0; + var checkpointSkippedReason = options.DryRun + ? "dry_run" + : options.NoCheckpoint + ? "disabled_by_option" + : !mutationRequired && !options.Checkpoint + ? "already_complete" + : null; + var checkpointSkipped = checkpointSkippedReason != null; if (options.DryRun) { - (symbols, symbolReferences) = writer.CountBackfillFoldedColumns(rewriteAll); + (symbols, symbolReferences) = pendingRows; } else { - if (!options.NoCheckpoint) + backfillCancellation.Token.ThrowIfCancellationRequested(); + if (!checkpointSkipped) DbCommandRunner.CreateAutomaticCheckpoint(options.DbPath); - (symbols, symbolReferences) = writer.BackfillFoldedColumns( - rewriteAll, - backfillCancellation.Token); - // Row rewrites commit before the final FoldReady stamp so interrupted - // backfills can resume from the remaining rows. - // 行更新は FoldReady stamp より前に永続化し、中断後に残り行から再開できるようにする。 - using var transaction = writer.BeginTransaction(backfillCancellation.Token, "backfill fold readiness stamp"); - verified = writer.MarkFoldReady(); - if (!verified) + if (mutationRequired) { - return WriteCommandError( - options.Json, - jsonOptions, - "folded-name backfill verification failed: some rows still have NULL folded values", - CommandExitCodes.DatabaseError, - "Retry `cdidx backfill-fold`. If the DB still does not verify, rebuild it with `cdidx index --rebuild`.", - CommandErrorCodes.DbError); - } - writer.MarkCSharpSymbolNameContractReady(); + (symbols, symbolReferences) = writer.BackfillFoldedColumns( + rewriteAll, + backfillCancellation.Token); + // Row rewrites commit before the final FoldReady stamp so interrupted + // backfills can resume from the remaining rows. + // 行更新は FoldReady stamp より前に永続化し、中断後に残り行から再開できるようにする。 + using var transaction = writer.BeginTransaction(backfillCancellation.Token, "backfill fold readiness stamp"); + verified = writer.MarkFoldReady(); + if (!verified) + { + return WriteCommandError( + options.Json, + jsonOptions, + "folded-name backfill verification failed: some rows still have NULL folded values", + CommandExitCodes.DatabaseError, + "Retry `cdidx backfill-fold`. If the DB still does not verify, rebuild it with `cdidx index --rebuild`.", + CommandErrorCodes.DbError); + } + writer.MarkCSharpSymbolNameContractReady(); - transaction.Commit(); - userVersionAfter = db.GetUserVersion(); + transaction.Commit(); + userVersionAfter = db.GetUserVersion(); + } + else + { + verified = true; + } } var foldMetadataCurrentAfter = options.DryRun ? foldMetadataCurrentBefore : true; var foldReadyAfter = (userVersionAfter & DbContext.FoldReadyFlag) != 0 && foldMetadataCurrentAfter; - var wasAlreadyComplete = foldReadyBefore && !rewriteAll && symbols == 0 && symbolReferences == 0; if (options.Json) { @@ -670,7 +704,9 @@ internal static int RunBackfillFold( verified, userVersionBefore, userVersionAfter, - foldReadyAfter), jsonContext.BackfillFoldJsonResult)); + foldReadyAfter, + checkpointSkipped, + checkpointSkippedReason), jsonContext.BackfillFoldJsonResult)); } else { @@ -683,6 +719,13 @@ internal static int RunBackfillFold( if (rewriteAll) CommandOutputWriter.WriteLine(" mode: full folded-key refresh (fold metadata missing or mismatched)"); CommandOutputWriter.WriteLine($" already complete: {(wasAlreadyComplete ? "yes" : "no")}"); + CommandOutputWriter.WriteLine(checkpointSkippedReason switch + { + "dry_run" => " checkpoint: skipped (dry run)", + "disabled_by_option" => " checkpoint: skipped (--no-checkpoint)", + "already_complete" => " checkpoint: skipped (already complete)", + _ => " checkpoint: created", + }); CommandOutputWriter.WriteLine($" fold_ready: {foldReadyBefore} -> {foldReadyAfter}"); if (!options.DryRun) { @@ -693,6 +736,21 @@ internal static int RunBackfillFold( return CommandExitCodes.Success; } + catch (IndexLockConflictException ex) + { + var holderDescription = DescribeLockHolder(ex.Holder); + return MaintenanceDatabaseErrorWriter.Write( + options.Json, + jsonOptions, + MaintenanceDatabaseErrorClassifier.Create( + "backfill-fold", + options.DbPath, + options.ShowPaths, + MaintenanceDatabaseFailureKind.Locked, + details: string.IsNullOrEmpty(holderDescription) + ? null + : [holderDescription])); + } catch (OperationCanceledException) { return WriteCommandError( @@ -769,6 +827,7 @@ private static BackfillFoldCommandOptions ParseBackfillFoldArgs(string[] args) var dbPath = Path.Combine(".cdidx", "codeindex.db"); var json = false; var dryRun = false; + var checkpoint = false; var noCheckpoint = false; var showPaths = false; string? parseError = null; @@ -786,6 +845,9 @@ private static BackfillFoldCommandOptions ParseBackfillFoldArgs(string[] args) case "--dry-run": dryRun = true; break; + case "--checkpoint": + checkpoint = true; + break; case "--no-checkpoint": noCheckpoint = true; break; @@ -793,7 +855,7 @@ private static BackfillFoldCommandOptions ParseBackfillFoldArgs(string[] args) showPaths = true; break; case "--help" or "-h": - return new BackfillFoldCommandOptions { ShowHelp = true, DbPath = dbPath, Json = json, DryRun = dryRun, NoCheckpoint = noCheckpoint, ShowPaths = showPaths }; + return new BackfillFoldCommandOptions { ShowHelp = true, DbPath = dbPath, Json = json, DryRun = dryRun, Checkpoint = checkpoint, NoCheckpoint = noCheckpoint, ShowPaths = showPaths }; default: if (args[i].StartsWith("-", StringComparison.Ordinal)) { @@ -807,11 +869,15 @@ private static BackfillFoldCommandOptions ParseBackfillFoldArgs(string[] args) } } + if (checkpoint && noCheckpoint) + parseError ??= "--checkpoint and --no-checkpoint cannot be used together"; + return new BackfillFoldCommandOptions { DbPath = dbPath, Json = json, DryRun = dryRun, + Checkpoint = checkpoint, NoCheckpoint = noCheckpoint, ShowPaths = showPaths, ParseError = parseError, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index a452c371c..881a4840b 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -745,6 +745,7 @@ public sealed class BackfillFoldCommandOptions public bool Json { get; init; } public bool DryRun { get; init; } public bool ShowPaths { get; init; } + public bool Checkpoint { get; init; } public bool NoCheckpoint { get; init; } public string? ParseError { get; init; } } diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index e9be53a4e..f92fff506 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -23,6 +23,9 @@ internal sealed record BackfillFoldJsonResult( [property: JsonPropertyName("user_version_before")] int UserVersionBefore, [property: JsonPropertyName("user_version_after")] int UserVersionAfter, [property: JsonPropertyName("fold_ready")] bool FoldReady, + [property: JsonPropertyName("checkpoint_skipped")] bool CheckpointSkipped, + [property: JsonPropertyName("checkpoint_skipped_reason")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? CheckpointSkippedReason, [property: JsonPropertyName("api_version")] string ApiVersion = JsonOutputContract.ApiVersion) : IVersionedJsonResult; internal sealed record OptimizeFtsJsonResult( diff --git a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs index bb385f233..f1d8ae14e 100644 --- a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs +++ b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs @@ -430,6 +430,12 @@ FROM symbol_references } } + internal bool IsFoldBackfillGraphRefreshPending() + => string.Equals( + GetMetaString(FoldBackfillGraphRefreshPendingMetaKey), + "1", + StringComparison.Ordinal); + private static int ToInt32Count(object? value) { var count = value is long l ? l : (value is int i ? i : 0); diff --git a/tests/CodeIndex.Tests/ConsoleUiTests.cs b/tests/CodeIndex.Tests/ConsoleUiTests.cs index 055e639b3..6dbb46ef4 100644 --- a/tests/CodeIndex.Tests/ConsoleUiTests.cs +++ b/tests/CodeIndex.Tests/ConsoleUiTests.cs @@ -396,7 +396,7 @@ public void PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags() Assert.Contains("cdidx index --commits [commit-ref ...] [--db ] [--verbose] [--dry-run [--dry-run-path-limit ]] [--json] [--allow-partial] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]", output); Assert.Contains("cdidx index --changed-between [--db ] [--verbose] [--dry-run [--dry-run-path-limit ]] [--json] [--allow-partial] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]", output); Assert.Contains("cdidx index --files [path ...] [--db ] [--verbose] [--dry-run [--dry-run-path-limit ]] [--json] [--allow-partial] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]", output); - Assert.Contains("cdidx backfill-fold [--db ] [--dry-run] [--no-checkpoint] [--show-paths] [--json]", output); + Assert.Contains("cdidx backfill-fold [--db ] [--dry-run] [--checkpoint|--no-checkpoint] [--show-paths] [--json]", output); Assert.Contains("cdidx optimize [--db ] [--dry-run] [--show-paths] [--json]", output); Assert.Contains("cdidx license", output); Assert.Contains("cdidx completions ", output); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 3e88de71f..2e613be6b 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -187,6 +187,36 @@ public void RunBackfillFold_UnknownOptionBeforeJson_ReturnsJsonUsageError() } } + [Fact] + public void RunBackfillFold_ConflictingCheckpointOptions_ReturnJsonUsageError_Issue4889() + { + var missingDb = CreateTempDbPath("cdidx_backfill_checkpoint_conflict_4889"); + + lock (TestConsoleLock.Gate) + { + var originalOut = Console.Out; + using var stdout = new StringWriter(); + try + { + Console.SetOut(stdout); + var exitCode = IndexCommandRunner.RunBackfillFold( + ["--db", missingDb, "--checkpoint", "--no-checkpoint", "--json"], + _jsonOptions); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + using var document = JsonDocument.Parse(stdout.ToString()); + Assert.Equal( + "--checkpoint and --no-checkpoint cannot be used together", + document.RootElement.GetProperty("message").GetString()); + Assert.False(Directory.Exists(missingDb + ".checkpoints")); + } + finally + { + Console.SetOut(originalOut); + } + } + } + [Fact] public void FormatIndexFileException_RegexTimeout_UsesBoundedExtractionMessage() { @@ -5279,6 +5309,7 @@ public void RunBackfillFold_MissingDb_JsonIncludesHint() Assert.True(json.GetProperty("path_redacted").GetBoolean()); Assert.Contains("database file was not found", json.GetProperty("message").GetString()); Assert.Contains("Create or refresh the index", json.GetProperty("hint").GetString()); + Assert.False(Directory.Exists(missingDb + ".checkpoints")); } finally { @@ -6243,6 +6274,11 @@ public void RunBackfillFold_BackfillsLegacyRowsAndStampsFoldReady() Assert.Equal(27, json.GetProperty("user_version_before").GetInt32()); Assert.Equal(31, json.GetProperty("user_version_after").GetInt32()); Assert.True(json.GetProperty("fold_ready").GetBoolean()); + Assert.False(json.GetProperty("checkpoint_skipped").GetBoolean()); + Assert.False(json.TryGetProperty("checkpoint_skipped_reason", out _)); + var checkpointPath = Assert.Single(Directory.GetDirectories(dbPath + ".checkpoints")); + Assert.True(File.Exists(Path.Combine(checkpointPath, Path.GetFileName(dbPath)))); + Assert.True(File.Exists(Path.Combine(checkpointPath, "manifest.txt"))); using var verifyDb = new DbContext(DbOpenIntent.WriteIndex, dbPath); verifyDb.TryMigrateForRead(); @@ -6255,6 +6291,7 @@ public void RunBackfillFold_BackfillsLegacyRowsAndStampsFoldReady() { SqliteConnection.ClearAllPools(); DeleteFile(dbPath); + TestProjectHelper.DeleteDirectory(dbPath + ".checkpoints"); } } @@ -6315,6 +6352,9 @@ public void RunBackfillFold_DryRunReportsRowsWithoutWriting() Assert.Equal(1, json.GetProperty("symbols").GetInt32()); Assert.False(json.GetProperty("verified").GetBoolean()); Assert.False(json.GetProperty("fold_ready_after").GetBoolean()); + Assert.True(json.GetProperty("checkpoint_skipped").GetBoolean()); + Assert.Equal("dry_run", json.GetProperty("checkpoint_skipped_reason").GetString()); + Assert.False(Directory.Exists(dbPath + ".checkpoints")); using var verifyDb = new DbContext(DbOpenIntent.WriteIndex, dbPath); using var count = verifyDb.Connection.CreateCommand(); @@ -6648,6 +6688,7 @@ public void RunBackfillFold_PreservesNewerCSharpIdentityContractWithoutCSharpFil public void RunBackfillFold_DoesNotRewriteCurrentFoldRowsWhenCSharpIsAbsent_Issue4866Review() { var dbPath = CreateTempDbPath("cdidx_backfill_fold_without_csharp"); + var checkpointRoot = dbPath + ".checkpoints"; try { using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) @@ -6678,8 +6719,86 @@ public void RunBackfillFold_DoesNotRewriteCurrentFoldRowsWhenCSharpIsAbsent_Issu writer.SetMeta(DbContext.CSharpSymbolNameContractVersionMetaKey, null); } - JsonElement json; - int exitCode; + (int ExitCode, JsonElement Json) RunBackfill(params string[] additionalArguments) + { + var arguments = new List { "--db", dbPath, "--json" }; + arguments.AddRange(additionalArguments); + lock (TestConsoleLock.Gate) + { + var originalOut = Console.Out; + using var output = new StringWriter(); + try + { + Console.SetOut(output); + var exitCode = IndexCommandRunner.RunBackfillFold( + arguments.ToArray(), + _jsonOptions); + using var document = JsonDocument.Parse(output.ToString()); + return (exitCode, document.RootElement.Clone()); + } + finally + { + Console.SetOut(originalOut); + } + } + } + + var noOp = RunBackfill(); + Assert.Equal(CommandExitCodes.Success, noOp.ExitCode); + Assert.False(noOp.Json.GetProperty("rewrite_all").GetBoolean()); + Assert.Equal(0, noOp.Json.GetProperty("symbols").GetInt32()); + Assert.Equal(0, noOp.Json.GetProperty("symbol_references").GetInt32()); + Assert.True(noOp.Json.GetProperty("was_already_complete").GetBoolean()); + Assert.True(noOp.Json.GetProperty("verified").GetBoolean()); + Assert.True(noOp.Json.GetProperty("checkpoint_skipped").GetBoolean()); + Assert.Equal( + "already_complete", + noOp.Json.GetProperty("checkpoint_skipped_reason").GetString()); + Assert.False(Directory.Exists(checkpointRoot)); + + using (var walKeeper = new SqliteConnection($"Data Source={dbPath}")) + { + walKeeper.Open(); + using var walWrite = walKeeper.CreateCommand(); + walWrite.CommandText = """ + CREATE TABLE IF NOT EXISTS issue4889_wal_probe(value INTEGER NOT NULL); + INSERT INTO issue4889_wal_probe(value) VALUES (1); + """; + walWrite.ExecuteNonQuery(); + + var forced = RunBackfill("--checkpoint"); + Assert.Equal(CommandExitCodes.Success, forced.ExitCode); + Assert.True(forced.Json.GetProperty("was_already_complete").GetBoolean()); + Assert.False(forced.Json.GetProperty("checkpoint_skipped").GetBoolean()); + Assert.False(forced.Json.TryGetProperty("checkpoint_skipped_reason", out _)); + + var forcedCheckpoint = Assert.Single(Directory.GetDirectories(checkpointRoot)); + Assert.True(File.Exists(Path.Combine(forcedCheckpoint, Path.GetFileName(dbPath)))); + Assert.True(File.Exists(Path.Combine(forcedCheckpoint, Path.GetFileName(dbPath) + "-wal"))); + Assert.True(File.Exists(Path.Combine(forcedCheckpoint, Path.GetFileName(dbPath) + "-shm"))); + Assert.True(File.Exists(Path.Combine(forcedCheckpoint, "manifest.txt"))); + } + + using (var conn = new SqliteConnection($"Data Source={dbPath}")) + { + conn.Open(); + using var invalidate = conn.CreateCommand(); + invalidate.CommandText = "UPDATE symbols SET name_folded = NULL WHERE name = 'run'"; + Assert.Equal(1, invalidate.ExecuteNonQuery()); + } + + var disabled = RunBackfill("--no-checkpoint"); + Assert.Equal(CommandExitCodes.Success, disabled.ExitCode); + Assert.Equal(1, disabled.Json.GetProperty("symbols").GetInt32()); + Assert.False(disabled.Json.GetProperty("was_already_complete").GetBoolean()); + Assert.True(disabled.Json.GetProperty("checkpoint_skipped").GetBoolean()); + Assert.Equal( + "disabled_by_option", + disabled.Json.GetProperty("checkpoint_skipped_reason").GetString()); + Assert.Single(Directory.GetDirectories(checkpointRoot)); + + string humanOutput; + int humanExitCode; lock (TestConsoleLock.Gate) { var originalOut = Console.Out; @@ -6687,11 +6806,10 @@ public void RunBackfillFold_DoesNotRewriteCurrentFoldRowsWhenCSharpIsAbsent_Issu try { Console.SetOut(output); - exitCode = IndexCommandRunner.RunBackfillFold( - ["--db", dbPath, "--json"], + humanExitCode = IndexCommandRunner.RunBackfillFold( + ["--db", dbPath], _jsonOptions); - using var document = JsonDocument.Parse(output.ToString()); - json = document.RootElement.Clone(); + humanOutput = output.ToString(); } finally { @@ -6699,16 +6817,85 @@ public void RunBackfillFold_DoesNotRewriteCurrentFoldRowsWhenCSharpIsAbsent_Issu } } - Assert.Equal(CommandExitCodes.Success, exitCode); - Assert.False(json.GetProperty("rewrite_all").GetBoolean()); - Assert.Equal(0, json.GetProperty("symbols").GetInt32()); - Assert.Equal(0, json.GetProperty("symbol_references").GetInt32()); - Assert.True(json.GetProperty("was_already_complete").GetBoolean()); + Assert.Equal(CommandExitCodes.Success, humanExitCode); + Assert.Contains("checkpoint: skipped (already complete)", humanOutput, StringComparison.Ordinal); + Assert.Single(Directory.GetDirectories(checkpointRoot)); + + using (var pendingDb = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var pendingWriter = new DbWriter(pendingDb.Connection); + pendingWriter.SetMeta(DbWriter.FoldBackfillGraphRefreshPendingMetaKey, "1"); + } + + var resumed = RunBackfill(); + Assert.Equal(CommandExitCodes.Success, resumed.ExitCode); + Assert.Equal(0, resumed.Json.GetProperty("symbols").GetInt32()); + Assert.Equal(0, resumed.Json.GetProperty("symbol_references").GetInt32()); + Assert.False(resumed.Json.GetProperty("was_already_complete").GetBoolean()); + Assert.False(resumed.Json.GetProperty("checkpoint_skipped").GetBoolean()); + Assert.Equal(2, Directory.GetDirectories(checkpointRoot).Length); + using var verifyPending = new DbContext(DbOpenIntent.QueryOnly, dbPath); + Assert.Null(verifyPending.GetMetaString(DbWriter.FoldBackfillGraphRefreshPendingMetaKey)); } finally { SqliteConnection.ClearAllPools(); DeleteFile(dbPath); + TestProjectHelper.DeleteDirectory(checkpointRoot); + } + } + + [Fact] + public void RunBackfillFold_ForcedCheckpointRespectsIndexLock_Issue4889() + { + var dbPath = CreateTempDbPath("cdidx_backfill_fold_checkpoint_locked_4889"); + var checkpointRoot = dbPath + ".checkpoints"; + var lockPath = IndexLock.GetLockPath(dbPath); + try + { + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + db.InitializeSchema(); + var writer = new DbWriter(db.Connection); + Assert.True(writer.MarkFoldReady()); + } + + using (IndexLock.Acquire(lockPath, Path.GetDirectoryName(dbPath)!)) + { + JsonElement json; + int exitCode; + lock (TestConsoleLock.Gate) + { + var originalOut = Console.Out; + using var output = new StringWriter(); + try + { + Console.SetOut(output); + exitCode = IndexCommandRunner.RunBackfillFold( + ["--db", dbPath, "--checkpoint", "--json"], + _jsonOptions); + using var document = JsonDocument.Parse(output.ToString()); + json = document.RootElement.Clone(); + } + finally + { + Console.SetOut(originalOut); + } + } + + Assert.Equal(CommandExitCodes.TransientDatabaseError, exitCode); + Assert.Equal(CommandErrorCodes.DbLocked, json.GetProperty("error_code").GetString()); + Assert.Equal("database_locked", json.GetProperty("category").GetString()); + Assert.False(Directory.Exists(checkpointRoot)); + } + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteFile(dbPath); + DeleteFile(lockPath + ".info"); + DeleteFile(lockPath); + TestProjectHelper.DeleteDirectory(checkpointRoot); } } @@ -7019,6 +7206,8 @@ public void RunBackfillFold_RewritesAllWhenOnlyFingerprintDrifted() Assert.True(json.GetProperty("rewrite_all").GetBoolean()); Assert.True(json.GetProperty("verified").GetBoolean()); Assert.True(json.GetProperty("fold_ready").GetBoolean()); + Assert.False(json.GetProperty("checkpoint_skipped").GetBoolean()); + Assert.Single(Directory.GetDirectories(dbPath + ".checkpoints")); using var verifyDb = new DbContext(DbOpenIntent.WriteIndex, dbPath); verifyDb.TryMigrateForRead(); @@ -7032,6 +7221,7 @@ public void RunBackfillFold_RewritesAllWhenOnlyFingerprintDrifted() { SqliteConnection.ClearAllPools(); DeleteFile(dbPath); + TestProjectHelper.DeleteDirectory(dbPath + ".checkpoints"); } } From 66a9d1fc33d94a0b829c0004fd6ff99e989c3453 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 30 Jul 2026 13:50:59 +0900 Subject: [PATCH 2/2] Fix #4889 preserve fold validation before no-op --- DEVELOPER_GUIDE.md | 4 ++-- TESTING_GUIDE.md | 4 ++-- changelog.d/unreleased/4889.fixed.md | 4 ++-- .../Cli/IndexCommandRunner.Maintenance.cs | 18 ++++++++++++++- .../Database/DbWriter.FoldBackfill.cs | 9 ++++++-- .../IndexCommandRunnerTests.cs | 23 ++++++++++++++++++- 6 files changed, 52 insertions(+), 10 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index a0aade5dd..c494278aa 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -50,7 +50,7 @@ Development contracts: | Read-only database queries | `cdidx status --db /artifacts/codeindex.db --read-only --json`; `cdidx search AuthService --db /artifacts/codeindex.db --immutable` | Query commands accept `--read-only` (alias `--immutable`) to open an existing CodeIndex database through SQLite's immutable read-only URI mode. Use this for CI artifacts, mounted caches, and sandboxes where creating or updating `codeindex.db-wal` / `codeindex.db-shm` sidecars is not allowed. | | Mutating commands | `index`, `backfill-fold`, `optimize`, `vacuum` | These require writable storage and reject read-only database opens. | | Reusable index artifact | `cdidx export codeindex.cdidx.zip`; `cdidx import codeindex.cdidx.zip --db `; `cdidx import codeindex.cdidx.zip --dry-run --json` | Run export after indexing and upload the archive. Export refuses an existing destination unless `--overwrite` is explicit, publishes atomically from an owner-only temporary file, and verifies POSIX mode `0600`. Successful export JSON adds final archive byte size and SHA-256 plus the complete immutable manifest while retaining the prior result fields. Consumers import before query commands, or use `--dry-run` / `--check` to validate the archive without replacing the destination DB. Use `--prune-paths` when the archive comes from another checkout and the restored DB should advertise the import target project root; imports targeting `.../.cdidx/codeindex.db` use the sibling project directory, while other DB paths fall back to the process current directory. The archive contains only `manifest.json` plus `codeindex.db`; import validates ZIP entry names through `ZipArchiveSafetyPolicy` and rejects absolute, parent-directory, backslash, NUL, non-canonical, duplicate, and extra entries before extraction. The manifest carries bounded summary/readiness metadata including row counts, readiness bits, writer/indexed-head metadata, schema contract stamps, and unknown-extension summary when available. Import validates manifest format, manifest `user_version`, `database_sha256`, present summary counts, and the embedded SQLite file as a CodeIndex database before replacing the destination DB. Import rejects archive `codeindex.db` entries whose compressed or uncompressed metadata exceeds 8 GiB, and the extraction stream is also capped at 8 GiB. | -| Maintenance checkpoint and managed rollback | `cdidx db checkpoint [--dry-run]`; `cdidx db checkpoints --list|--delete |--prune --keep [--dry-run]`; `cdidx db restore [--dry-run] [--no-backup]`; `cdidx db restore-backups --list|--prune --keep |--restore [--dry-run] [--no-backup]` | Checkpoint snapshots `codeindex.db` plus existing WAL/SHM sidecars before risky maintenance. Import and both restore forms create a consistent, verified managed SQLite rollback snapshot before replacing an existing DB unless `--no-backup` is explicit. Managed directories use `.restore-backup-/` and contain a bounded manifest plus one standalone database payload; the manifest records SHA-256, byte count, supported `user_version`, provenance, and an optional source identifier, but no local absolute source path. `restore-backups --list` exposes the ID and provenance while retaining legacy directory metadata; existing prune retention remains compatible. `restore-backups --restore ` revalidates the directory boundary, manifest, payload hash, schema, and combined staging/rollback free space, then performs an atomic replacement with transient rollback-on-failure. Its `--dry-run` reports every validation and planned backup without mutation. Checkpoint delete/prune and restore-backup prune require an explicit mutation action, and checkpoint prune skips all deletion if its bounded 1,000-directory scan is truncated. Checkpoints live under `.checkpoints//`. `backfill-fold` preflights folded rows and readiness under the index lock, creates an automatic checkpoint only when a mutation is required, accepts `--checkpoint` to force a snapshot for an already-complete DB, and accepts `--no-checkpoint` to skip mutation protection explicitly. JSON reports `checkpoint_skipped` plus `checkpoint_skipped_reason` (`already_complete`, `dry_run`, or `disabled_by_option`), and human output reports the same decision. | +| Maintenance checkpoint and managed rollback | `cdidx db checkpoint [--dry-run]`; `cdidx db checkpoints --list|--delete |--prune --keep [--dry-run]`; `cdidx db restore [--dry-run] [--no-backup]`; `cdidx db restore-backups --list|--prune --keep |--restore [--dry-run] [--no-backup]` | Checkpoint snapshots `codeindex.db` plus existing WAL/SHM sidecars before risky maintenance. Import and both restore forms create a consistent, verified managed SQLite rollback snapshot before replacing an existing DB unless `--no-backup` is explicit. Managed directories use `.restore-backup-/` and contain a bounded manifest plus one standalone database payload; the manifest records SHA-256, byte count, supported `user_version`, provenance, and an optional source identifier, but no local absolute source path. `restore-backups --list` exposes the ID and provenance while retaining legacy directory metadata; existing prune retention remains compatible. `restore-backups --restore ` revalidates the directory boundary, manifest, payload hash, schema, and combined staging/rollback free space, then performs an atomic replacement with transient rollback-on-failure. Its `--dry-run` reports every validation and planned backup without mutation. Checkpoint delete/prune and restore-backup prune require an explicit mutation action, and checkpoint prune skips all deletion if its bounded 1,000-directory scan is truncated. Checkpoints live under `.checkpoints//`. `backfill-fold` preflights folded rows, persisted folded values, and readiness under the index lock, creates an automatic checkpoint only when a mutation is required, accepts `--checkpoint` to force a snapshot for an already-complete DB, and accepts `--no-checkpoint` to skip mutation protection explicitly. JSON reports `checkpoint_skipped` plus `checkpoint_skipped_reason` (`already_complete`, `dry_run`, or `disabled_by_option`), and human output reports the same decision. | | Binary compatibility | [COMPATIBILITY.md](COMPATIBILITY.md) | Database compatibility across `cdidx` binary upgrades and downgrades is documented there. Keep that policy updated whenever readiness bits, `codeindex_meta` contract stamps, or rebuild requirements change. | | Fold backfill preview and recovery | `backfill-fold --dry-run`; `backfill-fold --checkpoint`; MCP `backfill_fold` with `dry_run: true` or `force: true` | Dry-run previews folded-key rows without mutating the DB or stamping FoldReady. `--checkpoint` explicitly preserves a snapshot even when CLI preflight finds no mutation, while the default completed no-op leaves no checkpoint artifacts. MCP accepts the same preview and can force rewriting all folded keys when an operator needs to recover from suspicious fold metadata or row state even though the stored version/fingerprint appears current. Non-dry-run row rewrites are resumable after interruption: completed row updates remain durable, and final FoldReady metadata is stamped only after verification succeeds. MCP responses include `progress.rows_done`, `progress.rows_total`, and `progress.fraction` so clients can report and retry long backfills. | @@ -3394,7 +3394,7 @@ net9 CI lane に合わせる場合は `FRAMEWORK=net9.0 make test` を使いま | read-only database query | `cdidx status --db /artifacts/codeindex.db --read-only --json`; `cdidx search AuthService --db /artifacts/codeindex.db --immutable` | query コマンドは `--read-only`（alias: `--immutable`）を受け付け、既存の CodeIndex database を SQLite の immutable read-only URI mode で開けます。CI artifact、mounted cache、`codeindex.db-wal` / `codeindex.db-shm` sidecar を作成・更新できない sandbox で使います。 | | 変更系コマンド | `index`、`backfill-fold`、`optimize`、`vacuum` | 書き込み可能な storage を必要とし、read-only database open を拒否します。 | | 再利用可能な index artifact | `cdidx export codeindex.cdidx.zip`; `cdidx import codeindex.cdidx.zip --db `; `cdidx import codeindex.cdidx.zip --dry-run --json` | CI job では index 後に export して archive を upload します。export は `--overwrite` を明示しない限り既存 destination を拒否し、owner-only temporary file から atomic に publish して POSIX mode `0600` を検証します。export 成功時の JSON は従来 field を維持し、最終 archive の byte 数と SHA-256、完全で immutable な manifest を追加します。利用側は query コマンドの前に import でき、`--dry-run` / `--check` で destination DB を置き換えず archive を検証できます。別 checkout 由来の archive を import 先 project root として扱いたい場合は `--prune-paths` を使います。`.../.cdidx/codeindex.db` を import 先にした場合は sibling の project directory を使い、それ以外の DB path では process current directory に fallback します。archive は `manifest.json` と `codeindex.db` だけを含みます。import は ZIP entry 名を `ZipArchiveSafetyPolicy` で検証し、absolute path、parent-directory segment、backslash、NUL、non-canonical name、duplicate entry、extra entry を extraction 前に拒否します。manifest は row count、readiness bit、writer / indexed-head metadata、schema contract stamp、利用可能な unknown-extension summary などの bounded summary/readiness metadata を持ちます。import は manifest format、manifest `user_version`、`database_sha256`、存在する summary count、embedded SQLite file が CodeIndex database であることを検証してから destination DB を置き換えます。archive の `codeindex.db` entry は compressed / uncompressed metadata と extraction stream の双方で 8 GiB を上限に拒否されます。 | -| maintenance checkpoint と managed rollback | `cdidx db checkpoint [--dry-run]`; `cdidx db checkpoints --list|--delete |--prune --keep [--dry-run]`; `cdidx db restore [--dry-run] [--no-backup]`; `cdidx db restore-backups --list|--prune --keep |--restore [--dry-run] [--no-backup]` | 危険な maintenance の前に `codeindex.db` と既存 WAL/SHM sidecar の checkpoint を作成できます。import と2種類の restore は、既存 DB を置き換える前に consistent かつ検証済みの managed SQLite rollback snapshot を既定で作成し、`--no-backup` を明示した場合だけ省略します。managed directory は `.restore-backup-/` で、bounded manifest と standalone database payload 1個を含みます。manifest は SHA-256、byte 数、対応する `user_version`、provenance、任意の source identifier を記録しますが、local absolute source path は記録しません。`restore-backups --list` は従来 directory metadata との互換性を維持しつつ ID と provenance を表示し、既存 prune retention もそのまま利用できます。`restore-backups --restore ` は directory 境界、manifest、payload hash、schema、staging と rollback を合わせた free space を再検証してから、失敗時の transient rollback を伴う atomic replacement を実行します。`--dry-run` は変更せず、すべての検証と作成予定 backup を報告します。checkpoint の delete / prune と restore-backup の prune は明示的な変更 action を必要とし、checkpoint prune の bounded scan が truncated の場合は削除をすべて skip します。checkpoint は `.checkpoints//` に置かれます。`backfill-fold` は index lock 内で folded row と readiness を事前確認し、mutation が必要な場合だけ automatic checkpoint を作ります。完了済みDBでもsnapshotを明示的に残すには `--checkpoint`、mutation protectionを明示的に省略するには `--no-checkpoint` を使います。JSON は `checkpoint_skipped` と `checkpoint_skipped_reason`（`already_complete`、`dry_run`、`disabled_by_option`）を返し、human output も同じ判断を表示します。 | +| maintenance checkpoint と managed rollback | `cdidx db checkpoint [--dry-run]`; `cdidx db checkpoints --list|--delete |--prune --keep [--dry-run]`; `cdidx db restore [--dry-run] [--no-backup]`; `cdidx db restore-backups --list|--prune --keep |--restore [--dry-run] [--no-backup]` | 危険な maintenance の前に `codeindex.db` と既存 WAL/SHM sidecar の checkpoint を作成できます。import と2種類の restore は、既存 DB を置き換える前に consistent かつ検証済みの managed SQLite rollback snapshot を既定で作成し、`--no-backup` を明示した場合だけ省略します。managed directory は `.restore-backup-/` で、bounded manifest と standalone database payload 1個を含みます。manifest は SHA-256、byte 数、対応する `user_version`、provenance、任意の source identifier を記録しますが、local absolute source path は記録しません。`restore-backups --list` は従来 directory metadata との互換性を維持しつつ ID と provenance を表示し、既存 prune retention もそのまま利用できます。`restore-backups --restore ` は directory 境界、manifest、payload hash、schema、staging と rollback を合わせた free space を再検証してから、失敗時の transient rollback を伴う atomic replacement を実行します。`--dry-run` は変更せず、すべての検証と作成予定 backup を報告します。checkpoint の delete / prune と restore-backup の prune は明示的な変更 action を必要とし、checkpoint prune の bounded scan が truncated の場合は削除をすべて skip します。checkpoint は `.checkpoints//` に置かれます。`backfill-fold` は index lock 内で folded row、永続化されたfolded値、readiness を事前確認し、mutation が必要な場合だけ automatic checkpoint を作ります。完了済みDBでもsnapshotを明示的に残すには `--checkpoint`、mutation protectionを明示的に省略するには `--no-checkpoint` を使います。JSON は `checkpoint_skipped` と `checkpoint_skipped_reason`（`already_complete`、`dry_run`、`disabled_by_option`）を返し、human output も同じ判断を表示します。 | | binary compatibility | [COMPATIBILITY.md](COMPATIBILITY.md) | `cdidx` binary の upgrade / downgrade をまたぐ database compatibility を記載します。readiness bit、`codeindex_meta` contract stamp、rebuild requirement を変える場合は、この policy も更新してください。 | | Fold backfill の preview / recovery | `backfill-fold --dry-run`; `backfill-fold --checkpoint`; MCP `backfill_fold` の `dry_run: true` または `force: true` | dry-run は DB を変更せず FoldReady stamp も書かずに、rewrite 対象の folded-key row をプレビューします。CLI preflight でmutation不要と判断された場合でもsnapshotを明示的に保存するには `--checkpoint` を使います。既定の完了済みno-opはcheckpoint artifactを作りません。MCP も同じ preview を受け付け、stored version / fingerprint が current に見える場合でも suspicious な fold metadata や row state を復旧するため `force: true` を受け付けます。non-dry-run rewrite は中断後に resume でき、完了済み row update は durable に残り、最終 FoldReady metadata は verification 成功後にだけ stamp されます。MCP response は `progress.rows_done`、`progress.rows_total`、`progress.fraction` を含みます。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 27050f0dc..6c3adbdd7 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -364,7 +364,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding - `IndexCommandRunnerTests.RunOptimizeFts_DryRunPreviewsWithoutWritingThenOptimizeMutates_Issue4577`, `RunOptimizeFts_LockHeld_ReportsDbLocked`, and `RunOptimizeFts_ReadOnlyUri_ReturnsDbNotWritable` share seeded databases to verify that optimize dry-run reports sizes, readiness, lock state, recommendations, and planned work while preserving the source bytes and creating no lock artifacts; the same fixtures retain the writable optimize and lock/read-only URI mutation guards. `ConsoleUiTests.PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags` keeps the optimize dry-run flag visible in full CLI usage. -- Issue #4889 backfill-checkpoint coverage keeps the mutation preflight, index lock, and artifact decision in one contract. Reuse the completed fixture for default no-op, forced `--checkpoint`, `--no-checkpoint` mutation, human output, and a zero-row pending graph refresh; keep WAL/SHM open during the forced snapshot so Windows sharing semantics and the complete DB/WAL/SHM/manifest file set are exercised. Separate dry-run, invalid-DB, and held-lock assertions must prove that no checkpoint directory is created, while a real pending rewrite or pending graph refresh must retain the automatic checkpoint. +- Issue #4889 backfill-checkpoint coverage keeps the mutation preflight, folded-value validation, index lock, and artifact decision in one contract. Reuse the completed fixture for default no-op, forced `--checkpoint`, `--no-checkpoint` mutation, human output, a non-NULL drifted fold that requires a protected full repair, and a zero-row pending graph refresh; keep WAL/SHM open during the forced snapshot so Windows sharing semantics and the complete DB/WAL/SHM/manifest file set are exercised. Separate dry-run, invalid-DB, and held-lock assertions must prove that no checkpoint directory is created, while a real pending rewrite or pending graph refresh must retain the automatic checkpoint. - Issue #4856 maintenance-error coverage spans `CommandErrorWriterTests`, `QueryCommandRunnerTests`, `IndexCommandRunnerTests`, and `DbCommandRunnerTests`. Keep the SQLite primary-code matrix (`5`, `6`, `8`, `11`, `26`), caller-spelled relative paths, whole-value Unix/Windows/file-URI redaction, missing and invalid-header files, directory/inaccessible preflight classification, integrity corruption rows, lock-holder metadata, the `index --optimize` human preamble, valid JSON, and category-specific recovery hints aligned with classifier version `1`. - `IndexCommandRunnerTests.Run_DryRunWithRebuildAndMemoryTrace_SkipsConfirmationAndPreservesWorkspace_Issue4580` @@ -1304,7 +1304,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - `IndexCommandRunnerTests.RunOptimizeFts_DryRunPreviewsWithoutWritingThenOptimizeMutates_Issue4577`、`RunOptimizeFts_LockHeld_ReportsDbLocked`、`RunOptimizeFts_ReadOnlyUri_ReturnsDbNotWritable` seed 済み database を共有し、optimize dry-run が size、readiness、lock state、推奨、planned work を報告しつつ source byte を保持し、lock artifact を作成しないことを検証する。同じ fixture で、書き込み版 optimize と lock/read-only URI の mutation guard も維持する。 `ConsoleUiTests.PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags` は、full CLI usage に optimize dry-run flag が表示され続けることを固定します。 -- Issue #4889 の backfill-checkpoint coverage は、mutation preflight、index lock、artifact 作成判断を1つの契約として固定します。完了済み fixture を既定no-op、明示 `--checkpoint`、`--no-checkpoint` mutation、human output、zero-rowのpending graph refreshで再利用し、明示snapshot中は WAL/SHM をopenしたままにしてWindowsの共有 semanticsとDB/WAL/SHM/manifest一式を検証してください。dry-run、無効DB、lock保持中のassertionではcheckpoint directoryが作られないことを個別に証明し、実際にpending rewriteまたはpending graph refreshがある場合はautomatic checkpointを維持してください。 +- Issue #4889 の backfill-checkpoint coverage は、mutation preflight、folded値の検証、index lock、artifact 作成判断を1つの契約として固定します。完了済み fixture を既定no-op、明示 `--checkpoint`、`--no-checkpoint` mutation、human output、保護された全行修復を必要とする非NULLのfold drift、zero-rowのpending graph refreshで再利用し、明示snapshot中は WAL/SHM をopenしたままにしてWindowsの共有 semanticsとDB/WAL/SHM/manifest一式を検証してください。dry-run、無効DB、lock保持中のassertionではcheckpoint directoryが作られないことを個別に証明し、実際にpending rewriteまたはpending graph refreshがある場合はautomatic checkpointを維持してください。 - Issue #4856 の maintenance-error coverage は `CommandErrorWriterTests`、`QueryCommandRunnerTests`、`IndexCommandRunnerTests`、`DbCommandRunnerTests` にまたがります。 SQLite primary-code matrix (`5`、`6`、`8`、`11`、`26`)、呼び出し側表記を維持する relative path、Unix / Windows / file URI の値全体 redaction、missing / invalid-header file、directory / inaccessible preflight classification、integrity corruption row、lock-holder metadata、`index --optimize` の human preamble、有効な JSON、category 別 recovery hint を classifier version `1` と同期させてください。 - `IndexCommandRunnerTests.Run_DryRunWithRebuildAndMemoryTrace_SkipsConfirmationAndPreservesWorkspace_Issue4580` diff --git a/changelog.d/unreleased/4889.fixed.md b/changelog.d/unreleased/4889.fixed.md index 2199cf212..44f06cc49 100644 --- a/changelog.d/unreleased/4889.fixed.md +++ b/changelog.d/unreleased/4889.fixed.md @@ -10,8 +10,8 @@ affected: ## English -- **Completed fold backfills no longer create automatic checkpoint artifacts (#4889)** — `backfill-fold` now preflights pending rows and readiness under the index lock, creates an automatic DB/WAL/SHM/manifest checkpoint only before a real mutation, reports bounded checkpoint-skip reasons in JSON and human output, and accepts `--checkpoint` when an operator explicitly wants a snapshot of an already-complete database. +- **Completed fold backfills no longer create automatic checkpoint artifacts (#4889)** — `backfill-fold` now preflights pending rows, persisted folded values, and readiness under the index lock, creates an automatic DB/WAL/SHM/manifest checkpoint only before a real mutation, reports bounded checkpoint-skip reasons in JSON and human output, and accepts `--checkpoint` when an operator explicitly wants a snapshot of an already-complete database. ## 日本語 -- **完了済みの fold backfill が automatic checkpoint artifact を作成しなくなりました (#4889)** — `backfill-fold` は index lock 内で pending row と readiness を事前確認し、実際の mutation 前にだけ DB/WAL/SHM/manifest checkpoint を自動作成します。JSON と human output には上限付きの checkpoint skip reason を表示し、完了済みDBでもoperatorが明示的にsnapshotを残したい場合は `--checkpoint` を利用できます。 +- **完了済みの fold backfill が automatic checkpoint artifact を作成しなくなりました (#4889)** — `backfill-fold` は index lock 内で pending row、永続化されたfolded値、readiness を事前確認し、実際の mutation 前にだけ DB/WAL/SHM/manifest checkpoint を自動作成します。JSON と human output には上限付きの checkpoint skip reason を表示し、完了済みDBでもoperatorが明示的にsnapshotを残したい場合は `--checkpoint` を利用できます。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs b/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs index c30bef031..95f752ec4 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs @@ -624,16 +624,32 @@ internal static int RunBackfillFold( var verified = false; var userVersionAfter = userVersionBefore; var pendingRows = writer.CountBackfillFoldedColumns(rewriteAll); + if (!rewriteAll + && pendingRows.Symbols == 0 + && pendingRows.SymbolReferences == 0 + && !writer.AllFoldedColumnsBackfilled(requireCurrentFoldKeys: true)) + { + // A current metadata stamp plus non-NULL folded values is not sufficient: + // verify the persisted values before declaring a no-op. If they drifted, + // repair every row under the same checkpoint protection as stale metadata. + // current metadata と非 NULL 値だけでは no-op と断定せず実値を検証する。 + // drift があれば stale metadata と同様に checkpoint 保護下で全行を修復する。 + rewriteAll = true; + pendingRows = writer.CountBackfillFoldedColumns(rewriteAll); + } var graphRefreshPending = writer.IsFoldBackfillGraphRefreshPending(); + var symbolExtractorVersionsCurrent = writer.SymbolExtractorVersionsMatchCurrent(); var mutationRequired = pendingRows.Symbols > 0 || pendingRows.SymbolReferences > 0 || !foldReadyBefore || csharpSymbolNameContractUpgradeRequired - || graphRefreshPending; + || graphRefreshPending + || !symbolExtractorVersionsCurrent; var wasAlreadyComplete = foldReadyBefore && !rewriteAll && !csharpSymbolNameContractUpgradeRequired && !graphRefreshPending + && symbolExtractorVersionsCurrent && pendingRows.Symbols == 0 && pendingRows.SymbolReferences == 0; var checkpointSkippedReason = options.DryRun diff --git a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs index f1d8ae14e..79a2afa1b 100644 --- a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs +++ b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs @@ -129,9 +129,14 @@ private bool AllFoldedColumnsBackfilledCore( public bool AllFoldedColumnValuesMatchCurrentFold() { var markdownSymbolIdentityFolds = BuildMarkdownSymbolIdentityFoldMap(); + var hasDisplayNameFolded = + DbSchemaCache.LoadColumns(_conn, "symbols").Contains("display_name_folded"); + var displayNameFoldedProjection = hasDisplayNameFolded + ? "s.display_name_folded" + : "NULL"; var symbols = RentCommand( - """ - SELECT s.id, s.name, s.name_folded, s.display_name_folded, + $""" + SELECT s.id, s.name, s.name_folded, {displayNameFoldedProjection}, f.lang, s.kind, s.signature FROM symbols s JOIN files f ON f.id = s.file_id diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 2e613be6b..fe890a23a 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -6821,6 +6821,27 @@ public void RunBackfillFold_DoesNotRewriteCurrentFoldRowsWhenCSharpIsAbsent_Issu Assert.Contains("checkpoint: skipped (already complete)", humanOutput, StringComparison.Ordinal); Assert.Single(Directory.GetDirectories(checkpointRoot)); + using (var driftedDb = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + using var drift = driftedDb.Connection.CreateCommand(); + drift.CommandText = "UPDATE symbols SET name_folded = 'definitely_wrong' WHERE name = 'run'"; + Assert.Equal(1, drift.ExecuteNonQuery()); + } + + var repaired = RunBackfill(); + Assert.Equal(CommandExitCodes.Success, repaired.ExitCode); + Assert.True(repaired.Json.GetProperty("rewrite_all").GetBoolean()); + Assert.Equal(1, repaired.Json.GetProperty("symbols").GetInt32()); + Assert.False(repaired.Json.GetProperty("was_already_complete").GetBoolean()); + Assert.False(repaired.Json.GetProperty("checkpoint_skipped").GetBoolean()); + Assert.Equal(2, Directory.GetDirectories(checkpointRoot).Length); + using (var repairedDb = new DbContext(DbOpenIntent.QueryOnly, dbPath)) + { + using var folded = repairedDb.Connection.CreateCommand(); + folded.CommandText = "SELECT name_folded FROM symbols WHERE name = 'run'"; + Assert.Equal("run", folded.ExecuteScalar()); + } + using (var pendingDb = new DbContext(DbOpenIntent.WriteIndex, dbPath)) { var pendingWriter = new DbWriter(pendingDb.Connection); @@ -6833,7 +6854,7 @@ public void RunBackfillFold_DoesNotRewriteCurrentFoldRowsWhenCSharpIsAbsent_Issu Assert.Equal(0, resumed.Json.GetProperty("symbol_references").GetInt32()); Assert.False(resumed.Json.GetProperty("was_already_complete").GetBoolean()); Assert.False(resumed.Json.GetProperty("checkpoint_skipped").GetBoolean()); - Assert.Equal(2, Directory.GetDirectories(checkpointRoot).Length); + Assert.Equal(3, Directory.GetDirectories(checkpointRoot).Length); using var verifyPending = new DbContext(DbOpenIntent.QueryOnly, dbPath); Assert.Null(verifyPending.GetMetaString(DbWriter.FoldBackfillGraphRefreshPendingMetaKey)); }