From d94b4164eb444dd1ebb01b2007fe4aa3429e9801 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 18:19:00 +0900 Subject: [PATCH 1/5] Document DB compatibility policy (#1718) --- COMPATIBILITY.md | 127 ++++++++++++++++++++++++++++ DEVELOPER_GUIDE.md | 5 ++ README.md | 5 ++ changelog.d/unreleased/1718.docs.md | 17 ++++ 4 files changed, 154 insertions(+) create mode 100644 COMPATIBILITY.md create mode 100644 changelog.d/unreleased/1718.docs.md diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md new file mode 100644 index 0000000000..64ef0d942d --- /dev/null +++ b/COMPATIBILITY.md @@ -0,0 +1,127 @@ +# CodeIndex DB Compatibility + +> **[日本語版はこちら / Japanese version](#codeindex-db-compatibility日本語)** + +This document defines the compatibility contract between `cdidx` binaries and +the local SQLite database under `.cdidx/codeindex.db`. + +## Supported Readers + +The public compatibility boundary is the `cdidx` CLI and MCP server reading a +database created by a released `cdidx` binary. The SQLite schema is an internal +storage format, not a public API. + +Within a supported release line, newer binaries must read older databases and +degrade optional features explicitly when stored readiness metadata is missing +or stale. Older binaries may read newer databases only when the newer database +does not advertise unknown readiness or contract stamps. If an older binary sees +unknown persisted contract stamps, it must degrade loudly in `status` output and +must refuse writes that could silently discard newer data. + +## Schema and Readiness Stamps + +`PRAGMA user_version` is a readiness bitmap, not a linear migration number: + +| Bit | Field | Meaning | +|---|---|---| +| `1` | `graph_table_available` / graph readiness | `symbol_references` has been fully populated for graph queries. | +| `2` | `issues_table_available` / issue readiness | `file_issues` has been populated for validation results. | +| `4` | `fold_ready` | Folded-name columns are current for Unicode-aware exact-name matching. | + +Additional per-feature contract versions live in `codeindex_meta`, including +folded-key metadata, C# symbol-name and metadata-target versions, SQL graph +contract stamps, hotspot-family readiness, index writer version, indexed HEAD +metadata, unknown-extension counts, filesystem case-sensitivity, MAC profile, +and DB/WAL/status diagnostics. These stamps let readers distinguish a feature +that is absent, stale, or newer than the running binary. + +## Version Skew Behavior + +Use `cdidx status --json` or `cdidx status --check --json` before relying on a +database across binary upgrades or downgrades. + +| Scenario | Expected behavior | Operator action | +|---|---|---| +| New binary reads an older DB | Queries continue where possible. Missing readiness fields report degraded status and include remediation strings. | Run the recommended maintenance command from `status`, usually `cdidx backfill-fold` or `cdidx index --rebuild`. | +| Same binary reads its own DB | `status --check --json` reports `index_matches_workspace: true` when file content and HEAD metadata match. | No rebuild required. | +| Older binary reads a newer DB | `index_newer_than_reader` becomes `true` when unknown readiness bits or contract stamps exceed the binary's maximum. Mutating commands refuse to write unsafe newer DBs. | Use the newer `cdidx` binary that wrote the DB, or rebuild the index with the older binary only after accepting loss of newer feature data. | +| Read-only CI artifact | Query commands may use `--read-only` / `--immutable`. Mutating commands reject read-only DBs. | Pin the `cdidx` binary version with the DB artifact when possible. | + +## Rebuild Requirements + +Additive schema changes should be readable by newer binaries without requiring a +full rebuild. Prefer in-place maintenance for derived data, such as +`cdidx backfill-fold`, when a feature can be refreshed from existing rows. + +A rebuild is required when: + +- `status` recommends `cdidx index --rebuild`; +- the workspace and DB are intentionally being reset to an older binary version; +- the database is corrupt or fails `cdidx db --integrity-check`; +- a release note explicitly calls out a breaking storage change. + +Breaking DB changes must be rare and must document the minimum binary version, +the downgrade behavior, and the rebuild path in release notes. + +## CodeIndex DB Compatibility（日本語） + +この文書は、`cdidx` binary と `.cdidx/codeindex.db` のローカル SQLite database +の互換性契約を定義します。 + +## 対応する reader + +公開される互換性境界は、release 済み `cdidx` binary が作成した database を +`cdidx` CLI / MCP server が読むことです。SQLite schema は内部 storage format +であり、公開 API ではありません。 + +対応 release line 内では、新しい binary は古い database を読み、保存済みの +readiness metadata が不足または stale の場合は optional feature を明示的に +degrade しなければなりません。古い binary が新しい database を読めるのは、 +その database が未知の readiness / contract stamp を示していない場合だけです。 +未知の永続 contract stamp を見た古い binary は `status` で明示的に degrade を +報告し、新しい data を黙って破棄しうる write を拒否します。 + +## Schema と readiness stamp + +`PRAGMA user_version` は線形 migration number ではなく readiness bitmap です。 + +| Bit | Field | 意味 | +|---|---|---| +| `1` | `graph_table_available` / graph readiness | graph query 用の `symbol_references` が完全に作成済み。 | +| `2` | `issues_table_available` / issue readiness | validation result 用の `file_issues` が作成済み。 | +| `4` | `fold_ready` | Unicode-aware exact-name matching 用の folded-name column が最新。 | + +追加の feature contract version は `codeindex_meta` に保存されます。これには +folded-key metadata、C# symbol-name / metadata-target version、SQL graph +contract stamp、hotspot-family readiness、index writer version、indexed HEAD +metadata、unknown-extension count、filesystem case-sensitivity、MAC profile、 +DB/WAL/status diagnostics が含まれます。reader はこれらの stamp により、feature +が存在しないのか、stale なのか、実行中 binary より新しいのかを判別できます。 + +## Version skew 時の動作 + +binary upgrade / downgrade をまたいで database を使う前に、 +`cdidx status --json` または `cdidx status --check --json` を確認してください。 + +| 状況 | 期待される動作 | 操作者の対応 | +|---|---|---| +| 新しい binary が古い DB を読む | 可能な query は継続します。不足した readiness field は degraded status と remediation を返します。 | `status` の推奨に従い、通常は `cdidx backfill-fold` または `cdidx index --rebuild` を実行します。 | +| 同じ binary が自身の DB を読む | file content と HEAD metadata が一致すると `status --check --json` は `index_matches_workspace: true` を返します。 | rebuild は不要です。 | +| 古い binary が新しい DB を読む | 未知の readiness bit または contract stamp が binary の最大値を超えると `index_newer_than_reader` が `true` になります。mutating command は unsafe な write を拒否します。 | その DB を書いた新しい `cdidx` binary を使うか、新しい feature data が失われることを受け入れて古い binary で index を作り直します。 | +| read-only CI artifact | query command は `--read-only` / `--immutable` を利用できます。mutating command は read-only DB を拒否します。 | 可能なら DB artifact と `cdidx` binary version を一緒に pin します。 | + +## Rebuild が必要な場合 + +Additive schema change は、full rebuild を要求せずに新しい binary で読めるべきです。 +既存 row から再生成できる derived data は、`cdidx backfill-fold` のような +in-place maintenance を優先します。 + +rebuild が必要なのは次の場合です。 + +- `status` が `cdidx index --rebuild` を推奨している; +- workspace と DB を意図的に古い binary version へ戻す; +- database が壊れている、または `cdidx db --integrity-check` に失敗する; +- release note が breaking storage change を明示している。 + +Breaking DB change は稀であるべきで、minimum binary version、downgrade behavior、 +rebuild path を release note に記載しなければなりません。 diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 73ca325365..2823ff15b4 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -54,6 +54,11 @@ cdidx search AuthService --db /artifacts/codeindex.db --immutable Mutating commands such as `index`, `backfill-fold`, `optimize`, and `vacuum` require writable storage and reject read-only database opens. +Database compatibility across `cdidx` binary upgrades and downgrades is +documented in [COMPATIBILITY.md](COMPATIBILITY.md). Keep that policy updated +whenever readiness bits, `codeindex_meta` contract stamps, or rebuild +requirements change. + `backfill-fold --dry-run` previews the folded-key rows that would be rewritten without mutating the DB or stamping FoldReady. The MCP `backfill_fold` tool accepts the same preview as `dry_run: true`, and also accepts `force: true` to diff --git a/README.md b/README.md index ba52e204ff..3e1b15529d 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,8 @@ Install choice and network notes: See [DISTRIBUTION.md](DISTRIBUTION.md) for the full channel matrix and [isolated network install notes](USER_GUIDE.md#isolated-networks-and-proxies). +For database compatibility across `cdidx` upgrades and downgrades, see +[COMPATIBILITY.md](COMPATIBILITY.md). ### Validate @@ -337,6 +339,9 @@ extractor fixture を確認できます。詳細は | UTF-8 JSON pipeline | CLI の `--json` 出力は BOM なし UTF-8 で書き出され、human output 向けに色を強制していても ANSI escape sequence を含みません。 | | script 向け query pipeline の stderr を静かにする | `--quiet`、`-q`、`--silent`、`CDIDX_QUIET=1` で informational stderr を抑制し、error 行だけを残します。`--quiet` は `--verbose` より優先されます。 | +`cdidx` の upgrade / downgrade をまたぐ database 互換性については +[COMPATIBILITY.md](COMPATIBILITY.md) を参照してください。 + ターミナル、スクリプト、CI、AI ツールから同じリポジトリを繰り返し検索する 場合は `cdidx` が向いています。1回限りのテキスト検索には `rg` が向いています。 diff --git a/changelog.d/unreleased/1718.docs.md b/changelog.d/unreleased/1718.docs.md new file mode 100644 index 0000000000..b4d36d41c5 --- /dev/null +++ b/changelog.d/unreleased/1718.docs.md @@ -0,0 +1,17 @@ +--- +category: docs +issues: + - 1718 +affected: + - COMPATIBILITY.md + - README.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Documented DB compatibility policy (#1718)** — Added a bilingual compatibility contract for schema readiness bits, `codeindex_meta` stamps, upgrade/downgrade behavior, and rebuild requirements. + +## 日本語 + +- **DB 互換性ポリシーを文書化しました (#1718)** — schema readiness bit、`codeindex_meta` stamp、upgrade / downgrade 時の動作、rebuild が必要な条件を説明する bilingual compatibility contract を追加しました。 From 8cbc2650a24312b3c4ab432f225506876ffab121 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 18:55:46 +0900 Subject: [PATCH 2/5] Add DB checkpoint restore commands (#1721) --- DEVELOPER_GUIDE.md | 7 + README.md | 2 + changelog.d/unreleased/1721.added.md | 23 ++ src/CodeIndex/Cli/CliFlagSchema.cs | 1 + src/CodeIndex/Cli/ConsoleUi.cs | 4 +- src/CodeIndex/Cli/DbCommandRunner.cs | 359 +++++++++++++++++- .../Cli/IndexCommandRunner.Maintenance.cs | 11 +- src/CodeIndex/Cli/IndexCommandRunner.cs | 1 + src/CodeIndex/Cli/JsonOutputContracts.cs | 28 ++ src/CodeIndex/Cli/ProgramRunner.cs | 2 +- tests/CodeIndex.Tests/ConsoleUiTests.cs | 2 +- tests/CodeIndex.Tests/DbCommandRunnerTests.cs | 95 ++++- 12 files changed, 514 insertions(+), 21 deletions(-) create mode 100644 changelog.d/unreleased/1721.added.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 2823ff15b4..7a1f3ce503 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -54,6 +54,13 @@ cdidx search AuthService --db /artifacts/codeindex.db --immutable Mutating commands such as `index`, `backfill-fold`, `optimize`, and `vacuum` require writable storage and reject read-only database opens. +Use `cdidx db checkpoint ` to take a filesystem snapshot of +`codeindex.db` plus existing WAL/SHM sidecars before risky maintenance, and use +`cdidx db restore ` to roll back. Checkpoints live next to the DB under +`.checkpoints//`; restore keeps the pre-restore files under +`.restore-backup-/`. `backfill-fold` creates an automatic +checkpoint before it mutates rows unless `--no-checkpoint` is passed. + Database compatibility across `cdidx` binary upgrades and downgrades is documented in [COMPATIBILITY.md](COMPATIBILITY.md). Keep that policy updated whenever readiness bits, `codeindex_meta` contract stamps, or rebuild diff --git a/README.md b/README.md index 3e1b15529d..3c1bd5ec4c 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ After the first command, use these cues and follow-up commands: | Edits or branch switches | Refresh incrementally with `--files`, `--commits`, or `--changed-between ` instead of rebuilding. See [Quick Start](USER_GUIDE.md#quick-start) and [Incremental update reliability](USER_GUIDE.md#incremental-update-reliability). | | Intentional rebuilds | Interactive terminals ask before deleting the DB. Scripts and CI must pass `--yes` or `--force`. | | Long-lived DB compaction | Run `cdidx optimize` or `cdidx index --optimize` to compact FTS5 segments immediately. Incremental refreshes also optimize opportunistically. | +| Maintenance rollback | Run `cdidx db checkpoint ` before risky DB maintenance and `cdidx db restore ` to roll back. `backfill-fold` creates an automatic checkpoint unless `--no-checkpoint` is passed. | | Permission or I/O scan errors | `cdidx` records the scan error, continues other directories, and writes `.cdidx/scan-checkpoint.json` so same-HEAD retries can skip completed directories. | Output controls: @@ -325,6 +326,7 @@ extractor fixture を確認できます。詳細は | 編集後やブランチ切り替え後 | 再構築ではなく `--files`、`--commits`、`--changed-between ` で差分更新します。詳細は [クイックスタート](USER_GUIDE.md#クイックスタート) と [インクリメンタル更新の信頼性](USER_GUIDE.md#インクリメンタル更新の信頼性) を参照してください。 | | 意図的な再構築 | interactive terminal では既存 DB 削除前に確認を求めます。script / CI では `--yes` または `--force` が必要です。 | | 長期間使っている DB の compact | `cdidx optimize` または `cdidx index --optimize` で FTS5 segment をすぐに compact できます。差分更新中も必要に応じて自動 optimize します。 | +| 保守作業の rollback | risky な DB 保守の前に `cdidx db checkpoint `、戻す場合は `cdidx db restore ` を使います。`backfill-fold` は `--no-checkpoint` を渡さない限り自動 checkpoint を作成します。 | | 権限や I/O の scan error | `cdidx` は scan error を記録し、他のディレクトリの走査を続けます。同じ HEAD の再実行では `.cdidx/scan-checkpoint.json` により成功済みディレクトリを読み飛ばせます。 | 出力を整える option: diff --git a/changelog.d/unreleased/1721.added.md b/changelog.d/unreleased/1721.added.md new file mode 100644 index 0000000000..611022ace1 --- /dev/null +++ b/changelog.d/unreleased/1721.added.md @@ -0,0 +1,23 @@ +--- +category: added +issues: + - 1721 +affected: + - src/CodeIndex/Cli/DbCommandRunner.cs + - src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - src/CodeIndex/Cli/CliFlagSchema.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - tests/CodeIndex.Tests/DbCommandRunnerTests.cs + - README.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Added DB checkpoints and restore (#1721)** — `cdidx db checkpoint`, `cdidx db checkpoints --list`, and `cdidx db restore` can snapshot and roll back `codeindex.db`, while `backfill-fold` now creates an automatic checkpoint before mutating rows unless `--no-checkpoint` is passed. + +## 日本語 + +- **DB checkpoint / restore を追加しました (#1721)** — `cdidx db checkpoint`、`cdidx db checkpoints --list`、`cdidx db restore` で `codeindex.db` を snapshot / rollback でき、`backfill-fold` は `--no-checkpoint` を渡さない限り row を変更する前に自動 checkpoint を作成します。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 5d74b0b149..66408913fc 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -273,6 +273,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--rebuild", Description = "Delete existing DB and rebuild from scratch", Commands = Set("index") }, new() { Name = "--optimize", Description = "Optimize the existing FTS5 table without scanning files", Commands = Set("index") }, new() { Name = "--dry-run", Description = "Preview without writing", Commands = Set("index", "backfill-fold") }, + new() { Name = "--no-checkpoint", Description = "Skip the automatic DB checkpoint before maintenance", Commands = Set("backfill-fold") }, new() { Name = "--force", Description = "Bypass the per-database index lock", Commands = Set("index") }, new() { Name = "--duration-format", ValuePlaceholder = "", Description = "Index elapsed time display format", Commands = Set("index") }, new() { Name = "--max-file-bytes", ValuePlaceholder = "", Description = "Override the per-file indexing size limit", Commands = Set("index") }, diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index 9debd46a6f..2504a9cae4 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -62,7 +62,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = [ ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), ("hooks", "cdidx hooks [--project ] [--force] [--json]"), - ("backfill-fold", "cdidx backfill-fold [--db ] [--dry-run] [--json]"), + ("backfill-fold", "cdidx backfill-fold [--db ] [--dry-run] [--no-checkpoint] [--json]"), ("optimize", "cdidx optimize [--db ] [--json]"), ("vacuum", "cdidx vacuum [--db ] [--json]"), ("index-commits", "cdidx index --commits [id ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), @@ -85,7 +85,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = ("workspace", "cdidx workspace [name] [--json]"), ("config", "cdidx config show [--json]"), ("validate-config", "cdidx validate-config"), - ("db", "cdidx db --integrity-check [--db ] [--json]"), + ("db", "cdidx db --integrity-check [--db ] [--json] | cdidx db checkpoint [name] [--db ] [--json] | cdidx db checkpoints --list [--db ] [--json] | cdidx db restore [--db ] [--json]"), ("diff", "cdidx diff [--json] [--summary-only] [--detailed] [--limit ]"), ("report", "cdidx report --output [--db ] [--json] [--log-lines ] [--no-log] [--include-args]"), ("validate", "cdidx validate [--db ] [--json] [--verbose] [--kind ] [--path ]"), diff --git a/src/CodeIndex/Cli/DbCommandRunner.cs b/src/CodeIndex/Cli/DbCommandRunner.cs index e1a9e91e8d..62b56484a8 100644 --- a/src/CodeIndex/Cli/DbCommandRunner.cs +++ b/src/CodeIndex/Cli/DbCommandRunner.cs @@ -10,7 +10,11 @@ namespace CodeIndex.Cli; /// public static class DbCommandRunner { - public static int RunIntegrityCheck(string[] cmdArgs, JsonSerializerOptions jsonOptions) + private const string CheckpointsDirectorySuffix = ".checkpoints"; + private const string AutoCheckpointPrefix = "auto-"; + private static readonly char[] InvalidCheckpointNameChars = Path.GetInvalidFileNameChars(); + + public static int Run(string[] cmdArgs, JsonSerializerOptions jsonOptions) { var options = ParseArgs(cmdArgs); if (options.ShowHelp) @@ -25,16 +29,55 @@ public static int RunIntegrityCheck(string[] cmdArgs, JsonSerializerOptions json jsonOptions, options.ParseError, CommandExitCodes.UsageError, - "Run `cdidx db --integrity-check --help` to see the supported command shape.", + "Run `cdidx db --help` to see the supported command shape.", CommandErrorCodes.UsageError); - if (!options.IntegrityCheck) + return options.Mode switch + { + DbCommandMode.IntegrityCheck => RunIntegrityCheck(options, jsonOptions), + DbCommandMode.Checkpoint => RunCheckpoint(options, jsonOptions), + DbCommandMode.ListCheckpoints => RunListCheckpoints(options, jsonOptions), + DbCommandMode.Restore => RunRestore(options, jsonOptions), + _ => WriteCommandError( + options.Json, + jsonOptions, + "db requires a mode", + CommandExitCodes.UsageError, + "Use `--integrity-check`, `checkpoint`, `checkpoints --list`, or `restore `.", + CommandErrorCodes.UsageError), + }; + } + + public static int RunIntegrityCheck(string[] cmdArgs, JsonSerializerOptions jsonOptions) + { + var options = ParseArgs(cmdArgs); + return options.ParseError == null && options.Mode == DbCommandMode.IntegrityCheck + ? RunIntegrityCheck(options, jsonOptions) + : Run(cmdArgs, jsonOptions); + } + + internal static string CreateAutomaticCheckpoint(string dbPath) + { + var fullDbPath = Path.GetFullPath(DbPathResolver.NormalizeDbPath(dbPath)); + var name = AutoCheckpointPrefix + DateTimeOffset.UtcNow.ToString("yyyyMMddHHmmssfff", System.Globalization.CultureInfo.InvariantCulture); + return CreateCheckpoint(fullDbPath, name).CheckpointPath; + } + + private static int RunIntegrityCheck(DbCommandOptions options, JsonSerializerOptions jsonOptions) + { + if (options.ShowHelp) + { + ConsoleUi.PrintUsage(); + return CommandExitCodes.Success; + } + + if (options.ParseError != null) return WriteCommandError( options.Json, jsonOptions, - "db requires a mode flag", + options.ParseError, CommandExitCodes.UsageError, - "Pass `--integrity-check` to run `PRAGMA integrity_check` on the database.", + "Run `cdidx db --integrity-check --help` to see the supported command shape.", CommandErrorCodes.UsageError); var dbPath = options.DbPath; @@ -96,6 +139,115 @@ public static int RunIntegrityCheck(string[] cmdArgs, JsonSerializerOptions json } } + private static int RunCheckpoint(DbCommandOptions options, JsonSerializerOptions jsonOptions) + { + if (!ValidateWritableFileDb(options, jsonOptions, "checkpoint", out var fullDbPath, out var validationExitCode)) + return validationExitCode; + + try + { + var result = CreateCheckpoint(fullDbPath, options.Name ?? MakeTimestampCheckpointName()); + if (options.Json) + { + Console.WriteLine(JsonSerializer.Serialize( + new DbCheckpointJsonResult("success", fullDbPath, result.Name, result.CheckpointPath, result.Files), + CliJsonSerializerContextFactory.Create(jsonOptions).DbCheckpointJsonResult)); + } + else + { + Console.WriteLine("Created database checkpoint."); + Console.WriteLine($" database : {fullDbPath}"); + Console.WriteLine($" name : {result.Name}"); + Console.WriteLine($" checkpoint: {result.CheckpointPath}"); + Console.WriteLine($" files : {ConsoleUi.Counted(result.Files.Count, "file")}"); + } + + return CommandExitCodes.Success; + } + catch (Exception ex) + { + return WriteCommandError( + options.Json, + jsonOptions, + $"failed to create database checkpoint: {ex.Message}", + CommandExitCodes.DatabaseError, + "Ensure the database and checkpoint directory are writable, then retry `cdidx db checkpoint`.", + CommandErrorCodes.DbError); + } + } + + private static int RunListCheckpoints(DbCommandOptions options, JsonSerializerOptions jsonOptions) + { + if (!TryResolveFileDb(options.DbPath, out var fullDbPath, out var error)) + return WriteCommandError(options.Json, jsonOptions, error, CommandExitCodes.DatabaseError, "Use a filesystem database path, not a SQLite URI.", CommandErrorCodes.DbError); + + var entries = ListCheckpoints(fullDbPath); + if (options.Json) + { + Console.WriteLine(JsonSerializer.Serialize( + new DbCheckpointListJsonResult(fullDbPath, entries), + CliJsonSerializerContextFactory.Create(jsonOptions).DbCheckpointListJsonResult)); + } + else + { + Console.WriteLine("Database checkpoints"); + Console.WriteLine($" database: {fullDbPath}"); + if (entries.Count == 0) + { + Console.WriteLine(" checkpoints: none"); + } + else + { + foreach (var entry in entries) + Console.WriteLine($" {entry.Name} {entry.CreatedAtUtc} {entry.Bytes:N0} bytes"); + } + } + + return CommandExitCodes.Success; + } + + private static int RunRestore(DbCommandOptions options, JsonSerializerOptions jsonOptions) + { + if (string.IsNullOrWhiteSpace(options.Name)) + return WriteCommandError(options.Json, jsonOptions, "restore requires a checkpoint name", CommandExitCodes.UsageError, "Use `cdidx db restore --db `.", CommandErrorCodes.UsageError); + if (!ValidateWritableFileDb(options, jsonOptions, "restore", out var fullDbPath, out var validationExitCode)) + return validationExitCode; + + try + { + var checkpointPath = GetCheckpointPath(fullDbPath, options.Name); + if (!Directory.Exists(checkpointPath)) + return WriteCommandError(options.Json, jsonOptions, $"checkpoint not found: {options.Name}", CommandExitCodes.NotFound, "Run `cdidx db checkpoints --list` to see available checkpoints.", CommandErrorCodes.DbNotFound); + + var backupPath = RestoreCheckpoint(fullDbPath, options.Name, checkpointPath); + if (options.Json) + { + Console.WriteLine(JsonSerializer.Serialize( + new DbRestoreJsonResult("success", fullDbPath, options.Name, checkpointPath, backupPath), + CliJsonSerializerContextFactory.Create(jsonOptions).DbRestoreJsonResult)); + } + else + { + Console.WriteLine("Restored database checkpoint."); + Console.WriteLine($" database : {fullDbPath}"); + Console.WriteLine($" checkpoint: {options.Name}"); + Console.WriteLine($" backup : {backupPath}"); + } + + return CommandExitCodes.Success; + } + catch (Exception ex) + { + return WriteCommandError( + options.Json, + jsonOptions, + $"failed to restore database checkpoint: {ex.Message}", + CommandExitCodes.DatabaseError, + "Ensure no cdidx writer is running, then retry `cdidx db restore `.", + CommandErrorCodes.DbError); + } + } + // PRAGMA integrity_check returns a single row `"ok"` when the file passes every consistency // probe, otherwise it returns up to N rows of corruption findings. The pragma itself only // reads the database, so a read-only connection is sufficient and avoids the WAL-mode @@ -130,7 +282,8 @@ internal static DbCommandOptions ParseArgs(string[] args) { var dbPath = Path.Combine(".cdidx", "codeindex.db"); var json = false; - var integrityCheck = false; + var mode = DbCommandMode.None; + string? name = null; string? parseError = null; for (var i = 0; i < args.Length; i++) @@ -147,7 +300,41 @@ internal static DbCommandOptions ParseArgs(string[] args) json = true; break; case "--integrity-check": - integrityCheck = true; + mode = mode == DbCommandMode.None ? DbCommandMode.IntegrityCheck : mode; + if (mode != DbCommandMode.IntegrityCheck) + parseError = "db accepts only one mode"; + break; + case "checkpoint": + mode = mode == DbCommandMode.None ? DbCommandMode.Checkpoint : mode; + if (mode != DbCommandMode.Checkpoint) + { + parseError = "db accepts only one mode"; + break; + } + if (i + 1 < args.Length && !args[i + 1].StartsWith("-", StringComparison.Ordinal)) + name = args[++i]; + break; + case "checkpoints": + mode = mode == DbCommandMode.None ? DbCommandMode.ListCheckpoints : mode; + if (mode != DbCommandMode.ListCheckpoints) + parseError = "db accepts only one mode"; + break; + case "restore": + mode = mode == DbCommandMode.None ? DbCommandMode.Restore : mode; + if (mode != DbCommandMode.Restore) + { + parseError = "db accepts only one mode"; + break; + } + if (i + 1 < args.Length && !args[i + 1].StartsWith("-", StringComparison.Ordinal)) + name = args[++i]; + else + parseError = "restore requires a checkpoint name"; + break; + case "--list": + if (mode == DbCommandMode.ListCheckpoints) + break; + parseError = "--list is only valid with `cdidx db checkpoints --list`"; break; case "--help" or "-h": return new DbCommandOptions { ShowHelp = true, DbPath = dbPath, Json = json }; @@ -155,7 +342,7 @@ internal static DbCommandOptions ParseArgs(string[] args) if (args[i].StartsWith('-')) parseError = $"db does not support option: '{args[i]}'"; else - parseError = $"db does not accept positional arguments: '{args[i]}'"; + parseError = $"unknown db command or argument: '{args[i]}'"; break; } @@ -167,11 +354,149 @@ internal static DbCommandOptions ParseArgs(string[] args) { DbPath = dbPath, Json = json, - IntegrityCheck = integrityCheck, + Mode = mode, + Name = name, ParseError = parseError, }; } + private static bool ValidateWritableFileDb(DbCommandOptions options, JsonSerializerOptions jsonOptions, string command, out string fullDbPath, out int exitCode) + { + exitCode = CommandExitCodes.Success; + if (!TryResolveFileDb(options.DbPath, out fullDbPath, out var error)) + { + WriteCommandError(options.Json, jsonOptions, error, CommandExitCodes.DatabaseError, "Use a filesystem database path, not a SQLite URI.", CommandErrorCodes.DbError); + exitCode = CommandExitCodes.DatabaseError; + return false; + } + + if (!File.Exists(LongPath.EnsureWindowsPrefix(fullDbPath))) + { + WriteCommandError( + options.Json, + jsonOptions, + $"database not found: {fullDbPath}", + CommandExitCodes.NotFound, + "Point `--db` at an existing `codeindex.db`, or run `cdidx index ` first to create one.", + CommandErrorCodes.DbNotFound); + exitCode = CommandExitCodes.NotFound; + return false; + } + + if (DbPathResolver.UriRequestsReadOnly(options.DbPath)) + { + WriteCommandError( + options.Json, + jsonOptions, + $"database must be writable for {command}: {options.DbPath}", + CommandExitCodes.DatabaseError, + "Point `--db` at a writable filesystem path.", + CommandErrorCodes.DbNotWritable); + exitCode = CommandExitCodes.DatabaseError; + return false; + } + + return true; + } + + private static bool TryResolveFileDb(string dbPath, out string fullDbPath, out string error) + { + fullDbPath = string.Empty; + error = string.Empty; + if (dbPath.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) + { + error = $"database command requires a filesystem path: {dbPath}"; + return false; + } + + fullDbPath = Path.GetFullPath(DbPathResolver.NormalizeDbPath(dbPath)); + return true; + } + + private static DbCheckpointOperationResult CreateCheckpoint(string fullDbPath, string name) + { + ValidateCheckpointName(name); + var checkpointPath = GetCheckpointPath(fullDbPath, name); + if (Directory.Exists(checkpointPath)) + throw new InvalidOperationException($"checkpoint already exists: {name}"); + + Directory.CreateDirectory(checkpointPath); + CopyIfExists(fullDbPath, Path.Combine(checkpointPath, Path.GetFileName(fullDbPath))); + CopyIfExists(fullDbPath + "-wal", Path.Combine(checkpointPath, Path.GetFileName(fullDbPath) + "-wal")); + CopyIfExists(fullDbPath + "-shm", Path.Combine(checkpointPath, Path.GetFileName(fullDbPath) + "-shm")); + File.WriteAllText(Path.Combine(checkpointPath, "manifest.txt"), $"name={name}{Environment.NewLine}created_at_utc={DateTimeOffset.UtcNow:O}{Environment.NewLine}db={fullDbPath}{Environment.NewLine}"); + + var files = Directory.GetFiles(checkpointPath).Select(Path.GetFileName).Where(f => f is not null).Select(f => f!).OrderBy(f => f, StringComparer.Ordinal).ToList(); + return new DbCheckpointOperationResult(name, checkpointPath, files); + } + + private static List ListCheckpoints(string fullDbPath) + { + var root = GetCheckpointRoot(fullDbPath); + if (!Directory.Exists(root)) + return []; + + return Directory.GetDirectories(root) + .Select(path => + { + var info = new DirectoryInfo(path); + var bytes = Directory.GetFiles(path).Sum(file => new FileInfo(file).Length); + return new DbCheckpointListEntryJsonResult(info.Name, path, info.CreationTimeUtc.ToString("O", System.Globalization.CultureInfo.InvariantCulture), bytes); + }) + .OrderBy(entry => entry.Name, StringComparer.Ordinal) + .ToList(); + } + + private static string RestoreCheckpoint(string fullDbPath, string name, string checkpointPath) + { + ValidateCheckpointName(name); + SqliteConnection.ClearAllPools(); + var backupPath = fullDbPath + ".restore-backup-" + DateTimeOffset.UtcNow.ToString("yyyyMMddHHmmssfff", System.Globalization.CultureInfo.InvariantCulture); + Directory.CreateDirectory(backupPath); + MoveIfExists(fullDbPath, Path.Combine(backupPath, Path.GetFileName(fullDbPath))); + MoveIfExists(fullDbPath + "-wal", Path.Combine(backupPath, Path.GetFileName(fullDbPath) + "-wal")); + MoveIfExists(fullDbPath + "-shm", Path.Combine(backupPath, Path.GetFileName(fullDbPath) + "-shm")); + + CopyIfExists(Path.Combine(checkpointPath, Path.GetFileName(fullDbPath)), fullDbPath); + CopyIfExists(Path.Combine(checkpointPath, Path.GetFileName(fullDbPath) + "-wal"), fullDbPath + "-wal"); + CopyIfExists(Path.Combine(checkpointPath, Path.GetFileName(fullDbPath) + "-shm"), fullDbPath + "-shm"); + return backupPath; + } + + private static void ValidateCheckpointName(string name) + { + if (string.IsNullOrWhiteSpace(name) + || name is "." or ".." + || name.IndexOfAny(InvalidCheckpointNameChars) >= 0 + || name.Contains(Path.DirectorySeparatorChar) + || (Path.AltDirectorySeparatorChar != '\0' && name.Contains(Path.AltDirectorySeparatorChar))) + throw new ArgumentException($"invalid checkpoint name: {name}"); + } + + private static string MakeTimestampCheckpointName() + => DateTimeOffset.UtcNow.ToString("yyyyMMddHHmmssfff", System.Globalization.CultureInfo.InvariantCulture); + + private static string GetCheckpointRoot(string fullDbPath) + => fullDbPath + CheckpointsDirectorySuffix; + + private static string GetCheckpointPath(string fullDbPath, string name) + { + ValidateCheckpointName(name); + return Path.Combine(GetCheckpointRoot(fullDbPath), name); + } + + private static void CopyIfExists(string source, string destination) + { + if (File.Exists(LongPath.EnsureWindowsPrefix(source))) + File.Copy(LongPath.EnsureWindowsPrefix(source), LongPath.EnsureWindowsPrefix(destination), overwrite: false); + } + + private static void MoveIfExists(string source, string destination) + { + if (File.Exists(LongPath.EnsureWindowsPrefix(source))) + File.Move(LongPath.EnsureWindowsPrefix(source), LongPath.EnsureWindowsPrefix(destination)); + } + private static int WriteCommandError(bool json, JsonSerializerOptions jsonOptions, string message, int exitCode, string? hint = null, string? errorCode = null) { if (json) @@ -194,6 +519,20 @@ internal sealed class DbCommandOptions public string DbPath { get; init; } = string.Empty; public bool Json { get; init; } public bool ShowHelp { get; init; } - public bool IntegrityCheck { get; init; } + public DbCommandMode Mode { get; init; } + public string? Name { get; init; } public string? ParseError { get; init; } + + public bool IntegrityCheck => Mode == DbCommandMode.IntegrityCheck; } + +internal enum DbCommandMode +{ + None, + IntegrityCheck, + Checkpoint, + ListCheckpoints, + Restore, +} + +internal sealed record DbCheckpointOperationResult(string Name, string CheckpointPath, List Files); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs b/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs index 97d5744c6f..ae3df7e4d8 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs @@ -10,6 +10,7 @@ public static partial class IndexCommandRunner private static readonly string[] AcceptedBackfillFoldFlags = [ "--db", "--json", "--dry-run", "--help", + "--no-checkpoint", ]; public static int RunBackfillFold(string[] cmdArgs, JsonSerializerOptions jsonOptions) => @@ -191,6 +192,9 @@ internal static int RunBackfillFold( } else { + if (!options.NoCheckpoint) + DbCommandRunner.CreateAutomaticCheckpoint(options.DbPath); + using var transaction = writer.BeginTransaction(); (symbols, symbolReferences) = writer.BackfillFoldedColumns( rewriteAll, @@ -321,6 +325,7 @@ private static BackfillFoldCommandOptions ParseBackfillFoldArgs(string[] args) var dbPath = Path.Combine(".cdidx", "codeindex.db"); var json = false; var dryRun = false; + var noCheckpoint = false; for (int i = 0; i < args.Length; i++) { @@ -335,8 +340,11 @@ private static BackfillFoldCommandOptions ParseBackfillFoldArgs(string[] args) case "--dry-run": dryRun = true; break; + case "--no-checkpoint": + noCheckpoint = true; + break; case "--help" or "-h": - return new BackfillFoldCommandOptions { ShowHelp = true, DbPath = dbPath, Json = json, DryRun = dryRun }; + return new BackfillFoldCommandOptions { ShowHelp = true, DbPath = dbPath, Json = json, DryRun = dryRun, NoCheckpoint = noCheckpoint }; default: if (args[i].StartsWith('-')) { @@ -360,6 +368,7 @@ private static BackfillFoldCommandOptions ParseBackfillFoldArgs(string[] args) DbPath = dbPath, Json = json, DryRun = dryRun, + NoCheckpoint = noCheckpoint, }; } diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index 78f638c8d9..f34beb228d 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -1351,6 +1351,7 @@ public sealed class BackfillFoldCommandOptions public string DbPath { get; init; } = Path.Combine(".cdidx", "codeindex.db"); public bool Json { get; init; } public bool DryRun { get; init; } + public bool NoCheckpoint { get; init; } public string? ParseError { get; init; } } diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 52efc6dbbf..4685010926 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -43,6 +43,30 @@ internal sealed record DbIntegrityCheckJsonResult( [property: JsonPropertyName("ok")] bool Ok, [property: JsonPropertyName("issues")] List Issues); +internal sealed record DbCheckpointJsonResult( + [property: JsonPropertyName("status")] string Status, + [property: JsonPropertyName("db_path")] string DbPath, + [property: JsonPropertyName("name")] string Name, + [property: JsonPropertyName("checkpoint_path")] string CheckpointPath, + [property: JsonPropertyName("files")] List Files); + +internal sealed record DbCheckpointListJsonResult( + [property: JsonPropertyName("db_path")] string DbPath, + [property: JsonPropertyName("checkpoints")] List Checkpoints); + +internal sealed record DbCheckpointListEntryJsonResult( + [property: JsonPropertyName("name")] string Name, + [property: JsonPropertyName("checkpoint_path")] string CheckpointPath, + [property: JsonPropertyName("created_at_utc")] string CreatedAtUtc, + [property: JsonPropertyName("bytes")] long Bytes); + +internal sealed record DbRestoreJsonResult( + [property: JsonPropertyName("status")] string Status, + [property: JsonPropertyName("db_path")] string DbPath, + [property: JsonPropertyName("name")] string Name, + [property: JsonPropertyName("checkpoint_path")] string CheckpointPath, + [property: JsonPropertyName("backup_path")] string BackupPath); + internal sealed record DiffSummaryJsonResult( [property: JsonPropertyName("left_file_count")] long LeftFileCount, [property: JsonPropertyName("right_file_count")] long RightFileCount, @@ -306,7 +330,11 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(CompactSearchResult[]))] [JsonSerializable(typeof(CommandErrorJsonResult))] [JsonSerializable(typeof(ConfigShowJsonResult))] +[JsonSerializable(typeof(DbCheckpointJsonResult))] +[JsonSerializable(typeof(DbCheckpointListEntryJsonResult))] +[JsonSerializable(typeof(DbCheckpointListJsonResult))] [JsonSerializable(typeof(DbIntegrityCheckJsonResult))] +[JsonSerializable(typeof(DbRestoreJsonResult))] [JsonSerializable(typeof(DefinitionResult))] [JsonSerializable(typeof(Dictionary))] [JsonSerializable(typeof(Dictionary))] diff --git a/src/CodeIndex/Cli/ProgramRunner.cs b/src/CodeIndex/Cli/ProgramRunner.cs index 450576e4c3..2a8a0bf0c6 100644 --- a/src/CodeIndex/Cli/ProgramRunner.cs +++ b/src/CodeIndex/Cli/ProgramRunner.cs @@ -262,7 +262,7 @@ internal static int Run( CommandExitCodes.UsageError, "use `cdidx config show`."), "workspace" => WorkspaceCommandRunner.Run(subArgs, jsonOptions), - "db" => DbCommandRunner.RunIntegrityCheck(subArgs, jsonOptions), + "db" => DbCommandRunner.Run(subArgs, jsonOptions), "report" => ReportCommandRunner.Run(subArgs, jsonOptions, appVersion), "test-extractor" => RunTestExtractor(subArgs, jsonOptions), _ when IsProjectPathArg(commandName) diff --git a/tests/CodeIndex.Tests/ConsoleUiTests.cs b/tests/CodeIndex.Tests/ConsoleUiTests.cs index f04fda55c0..1e68a286cb 100644 --- a/tests/CodeIndex.Tests/ConsoleUiTests.cs +++ b/tests/CodeIndex.Tests/ConsoleUiTests.cs @@ -61,7 +61,7 @@ public void PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags() Assert.Contains("cdidx hooks [--project ] [--force] [--json]", output); Assert.Contains("cdidx index --commits [id ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); Assert.Contains("cdidx index --files [path ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); - Assert.Contains("cdidx backfill-fold [--db ] [--dry-run] [--json]", output); + Assert.Contains("cdidx backfill-fold [--db ] [--dry-run] [--no-checkpoint] [--json]", output); Assert.Contains("cdidx optimize [--db ] [--json]", output); Assert.Contains("cdidx license", output); Assert.Contains("cdidx completions ", output); diff --git a/tests/CodeIndex.Tests/DbCommandRunnerTests.cs b/tests/CodeIndex.Tests/DbCommandRunnerTests.cs index 3c8f6faec9..752da95f3b 100644 --- a/tests/CodeIndex.Tests/DbCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/DbCommandRunnerTests.cs @@ -6,8 +6,8 @@ namespace CodeIndex.Tests; /// -/// Tests for `cdidx db --integrity-check` (issue #1517). -/// `cdidx db --integrity-check` のテスト (issue #1517)。 +/// Tests for `cdidx db` maintenance commands. +/// `cdidx db` 保守コマンドのテスト。 /// [Collection("SQLite pool sensitive")] public class DbCommandRunnerTests @@ -49,7 +49,26 @@ public void ParseArgs_PositionalArgRecordsParseError() var options = DbCommandRunner.ParseArgs(["something"]); Assert.NotNull(options.ParseError); - Assert.Contains("positional", options.ParseError); + Assert.Contains("unknown db command", options.ParseError); + } + + [Fact] + public void ParseArgs_CheckpointCommandSetsModeAndName() + { + var options = DbCommandRunner.ParseArgs(["checkpoint", "before-upgrade"]); + + Assert.Equal(DbCommandMode.Checkpoint, options.Mode); + Assert.Equal("before-upgrade", options.Name); + Assert.Null(options.ParseError); + } + + [Fact] + public void ParseArgs_RestoreRequiresName() + { + var options = DbCommandRunner.ParseArgs(["restore"]); + + Assert.Equal(DbCommandMode.Restore, options.Mode); + Assert.Contains("requires", options.ParseError); } [Fact] @@ -58,7 +77,7 @@ public void Run_WithoutModeFlag_ReturnsUsageError() var (exitCode, _, stderr) = RunAndCaptureStreams([]); Assert.Equal(CommandExitCodes.UsageError, exitCode); - Assert.Contains("db requires a mode flag", stderr); + Assert.Contains("db requires a mode", stderr); Assert.Contains("--integrity-check", stderr); } @@ -145,6 +164,70 @@ public void Run_CleanDb_JsonReportsOkTrueAndEmptyIssues() } } + [Fact] + public void Run_CheckpointAndRestore_RestoresDatabaseBytes() + { + var root = Path.Combine(Path.GetTempPath(), $"cdidx_db_checkpoint_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(root, "codeindex.db"); + Directory.CreateDirectory(root); + try + { + using (var db = new DbContext(dbPath)) + db.InitializeSchema(); + SqliteConnection.ClearAllPools(); + + var originalBytes = File.ReadAllBytes(dbPath); + var (checkpointExit, checkpointOut, _) = RunAndCaptureStreams(["checkpoint", "saved", "--db", dbPath]); + Assert.Equal(CommandExitCodes.Success, checkpointExit); + Assert.Contains("saved", checkpointOut); + + File.WriteAllText(dbPath, "changed"); + + var (restoreExit, restoreOut, _) = RunAndCaptureStreams(["restore", "saved", "--db", dbPath]); + + Assert.Equal(CommandExitCodes.Success, restoreExit); + Assert.Contains("Restored", restoreOut); + Assert.Equal(originalBytes, File.ReadAllBytes(dbPath)); + Assert.Single(Directory.GetDirectories(root, "codeindex.db.restore-backup-*")); + } + finally + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + + [Fact] + public void Run_CheckpointsList_JsonIncludesCreatedCheckpoint() + { + var root = Path.Combine(Path.GetTempPath(), $"cdidx_db_checkpoint_list_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(root, "codeindex.db"); + Directory.CreateDirectory(root); + try + { + using (var db = new DbContext(dbPath)) + db.InitializeSchema(); + SqliteConnection.ClearAllPools(); + + var (checkpointExit, _) = RunAndCaptureJson(["checkpoint", "listed", "--db", dbPath, "--json"]); + Assert.Equal(CommandExitCodes.Success, checkpointExit); + + var (listExit, json) = RunAndCaptureJson(["checkpoints", "--list", "--db", dbPath, "--json"]); + + Assert.Equal(CommandExitCodes.Success, listExit); + var checkpoints = json.GetProperty("checkpoints"); + Assert.Single(checkpoints.EnumerateArray()); + Assert.Equal("listed", checkpoints[0].GetProperty("name").GetString()); + } + finally + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + [Fact] public void Run_CorruptedDb_ReturnsDatabaseError() { @@ -180,14 +263,14 @@ public void Run_CorruptedDb_ReturnsDatabaseError() private (int ExitCode, string StdOut, string StdErr) RunAndCaptureStreams(string[] args) { using var capture = ConsoleCapture.Start(captureOut: true, captureError: true); - var exitCode = DbCommandRunner.RunIntegrityCheck(args, _jsonOptions); + var exitCode = DbCommandRunner.Run(args, _jsonOptions); return (exitCode, capture.Out!.ToString()!, capture.Error!.ToString()!); } private (int ExitCode, JsonElement Json) RunAndCaptureJson(string[] args) { using var capture = ConsoleCapture.Start(captureOut: true); - var exitCode = DbCommandRunner.RunIntegrityCheck(args, _jsonOptions); + var exitCode = DbCommandRunner.Run(args, _jsonOptions); using var document = JsonDocument.Parse(capture.Out!.ToString()!); return (exitCode, document.RootElement.Clone()); } From 8db0418b56062928ad307888d4b32cddd3f637ae Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 19:17:42 +0900 Subject: [PATCH 3/5] Harden DB checkpoint restore (#1721) --- src/CodeIndex/Cli/DbCommandRunner.cs | 30 ++++++++++++++--- tests/CodeIndex.Tests/DbCommandRunnerTests.cs | 32 +++++++++++++++++++ 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/CodeIndex/Cli/DbCommandRunner.cs b/src/CodeIndex/Cli/DbCommandRunner.cs index 8ed71d59f5..6a9ae1a211 100644 --- a/src/CodeIndex/Cli/DbCommandRunner.cs +++ b/src/CodeIndex/Cli/DbCommandRunner.cs @@ -575,15 +575,28 @@ private static bool TryResolveFileDb(string dbPath, out string fullDbPath, out s private static DbCheckpointOperationResult CreateCheckpoint(string fullDbPath, string name) { ValidateCheckpointName(name); + var root = GetCheckpointRoot(fullDbPath); var checkpointPath = GetCheckpointPath(fullDbPath, name); if (Directory.Exists(checkpointPath)) throw new InvalidOperationException($"checkpoint already exists: {name}"); - Directory.CreateDirectory(checkpointPath); - CopyIfExists(fullDbPath, Path.Combine(checkpointPath, Path.GetFileName(fullDbPath))); - CopyIfExists(fullDbPath + "-wal", Path.Combine(checkpointPath, Path.GetFileName(fullDbPath) + "-wal")); - CopyIfExists(fullDbPath + "-shm", Path.Combine(checkpointPath, Path.GetFileName(fullDbPath) + "-shm")); - File.WriteAllText(Path.Combine(checkpointPath, "manifest.txt"), $"name={name}{Environment.NewLine}created_at_utc={DateTimeOffset.UtcNow:O}{Environment.NewLine}db={fullDbPath}{Environment.NewLine}"); + Directory.CreateDirectory(root); + var tempPath = Path.Combine(root, ".tmp-" + name + "-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempPath); + try + { + CopyIfExists(fullDbPath, Path.Combine(tempPath, Path.GetFileName(fullDbPath))); + CopyIfExists(fullDbPath + "-wal", Path.Combine(tempPath, Path.GetFileName(fullDbPath) + "-wal")); + CopyIfExists(fullDbPath + "-shm", Path.Combine(tempPath, Path.GetFileName(fullDbPath) + "-shm")); + File.WriteAllText(Path.Combine(tempPath, "manifest.txt"), $"name={name}{Environment.NewLine}created_at_utc={DateTimeOffset.UtcNow:O}{Environment.NewLine}db={fullDbPath}{Environment.NewLine}"); + Directory.Move(tempPath, checkpointPath); + } + catch + { + if (Directory.Exists(tempPath)) + Directory.Delete(tempPath, recursive: true); + throw; + } var files = Directory.GetFiles(checkpointPath).Select(Path.GetFileName).Where(f => f is not null).Select(f => f!).OrderBy(f => f, StringComparer.Ordinal).ToList(); return new DbCheckpointOperationResult(name, checkpointPath, files); @@ -595,7 +608,10 @@ private static List ListCheckpoints(string full if (!Directory.Exists(root)) return []; + var dbFileName = Path.GetFileName(fullDbPath); return Directory.GetDirectories(root) + .Where(path => !Path.GetFileName(path).StartsWith(".tmp-", StringComparison.Ordinal)) + .Where(path => File.Exists(LongPath.EnsureWindowsPrefix(Path.Combine(path, dbFileName)))) .Select(path => { var info = new DirectoryInfo(path); @@ -610,6 +626,10 @@ private static string RestoreCheckpoint(string fullDbPath, string name, string c { ValidateCheckpointName(name); SqliteConnection.ClearAllPools(); + var checkpointDbPath = Path.Combine(checkpointPath, Path.GetFileName(fullDbPath)); + if (!File.Exists(LongPath.EnsureWindowsPrefix(checkpointDbPath))) + throw new InvalidOperationException($"checkpoint is incomplete: {name}"); + var backupPath = fullDbPath + ".restore-backup-" + DateTimeOffset.UtcNow.ToString("yyyyMMddHHmmssfff", System.Globalization.CultureInfo.InvariantCulture); Directory.CreateDirectory(backupPath); MoveIfExists(fullDbPath, Path.Combine(backupPath, Path.GetFileName(fullDbPath))); diff --git a/tests/CodeIndex.Tests/DbCommandRunnerTests.cs b/tests/CodeIndex.Tests/DbCommandRunnerTests.cs index 9818366cd1..b814c6e728 100644 --- a/tests/CodeIndex.Tests/DbCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/DbCommandRunnerTests.cs @@ -311,6 +311,38 @@ public void Run_CheckpointsList_JsonIncludesCreatedCheckpoint() } } + [Fact] + public void Run_RestoreIncompleteCheckpoint_ReturnsErrorAndKeepsDatabase() + { + var root = Path.Combine(Path.GetTempPath(), $"cdidx_db_checkpoint_bad_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(root, "codeindex.db"); + Directory.CreateDirectory(root); + try + { + using (var db = new DbContext(dbPath)) + db.InitializeSchema(); + SqliteConnection.ClearAllPools(); + + var originalBytes = File.ReadAllBytes(dbPath); + var checkpointPath = Path.Combine(root, "codeindex.db.checkpoints", "bad"); + Directory.CreateDirectory(checkpointPath); + File.WriteAllText(Path.Combine(checkpointPath, "manifest.txt"), "name=bad"); + + var (restoreExit, _, stderr) = RunAndCaptureStreams(["restore", "bad", "--db", dbPath]); + + Assert.Equal(CommandExitCodes.DatabaseError, restoreExit); + Assert.Contains("incomplete", stderr); + Assert.Equal(originalBytes, File.ReadAllBytes(dbPath)); + Assert.Empty(Directory.GetDirectories(root, "codeindex.db.restore-backup-*")); + } + finally + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + [Fact] public void Run_CorruptedDb_ReturnsDatabaseError() { From 9ff45b56579bf027edc66148fd005c38f2a613bf Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 19:23:17 +0900 Subject: [PATCH 4/5] Make DB restore failure-safe (#1721) --- src/CodeIndex/Cli/DbCommandRunner.cs | 60 ++++++++++++++++--- tests/CodeIndex.Tests/DbCommandRunnerTests.cs | 36 +++++++++++ 2 files changed, 88 insertions(+), 8 deletions(-) diff --git a/src/CodeIndex/Cli/DbCommandRunner.cs b/src/CodeIndex/Cli/DbCommandRunner.cs index 6a9ae1a211..a2bc44334b 100644 --- a/src/CodeIndex/Cli/DbCommandRunner.cs +++ b/src/CodeIndex/Cli/DbCommandRunner.cs @@ -14,6 +14,7 @@ public static class DbCommandRunner private const string CheckpointsDirectorySuffix = ".checkpoints"; private const string AutoCheckpointPrefix = "auto-"; private static readonly char[] InvalidCheckpointNameChars = Path.GetInvalidFileNameChars(); + internal static Action? RestoreFailureAfterBackupForTesting { get; set; } public static int Run(string[] cmdArgs, JsonSerializerOptions jsonOptions) { @@ -630,15 +631,39 @@ private static string RestoreCheckpoint(string fullDbPath, string name, string c if (!File.Exists(LongPath.EnsureWindowsPrefix(checkpointDbPath))) throw new InvalidOperationException($"checkpoint is incomplete: {name}"); + var restoreTempPath = fullDbPath + ".restore-tmp-" + DateTimeOffset.UtcNow.ToString("yyyyMMddHHmmssfff", System.Globalization.CultureInfo.InvariantCulture); var backupPath = fullDbPath + ".restore-backup-" + DateTimeOffset.UtcNow.ToString("yyyyMMddHHmmssfff", System.Globalization.CultureInfo.InvariantCulture); - Directory.CreateDirectory(backupPath); - MoveIfExists(fullDbPath, Path.Combine(backupPath, Path.GetFileName(fullDbPath))); - MoveIfExists(fullDbPath + "-wal", Path.Combine(backupPath, Path.GetFileName(fullDbPath) + "-wal")); - MoveIfExists(fullDbPath + "-shm", Path.Combine(backupPath, Path.GetFileName(fullDbPath) + "-shm")); - - CopyIfExists(Path.Combine(checkpointPath, Path.GetFileName(fullDbPath)), fullDbPath); - CopyIfExists(Path.Combine(checkpointPath, Path.GetFileName(fullDbPath) + "-wal"), fullDbPath + "-wal"); - CopyIfExists(Path.Combine(checkpointPath, Path.GetFileName(fullDbPath) + "-shm"), fullDbPath + "-shm"); + Directory.CreateDirectory(restoreTempPath); + try + { + CopyIfExists(checkpointDbPath, Path.Combine(restoreTempPath, Path.GetFileName(fullDbPath))); + CopyIfExists(Path.Combine(checkpointPath, Path.GetFileName(fullDbPath) + "-wal"), Path.Combine(restoreTempPath, Path.GetFileName(fullDbPath) + "-wal")); + CopyIfExists(Path.Combine(checkpointPath, Path.GetFileName(fullDbPath) + "-shm"), Path.Combine(restoreTempPath, Path.GetFileName(fullDbPath) + "-shm")); + if (!File.Exists(LongPath.EnsureWindowsPrefix(Path.Combine(restoreTempPath, Path.GetFileName(fullDbPath))))) + throw new InvalidOperationException($"checkpoint staging failed: {name}"); + + Directory.CreateDirectory(backupPath); + MoveIfExists(fullDbPath, Path.Combine(backupPath, Path.GetFileName(fullDbPath))); + MoveIfExists(fullDbPath + "-wal", Path.Combine(backupPath, Path.GetFileName(fullDbPath) + "-wal")); + MoveIfExists(fullDbPath + "-shm", Path.Combine(backupPath, Path.GetFileName(fullDbPath) + "-shm")); + + RestoreFailureAfterBackupForTesting?.Invoke(); + + MoveIfExists(Path.Combine(restoreTempPath, Path.GetFileName(fullDbPath)), fullDbPath); + MoveIfExists(Path.Combine(restoreTempPath, Path.GetFileName(fullDbPath) + "-wal"), fullDbPath + "-wal"); + MoveIfExists(Path.Combine(restoreTempPath, Path.GetFileName(fullDbPath) + "-shm"), fullDbPath + "-shm"); + } + catch + { + RestoreBackedUpFiles(fullDbPath, backupPath); + throw; + } + finally + { + if (Directory.Exists(restoreTempPath)) + Directory.Delete(restoreTempPath, recursive: true); + } + return backupPath; } @@ -676,6 +701,25 @@ private static void MoveIfExists(string source, string destination) File.Move(LongPath.EnsureWindowsPrefix(source), LongPath.EnsureWindowsPrefix(destination)); } + private static void RestoreBackedUpFiles(string fullDbPath, string backupPath) + { + if (!Directory.Exists(backupPath)) + return; + + DeleteIfExists(fullDbPath); + DeleteIfExists(fullDbPath + "-wal"); + DeleteIfExists(fullDbPath + "-shm"); + MoveIfExists(Path.Combine(backupPath, Path.GetFileName(fullDbPath)), fullDbPath); + MoveIfExists(Path.Combine(backupPath, Path.GetFileName(fullDbPath) + "-wal"), fullDbPath + "-wal"); + MoveIfExists(Path.Combine(backupPath, Path.GetFileName(fullDbPath) + "-shm"), fullDbPath + "-shm"); + } + + private static void DeleteIfExists(string path) + { + if (File.Exists(LongPath.EnsureWindowsPrefix(path))) + File.Delete(LongPath.EnsureWindowsPrefix(path)); + } + internal static DbCommandOptions ParseArgs(string[] args) { var dbPath = Path.Combine(".cdidx", "codeindex.db"); diff --git a/tests/CodeIndex.Tests/DbCommandRunnerTests.cs b/tests/CodeIndex.Tests/DbCommandRunnerTests.cs index b814c6e728..9428e6352a 100644 --- a/tests/CodeIndex.Tests/DbCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/DbCommandRunnerTests.cs @@ -343,6 +343,42 @@ public void Run_RestoreIncompleteCheckpoint_ReturnsErrorAndKeepsDatabase() } } + [Fact] + public void Run_RestoreFailureAfterBackup_RestoresOriginalDatabase() + { + var root = Path.Combine(Path.GetTempPath(), $"cdidx_db_checkpoint_fail_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(root, "codeindex.db"); + Directory.CreateDirectory(root); + try + { + using (var db = new DbContext(dbPath)) + db.InitializeSchema(); + SqliteConnection.ClearAllPools(); + + var originalBytes = File.ReadAllBytes(dbPath); + var (checkpointExit, _, _) = RunAndCaptureStreams(["checkpoint", "saved", "--db", dbPath]); + Assert.Equal(CommandExitCodes.Success, checkpointExit); + + File.WriteAllText(dbPath, "changed"); + DbCommandRunner.RestoreFailureAfterBackupForTesting = () => throw new IOException("injected restore failure"); + + var (restoreExit, _, stderr) = RunAndCaptureStreams(["restore", "saved", "--db", dbPath]); + + Assert.Equal(CommandExitCodes.DatabaseError, restoreExit); + Assert.Contains("injected restore failure", stderr); + Assert.Equal("changed", File.ReadAllText(dbPath)); + Assert.Single(Directory.GetDirectories(root, "codeindex.db.restore-backup-*")); + Assert.Empty(Directory.GetDirectories(root, "codeindex.db.restore-tmp-*")); + } + finally + { + DbCommandRunner.RestoreFailureAfterBackupForTesting = null; + SqliteConnection.ClearAllPools(); + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + [Fact] public void Run_CorruptedDb_ReturnsDatabaseError() { From 4f282a0a7c280c9145991f48cd625368f8d83203 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 19:53:00 +0900 Subject: [PATCH 5/5] Fix backfill usage output for checkpoint flag (#1721) --- src/CodeIndex/Cli/ConsoleUi.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index 25849ee9cb..30c4107eb5 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -62,7 +62,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = [ ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), ("hooks", "cdidx hooks [--project ] [--force] [--json]"), - ("backfill-fold", "cdidx backfill-fold [--db ] [--dry-run] [--json]"), + ("backfill-fold", "cdidx backfill-fold [--db ] [--dry-run] [--no-checkpoint] [--json]"), ("optimize", "cdidx optimize [--db ] [--json]"), ("vacuum", "cdidx vacuum [--db ] [--json]"), ("index-commits", "cdidx index --commits [id ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"),