From e3448bf09f746023dc9422a6cb2943e744f0c32d Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 21:19:06 +0900 Subject: [PATCH 1/4] Fix atomic sensitive write profiles (#3688) --- DEVELOPER_GUIDE.md | 4 +- changelog.d/unreleased/3688.security.md | 21 +++++++ src/CodeIndex/Cli/AtomicFileWriter.cs | 60 ++++++++++++++++++- src/CodeIndex/Cli/DataDirectorySecurity.cs | 2 +- .../Cli/IndexCommandRunner.FullScan.cs | 6 +- src/CodeIndex/Cli/SuggestionStore.cs | 2 +- src/CodeIndex/Cli/UpdateChecker.cs | 6 +- .../CodeIndex.Tests/AtomicFileWriterTests.cs | 33 ++++++++++ 8 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 changelog.d/unreleased/3688.security.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index a1ac5bfbd8..8fe7d644bb 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -59,7 +59,7 @@ Development contracts: | `.cdidx/` | Created with mode `0700`. | | `codeindex.db` plus WAL/SHM sidecars | Mode `0600` is applied when the files exist. | | `suggestions-*.json` suggestion stores | Written atomically with owner-only mode `0600` on POSIX. | -| Atomic file writes | `AtomicFileWriter` writes to a sibling temp file, applies the requested POSIX mode before replacement, flushes file contents, renames over the target, and fsyncs the parent directory on Unix. If the parent directory flush fails after replacement, the command fails explicitly so callers know the file was replaced but directory durability was not confirmed. Windows skips directory fsync because the helper only promises it on supported Unix platforms. | +| Atomic file writes | `AtomicFileWriter` writes to a sibling temp file, applies the requested POSIX mode before replacement, flushes file contents, renames over the target, and fsyncs the parent directory on Unix. Callers must use the `Sensitive` write profile for local state, caches, suggestions, checkpoints, and other private payloads; user-requested exports and reports use the default `Public` profile unless their content is explicitly private. If the parent directory flush fails after replacement, the command fails explicitly so callers know the file was replaced but directory durability was not confirmed. Windows skips directory fsync because the helper only promises it on supported Unix platforms. | | Index lock metadata sidecars and active workspace `active.json` | Written as owner-only files and read through small bounded buffers so stale or corrupted diagnostics cannot expose local paths more broadly or force unbounded allocation. | | Checkpoint roots, snapshot directories, manifest files, copied DB/WAL/SHM snapshots, and restore staging/backup directories | Forced owner-only on POSIX. | | `status --json` | Reports `data_dir_mode` and `db_file_mode` when the platform exposes Unix file modes. | @@ -2265,7 +2265,7 @@ net9 CI lane に合わせる場合は `FRAMEWORK=net9.0 make test` を使いま | `.cdidx/` | mode `0700` で作成。 | | `codeindex.db` と WAL/SHM sidecar | ファイルが存在する場合は mode `0600` を適用。 | | `suggestions-*.json` suggestion store | POSIX では owner-only の mode `0600` で atomic write します。 | -| atomic file write | `AtomicFileWriter` は sibling temp file に書き込み、要求された POSIX mode を置換前に適用し、file content を flush してから target へ rename し、Unix では parent directory を fsync します。置換後に parent directory flush が失敗した場合、file は置換済みだが directory durability を確認できていないことが caller に分かるよう command は明示的に失敗します。Windows では、この helper の directory fsync 保証は supported Unix platform に限定されるため skip します。 | +| atomic file write | `AtomicFileWriter` は sibling temp file に書き込み、要求された POSIX mode を置換前に適用し、file content を flush してから target へ rename し、Unix では parent directory を fsync します。local state、cache、suggestion、checkpoint など private payload には `Sensitive` write profile を使い、user-requested export や report は内容が明示的に private でない限り既定の `Public` profile を使います。置換後に parent directory flush が失敗した場合、file は置換済みだが directory durability を確認できていないことが caller に分かるよう command は明示的に失敗します。Windows では、この helper の directory fsync 保証は supported Unix platform に限定されるため skip します。 | | index lock metadata sidecar と active workspace の `active.json` | owner-only file として書き、stale / corrupt diagnostic が local path を広く漏らしたり unbounded allocation を強制したりしないよう小さな bounded buffer で読みます。 | | database checkpoint root、snapshot directory、manifest file、copy された DB/WAL/SHM snapshot、restore staging/backup directory | POSIX では owner-only に固定。 | | `status --json` | platform が Unix file mode を公開する場合、`data_dir_mode` と `db_file_mode` を報告。 | diff --git a/changelog.d/unreleased/3688.security.md b/changelog.d/unreleased/3688.security.md new file mode 100644 index 0000000000..0c5b1093f1 --- /dev/null +++ b/changelog.d/unreleased/3688.security.md @@ -0,0 +1,21 @@ +--- +category: security +issues: + - 3688 +affected: + - src/CodeIndex/Cli/AtomicFileWriter.cs + - src/CodeIndex/Cli/DataDirectorySecurity.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/SuggestionStore.cs + - src/CodeIndex/Cli/UpdateChecker.cs + - tests/CodeIndex.Tests/AtomicFileWriterTests.cs + - DEVELOPER_GUIDE.md +--- + +## English + +- **Sensitive atomic writes now use an explicit private write profile (#3688)** — local state JSON files such as scan checkpoints, update caches, and suggestion stores are written through a sensitive profile that creates POSIX temp files as owner-only before the atomic replacement and still reports parent-directory durability failures. + +## 日本語 + +- **sensitive な atomic write が明示的な private write profile を使うようになりました (#3688)** — scan checkpoint、update cache、suggestion store などの local state JSON は sensitive profile 経由で書き込まれ、atomic replace 前の POSIX temp file 生成時点から owner-only になり、parent directory durability の失敗も引き続き報告します。 diff --git a/src/CodeIndex/Cli/AtomicFileWriter.cs b/src/CodeIndex/Cli/AtomicFileWriter.cs index fec91a8550..25145934a0 100644 --- a/src/CodeIndex/Cli/AtomicFileWriter.cs +++ b/src/CodeIndex/Cli/AtomicFileWriter.cs @@ -9,6 +9,12 @@ internal static class AtomicFileWriter { internal static Action? FlushParentDirectoryForTesting { get; set; } + public enum WriteProfile + { + Public, + Sensitive, + } + public static void WriteText(string path, string contents, Encoding encoding, Action? applyFileMode = null) { Write( @@ -22,12 +28,45 @@ public static void WriteText(string path, string contents, Encoding encoding, Ac applyFileMode); } + public static void WriteText(string path, string contents, Encoding encoding, WriteProfile profile) + { + Write( + path, + stream => + { + using var writer = new StreamWriter(stream, encoding, bufferSize: 1024, leaveOpen: true); + writer.Write(contents); + writer.Flush(); + }, + profile); + } + public static void WriteJson(string path, T value, JsonSerializerOptions? options = null, Action? applyFileMode = null) { Write(path, stream => JsonSerializer.Serialize(stream, value, options), applyFileMode); } + public static void WriteJson(string path, T value, JsonSerializerOptions? options, WriteProfile profile) + { + Write(path, stream => JsonSerializer.Serialize(stream, value, options), profile); + } + + public static void WriteJson(string path, T value, WriteProfile profile) + { + WriteJson(path, value, options: null, profile); + } + public static void Write(string path, Action writeContents, Action? applyFileMode = null) + => WriteCore(path, writeContents, applyFileMode, WriteProfile.Public); + + public static void Write(string path, Action writeContents, WriteProfile profile) + => WriteCore(path, writeContents, ResolveProfileModeCallback(profile), profile); + + private static void WriteCore( + string path, + Action writeContents, + Action? applyFileMode, + WriteProfile profile) { ArgumentNullException.ThrowIfNull(writeContents); @@ -38,7 +77,7 @@ public static void Write(string path, Action writeContents, Action writeContents, Action? ResolveProfileModeCallback(WriteProfile profile) + => profile == WriteProfile.Sensitive ? DataDirectorySecurity.ApplyPrivateFileMode : null; + private static void FlushParentDirectory(string path) { var directory = Path.GetDirectoryName(Path.GetFullPath(path)); diff --git a/src/CodeIndex/Cli/DataDirectorySecurity.cs b/src/CodeIndex/Cli/DataDirectorySecurity.cs index cc1c4e6110..b4476188a4 100644 --- a/src/CodeIndex/Cli/DataDirectorySecurity.cs +++ b/src/CodeIndex/Cli/DataDirectorySecurity.cs @@ -59,7 +59,7 @@ public static void ApplyPrivateFileMode(string path) public static void WritePrivateText(string path, string contents, Encoding? encoding = null) { var outputEncoding = encoding is null || encoding.CodePage == Encoding.UTF8.CodePage ? Utf8NoBom : encoding; - AtomicFileWriter.WriteText(path, contents, outputEncoding, ApplyPrivateFileMode); + AtomicFileWriter.WriteText(path, contents, outputEncoding, AtomicFileWriter.WriteProfile.Sensitive); } public static byte[]? ReadBytesWithinLimit(string path, int maxBytes, FileShare share = FileShare.Read) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index a9b8bd8844..1b0c6b9b9d 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -500,7 +500,11 @@ private static void SaveScanCheckpoint( if (WriteScanCheckpointForTesting != null) WriteScanCheckpointForTesting(path); else - AtomicFileWriter.WriteJson(path, checkpoint, new JsonSerializerOptions { WriteIndented = true }); + AtomicFileWriter.WriteJson( + path, + checkpoint, + new JsonSerializerOptions { WriteIndented = true }, + AtomicFileWriter.WriteProfile.Sensitive); } catch (Exception ex) when (IsScanCheckpointPersistenceException(ex)) { diff --git a/src/CodeIndex/Cli/SuggestionStore.cs b/src/CodeIndex/Cli/SuggestionStore.cs index 7fd9899593..0b37335874 100644 --- a/src/CodeIndex/Cli/SuggestionStore.cs +++ b/src/CodeIndex/Cli/SuggestionStore.cs @@ -786,7 +786,7 @@ private void SaveUnlocked(List records) Directory.CreateDirectory(dir); NormalizeRecordDefaults(records); - AtomicFileWriter.WriteJson(_filePath, records, s_jsonOptions, DataDirectorySecurity.ApplyPrivateFileMode); + AtomicFileWriter.WriteJson(_filePath, records, s_jsonOptions, AtomicFileWriter.WriteProfile.Sensitive); } private static bool HasUpstreamSubmission(SuggestionRecord record) => diff --git a/src/CodeIndex/Cli/UpdateChecker.cs b/src/CodeIndex/Cli/UpdateChecker.cs index ecc6dc0344..09c9d721f8 100644 --- a/src/CodeIndex/Cli/UpdateChecker.cs +++ b/src/CodeIndex/Cli/UpdateChecker.cs @@ -425,7 +425,11 @@ private static void TryWriteCache(string cachePath, UpdateCheckCache cache) checked_at = cache.CheckedAt.UtcDateTime.ToString("O", CultureInfo.InvariantCulture), latest_tag = cache.LatestTag, }; - AtomicFileWriter.WriteJson(cachePath, payload, applyFileMode: DataDirectorySecurity.ApplyPrivateFileMode); + AtomicFileWriter.WriteJson( + cachePath, + payload, + options: null, + profile: AtomicFileWriter.WriteProfile.Sensitive); } catch (Exception ex) { diff --git a/tests/CodeIndex.Tests/AtomicFileWriterTests.cs b/tests/CodeIndex.Tests/AtomicFileWriterTests.cs index 65be431ce0..0227b55a4f 100644 --- a/tests/CodeIndex.Tests/AtomicFileWriterTests.cs +++ b/tests/CodeIndex.Tests/AtomicFileWriterTests.cs @@ -1,3 +1,4 @@ +using System.Runtime.InteropServices; using System.Text; using CodeIndex.Cli; @@ -83,4 +84,36 @@ public void WriteText_ParentDirectoryFlushFailure_ReportsPostReplaceDurabilityFa TestProjectHelper.DeleteDirectory(projectRoot); } } + + [Fact] + public void WriteJson_SensitiveProfile_WritesPrivateFileAndFlushesParentDirectory_Issue3688() + { + var projectRoot = TestProjectHelper.CreateTempProject("atomic_sensitive"); + try + { + var path = Path.Combine(projectRoot, "scan-checkpoint.json"); + string? flushedDirectory = null; + AtomicFileWriter.FlushParentDirectoryForTesting = directory => flushedDirectory = directory; + + AtomicFileWriter.WriteJson( + path, + new { current_head = "HEAD", directories = new[] { "src" } }, + AtomicFileWriter.WriteProfile.Sensitive); + + Assert.Equal(projectRoot, flushedDirectory); + Assert.Contains("\"current_head\"", File.ReadAllText(path, Utf8NoBom), StringComparison.Ordinal); + Assert.Empty(Directory.GetFiles(projectRoot, ".scan-checkpoint.json.*.tmp")); + + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + var mode = File.GetUnixFileMode(path) & DataDirectorySecurity.PermissionBits; + Assert.Equal(DataDirectorySecurity.PrivateFileMode, mode); + } + } + finally + { + AtomicFileWriter.FlushParentDirectoryForTesting = null; + TestProjectHelper.DeleteDirectory(projectRoot); + } + } } From d0328ff8893f7803884e8d817d27ae06734b4f5e Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 21:26:08 +0900 Subject: [PATCH 2/4] Bound exception chain diagnostics (#3725) --- changelog.d/unreleased/3725.security.md | 18 +++++++++++ src/CodeIndex/Cli/GlobalToolLog.cs | 32 +++++++++++++++--- src/CodeIndex/Database/DbDebug.cs | 11 +++++-- tests/CodeIndex.Tests/DbDebugTests.cs | 36 +++++++++++++++++++++ tests/CodeIndex.Tests/GlobalToolLogTests.cs | 18 +++++++++++ 5 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 changelog.d/unreleased/3725.security.md diff --git a/changelog.d/unreleased/3725.security.md b/changelog.d/unreleased/3725.security.md new file mode 100644 index 0000000000..8333500a25 --- /dev/null +++ b/changelog.d/unreleased/3725.security.md @@ -0,0 +1,18 @@ +--- +category: security +issues: + - 3725 +affected: + - src/CodeIndex/Cli/GlobalToolLog.cs + - src/CodeIndex/Database/DbDebug.cs + - tests/CodeIndex.Tests/GlobalToolLogTests.cs + - tests/CodeIndex.Tests/DbDebugTests.cs +--- + +## English + +- **Persistent logs and DB debug dumps now bound exception chains (#3725)** — exception-chain diagnostics are capped and truncated with an explicit marker while continuing to classify messages instead of persisting raw paths, SQL, or secret-like values. + +## 日本語 + +- **persistent log と DB debug dump の exception chain に上限を設けました (#3725)** — exception-chain diagnostic は明示的な marker 付きで truncation され、raw path、SQL、secret 風の値を保存せず分類名として記録する動作を維持します。 diff --git a/src/CodeIndex/Cli/GlobalToolLog.cs b/src/CodeIndex/Cli/GlobalToolLog.cs index be254d8c54..98d865ed67 100644 --- a/src/CodeIndex/Cli/GlobalToolLog.cs +++ b/src/CodeIndex/Cli/GlobalToolLog.cs @@ -23,8 +23,10 @@ internal static class GlobalToolLog internal const int MirroredStderrWriteMaxChars = 8192; internal const int MaxLogSizeMb = 1024; internal const long MaxLogSizeBytes = MaxLogSizeMb * 1024L * 1024L; + internal const int MaxExceptionChainChars = 8192; internal const int RedactionArgumentLengthLimit = 8192; internal const string RedactionTruncationMarker = ""; + internal const string ExceptionChainTruncationMarker = "..."; private const string RedactedValue = ""; private const int PrivateLogDiagnosticEmitLimit = 16; private static readonly TimeSpan RedactionRegexTimeout = TimeSpan.FromSeconds(1); @@ -114,10 +116,10 @@ internal static string FormatExceptionChain(Exception ex, bool includeStacks = f { var sb = new StringBuilder(); AppendException(sb, ex, 0, includeStacks); - return sb.ToString().TrimEnd(); + return TruncateExceptionChain(sb.ToString().TrimEnd()); } - private static void AppendException(StringBuilder sb, Exception ex, int depth, bool includeStacks) + private static bool AppendException(StringBuilder sb, Exception ex, int depth, bool includeStacks) { var indent = new string(' ', depth * 2); sb.Append(indent); @@ -129,6 +131,8 @@ private static void AppendException(StringBuilder sb, Exception ex, int depth, b sb.Append(" message="); sb.Append(QuoteLogValue(DiagnosticRedactor.ClassifyException(ex))); sb.AppendLine(); + if (HasReachedExceptionChainLimit(sb)) + return false; if (includeStacks && !string.IsNullOrWhiteSpace(ex.StackTrace)) { @@ -137,6 +141,8 @@ private static void AppendException(StringBuilder sb, Exception ex, int depth, b sb.Append(indent); sb.Append(" stack: "); sb.AppendLine(DiagnosticRedactor.FormatExceptionStackLine(line.TrimEnd('\r'))); + if (HasReachedExceptionChainLimit(sb)) + return false; } } @@ -148,15 +154,31 @@ private static void AppendException(StringBuilder sb, Exception ex, int depth, b sb.Append(indent); sb.Append(" aggregate_inner_index="); sb.AppendLine(index.ToString(System.Globalization.CultureInfo.InvariantCulture)); - AppendException(sb, inner, depth + 1, includeStacks); + if (HasReachedExceptionChainLimit(sb) || !AppendException(sb, inner, depth + 1, includeStacks)) + return false; index++; } - return; + return true; } if (ex.InnerException is not null) - AppendException(sb, ex.InnerException, depth + 1, includeStacks); + return AppendException(sb, ex.InnerException, depth + 1, includeStacks); + + return true; + } + + private static bool HasReachedExceptionChainLimit(StringBuilder sb) => sb.Length >= MaxExceptionChainChars; + + private static string TruncateExceptionChain(string value) + { + if (value.Length <= MaxExceptionChainChars) + return value; + + if (MaxExceptionChainChars <= ExceptionChainTruncationMarker.Length) + return ExceptionChainTruncationMarker[..MaxExceptionChainChars]; + + return value[..(MaxExceptionChainChars - ExceptionChainTruncationMarker.Length)] + ExceptionChainTruncationMarker; } private static string QuoteLogValue(string value) => diff --git a/src/CodeIndex/Database/DbDebug.cs b/src/CodeIndex/Database/DbDebug.cs index c684812808..1bce3c7e6f 100644 --- a/src/CodeIndex/Database/DbDebug.cs +++ b/src/CodeIndex/Database/DbDebug.cs @@ -352,7 +352,8 @@ internal static void SnapshotRow(SqliteDataReader reader) } catch (Exception ex) { - row.Add((name, $"")); + var message = mode == DebugMode.Unsafe ? ex.Message : DiagnosticRedactor.ClassifyException(ex); + row.Add((name, $"")); (_lastRowReadExceptions ??= new List()).Add(ex); (_lastRowReadExceptionChains ??= new List()) .Add($"[{name}]\n{GlobalToolLog.FormatExceptionChain(ex, includeStacks: mode == DebugMode.Unsafe)}"); @@ -406,11 +407,15 @@ public static void DumpToStderr(Exception ex) } var rootCause = GetDeepestExceptionIncludingRowReads(ex); if (_lastRowReadExceptionChains is { Count: > 0 }) - sb.AppendLine($"Root cause: {rootCause.GetType().Name}: {rootCause.Message}"); + { + var message = mode == DebugMode.Unsafe ? rootCause.Message : DiagnosticRedactor.ClassifyException(rootCause); + sb.AppendLine($"Root cause: {rootCause.GetType().Name}: {message}"); + } if (mode != DebugMode.Unsafe && ex.StackTrace != null) { sb.AppendLine("Stack:"); - sb.AppendLine(ex.StackTrace); + foreach (var line in ex.StackTrace.Split('\n')) + sb.AppendLine(DiagnosticRedactor.FormatExceptionStackLine(line.TrimEnd('\r'))); } sb.AppendLine("--- END CDIDX_DEBUG ---"); Console.Error.Write(sb.ToString()); diff --git a/tests/CodeIndex.Tests/DbDebugTests.cs b/tests/CodeIndex.Tests/DbDebugTests.cs index e0f24f7e83..b02bd12b00 100644 --- a/tests/CodeIndex.Tests/DbDebugTests.cs +++ b/tests/CodeIndex.Tests/DbDebugTests.cs @@ -429,6 +429,42 @@ public void DumpToStderr_RedactedMode_HashesLargeStringsWithBoundedShape() } } + [Fact] + public void DumpToStderr_RedactedMode_BoundsAndRedactsExceptionChain_Issue3725() + { + using var env = EnvironmentVariableScope.Capture("CDIDX_DEBUG"); + env.Set("CDIDX_DEBUG", "1"); + const string secretPath = "/Users/widthdom/private/project/token.txt"; + const string secretToken = "0123456789abcdef0123456789abcdef"; + try + { + DbDebug.ResetContext(); + using var conn = new SqliteConnection("Data Source=:memory:"); + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT 1 AS id"; + using (var reader = cmd.ExecuteTrackedReader()) + { + Assert.True(reader.TrackedRead()); + } + + Exception exception = new IOException($"failed to read {secretPath} token={secretToken}"); + for (var i = 0; i < 220; i++) + exception = new InvalidOperationException($"outer {i} raw path {secretPath} token={secretToken}", exception); + + var output = CaptureStderr(() => DbDebug.DumpToStderr(exception)); + + Assert.Contains(GlobalToolLog.ExceptionChainTruncationMarker, output); + Assert.Contains("message=\"invalid_operation\"", output); + Assert.DoesNotContain(secretPath, output); + Assert.DoesNotContain(secretToken, output); + } + finally + { + DbDebug.ResetContext(); + } + } + [Fact] public void DumpToStderr_UnsafeMode_IncludesRawContent() { diff --git a/tests/CodeIndex.Tests/GlobalToolLogTests.cs b/tests/CodeIndex.Tests/GlobalToolLogTests.cs index b9d94996e9..37dcd25e24 100644 --- a/tests/CodeIndex.Tests/GlobalToolLogTests.cs +++ b/tests/CodeIndex.Tests/GlobalToolLogTests.cs @@ -321,6 +321,24 @@ public void FormatExceptionChain_ClassifiesInnerExceptionMessages_Issue3371() Assert.DoesNotContain("hunter2", formatted); } + [Fact] + public void FormatExceptionChain_BoundsLongNestedMessagesAndRedactsSensitiveValues_Issue3725() + { + const string secretPath = "/Users/widthdom/private/project/token.txt"; + const string secretToken = "0123456789abcdef0123456789abcdef"; + Exception exception = new IOException($"failed to read {secretPath} token={secretToken}"); + for (var i = 0; i < 220; i++) + exception = new InvalidOperationException($"outer {i} raw path {secretPath} token={secretToken}", exception); + + var formatted = GlobalToolLog.FormatExceptionChain(exception); + + Assert.True(formatted.Length <= GlobalToolLog.MaxExceptionChainChars); + Assert.Contains(GlobalToolLog.ExceptionChainTruncationMarker, formatted); + Assert.Contains("message=\"invalid_operation\"", formatted); + Assert.DoesNotContain(secretPath, formatted); + Assert.DoesNotContain(secretToken, formatted); + } + [Fact] public void LogOptionsFromEnvironment_AcceptsMaximumMbValue() { From 490be5367ce94ba3082c35c62335ceb00f6bd6ec Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 22:30:09 +0900 Subject: [PATCH 3/4] Harden database checkpoint restore diagnostics (#3833) --- DEVELOPER_GUIDE.md | 4 +- changelog.d/unreleased/3833.security.md | 19 + src/CodeIndex/Cli/ConsoleUi.cs | 2 +- src/CodeIndex/Cli/DbCommandRunner.cs | 487 ++++++++++++++++-- src/CodeIndex/Cli/JsonOutputContracts.cs | 38 +- tests/CodeIndex.Tests/DbCommandRunnerTests.cs | 270 +++++++++- 6 files changed, 778 insertions(+), 42 deletions(-) create mode 100644 changelog.d/unreleased/3833.security.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 8fe7d644bb..f91e2246a3 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -48,7 +48,7 @@ Development contracts: | Read-only database queries | `cdidx status --db /artifacts/codeindex.db --read-only --json`; `cdidx search AuthService --db /artifacts/codeindex.db --immutable` | Query commands accept `--read-only` (alias `--immutable`) to open an existing CodeIndex database through SQLite's immutable read-only URI mode. Use this for CI artifacts, mounted caches, and sandboxes where creating or updating `codeindex.db-wal` / `codeindex.db-shm` sidecars is not allowed. | | Mutating commands | `index`, `backfill-fold`, `optimize`, `vacuum` | These require writable storage and reject read-only database opens. | | Reusable index artifact | `cdidx export codeindex.cdidx.zip`; `cdidx import codeindex.cdidx.zip --db `; `cdidx import codeindex.cdidx.zip --dry-run --json` | Run export after indexing and upload the archive. Consumers import before query commands, or use `--dry-run` / `--check` to validate the archive without replacing the destination DB. Use `--prune-paths` when the archive comes from another checkout and the restored DB should advertise the import target project root; imports targeting `.../.cdidx/codeindex.db` use the sibling project directory, while other DB paths fall back to the process current directory. The archive contains `manifest.json` plus `codeindex.db`; the manifest carries bounded summary/readiness metadata including row counts, readiness bits, writer/indexed-head metadata, schema contract stamps, and unknown-extension summary when available. Import validates manifest format, manifest `user_version`, `database_sha256`, present summary counts, and the embedded SQLite file as a CodeIndex database before replacing the destination DB. Import rejects archive `codeindex.db` entries whose compressed or uncompressed metadata exceeds 8 GiB, and the extraction stream is also capped at 8 GiB. | -| Maintenance checkpoint | `cdidx db checkpoint `; `cdidx db restore ` | Checkpoint snapshots `codeindex.db` plus existing WAL/SHM sidecars before risky maintenance. Restore rolls back and keeps pre-restore files under `.restore-backup-/`. Checkpoints live next to the DB under `.checkpoints//`. `backfill-fold` creates an automatic checkpoint before it mutates rows unless `--no-checkpoint` is passed. | +| Maintenance checkpoint | `cdidx db checkpoint `; `cdidx db restore `; `cdidx db restore-backups --list|--prune --keep ` | Checkpoint snapshots `codeindex.db` plus existing WAL/SHM sidecars before risky maintenance. Checkpoint manifests record only the database file name, not the local absolute DB path. Restore rolls back and keeps pre-restore files under `.restore-backup-/`; use `restore-backups --list` to inspect retained backups and `restore-backups --prune --keep ` to apply retention. Checkpoints live next to the DB under `.checkpoints//`. `backfill-fold` creates an automatic checkpoint before it mutates rows unless `--no-checkpoint` is passed. | | Binary compatibility | [COMPATIBILITY.md](COMPATIBILITY.md) | Database compatibility across `cdidx` binary upgrades and downgrades is documented there. Keep that policy updated whenever readiness bits, `codeindex_meta` contract stamps, or rebuild requirements change. | | Fold backfill preview and recovery | `backfill-fold --dry-run`; MCP `backfill_fold` with `dry_run: true` or `force: true` | Dry-run previews folded-key rows without mutating the DB or stamping FoldReady. MCP accepts the same preview and can force rewriting all folded keys when an operator needs to recover from suspicious fold metadata or row state even though the stored version/fingerprint appears current. Non-dry-run row rewrites are resumable after interruption: completed row updates remain durable, and final FoldReady metadata is stamped only after verification succeeds. MCP responses include `progress.rows_done`, `progress.rows_total`, and `progress.fraction` so clients can report and retry long backfills. | @@ -2254,7 +2254,7 @@ net9 CI lane に合わせる場合は `FRAMEWORK=net9.0 make test` を使いま | read-only database query | `cdidx status --db /artifacts/codeindex.db --read-only --json`; `cdidx search AuthService --db /artifacts/codeindex.db --immutable` | query コマンドは `--read-only`（alias: `--immutable`）を受け付け、既存の CodeIndex database を SQLite の immutable read-only URI mode で開けます。CI artifact、mounted cache、`codeindex.db-wal` / `codeindex.db-shm` sidecar を作成・更新できない sandbox で使います。 | | 変更系コマンド | `index`、`backfill-fold`、`optimize`、`vacuum` | 書き込み可能な storage を必要とし、read-only database open を拒否します。 | | 再利用可能な index artifact | `cdidx export codeindex.cdidx.zip`; `cdidx import codeindex.cdidx.zip --db `; `cdidx import codeindex.cdidx.zip --dry-run --json` | CI job では index 後に export して archive を upload します。利用側は query コマンドの前に import でき、`--dry-run` / `--check` で destination DB を置き換えず archive を検証できます。別 checkout 由来の archive を import 先 project root として扱いたい場合は `--prune-paths` を使います。`.../.cdidx/codeindex.db` を import 先にした場合は sibling の project directory を使い、それ以外の DB path では process current directory に fallback します。archive は `manifest.json` と `codeindex.db` を含み、manifest は row count、readiness bit、writer / indexed-head metadata、schema contract stamp、利用可能な unknown-extension summary などの bounded summary/readiness metadata を持ちます。import は manifest format、manifest `user_version`、`database_sha256`、存在する summary count、embedded SQLite file が CodeIndex database であることを検証してから destination DB を置き換えます。archive の `codeindex.db` entry は compressed / uncompressed metadata と extraction stream の双方で 8 GiB を上限に拒否されます。 | -| maintenance checkpoint | `cdidx db checkpoint `; `cdidx db restore ` | 危険な maintenance の前に `codeindex.db` と既存 WAL/SHM sidecar の filesystem snapshot を作成し、restore で戻します。checkpoint は DB の隣の `.checkpoints//` に置かれ、restore は pre-restore file を `.restore-backup-/` に保持します。`backfill-fold` は `--no-checkpoint` を渡さない限り、row mutation 前に automatic checkpoint を作ります。 | +| maintenance checkpoint | `cdidx db checkpoint `; `cdidx db restore `; `cdidx db restore-backups --list|--prune --keep ` | 危険な maintenance の前に `codeindex.db` と既存 WAL/SHM sidecar の filesystem snapshot を作成し、restore で戻します。checkpoint manifest は database file name だけを記録し、local absolute DB path は記録しません。checkpoint は DB の隣の `.checkpoints//` に置かれ、restore は pre-restore file を `.restore-backup-/` に保持します。保持された backup は `restore-backups --list` で確認し、`restore-backups --prune --keep ` で retention を適用できます。`backfill-fold` は `--no-checkpoint` を渡さない限り、row mutation 前に automatic checkpoint を作ります。 | | binary compatibility | [COMPATIBILITY.md](COMPATIBILITY.md) | `cdidx` binary の upgrade / downgrade をまたぐ database compatibility を記載します。readiness bit、`codeindex_meta` contract stamp、rebuild requirement を変える場合は、この policy も更新してください。 | | Fold backfill の preview / recovery | `backfill-fold --dry-run`; MCP `backfill_fold` の `dry_run: true` または `force: true` | dry-run は DB を変更せず FoldReady stamp も書かずに、rewrite 対象の folded-key row をプレビューします。MCP も同じ preview を受け付け、stored version / fingerprint が current に見える場合でも suspicious な fold metadata や row state を復旧するため `force: true` を受け付けます。non-dry-run rewrite は中断後に resume でき、完了済み row update は durable に残り、最終 FoldReady metadata は verification 成功後にだけ stamp されます。MCP response は `progress.rows_done`、`progress.rows_total`、`progress.fraction` を含みます。 | diff --git a/changelog.d/unreleased/3833.security.md b/changelog.d/unreleased/3833.security.md new file mode 100644 index 0000000000..b7d104b02a --- /dev/null +++ b/changelog.d/unreleased/3833.security.md @@ -0,0 +1,19 @@ +--- +category: security +issues: + - 3833 +affected: + - src/CodeIndex/Cli/DbCommandRunner.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - tests/CodeIndex.Tests/DbCommandRunnerTests.cs + - DEVELOPER_GUIDE.md +--- + +## English + +- **Database checkpoint and restore diagnostics are now privacy-hardened (#3833)** — checkpoint manifests omit absolute DB paths, checkpoint creation reports recoverable file-list diagnostics without failing success responses, restore errors are sanitized, rollback failures are exposed in JSON, and retained restore backups can now be listed or pruned. + +## 日本語 + +- **database checkpoint / restore diagnostic の privacy hardening を行いました (#3833)** — checkpoint manifest は absolute DB path を記録せず、checkpoint 作成後の file list 失敗は成功レスポンスを落とさず recoverable diagnostic として返し、restore error は sanitized され、rollback failure は JSON に構造化され、保持された restore backup は list / prune できるようになりました。 diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index 706bb17aa9..ce985d01d8 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -106,7 +106,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = ("config", "cdidx config show [--json]"), ("validate-config", "cdidx validate-config"), ("doctor", "cdidx doctor"), - ("db", "cdidx db --integrity-check|schema|prune [--dry-run|--apply] [--db ] [--json] | cdidx db checkpoint [name<=128] [--db ] [--json] | cdidx db checkpoints --list [--db ] [--json] | cdidx db restore [--db ] [--json]"), + ("db", "cdidx db --integrity-check|schema|prune [--dry-run|--apply] [--db ] [--json] | cdidx db checkpoint [name<=128] [--db ] [--json] | cdidx db checkpoints --list [--db ] [--json] | cdidx db restore [--db ] [--json] | cdidx db restore-backups --list|--prune [--keep ] [--db ] [--json]"), ("diff", "cdidx diff [--json] [--summary-only] [--detailed] [--limit ]"), ("report", "cdidx report --output [--db ] [--json] [--log-lines ] [--no-log] [--include-args]"), ("validate", "cdidx validate [--db ] [--json[=array]] [--format ] [--verbose] [--limit |--top ] [--kind ] [--severity ] [--path ]"), diff --git a/src/CodeIndex/Cli/DbCommandRunner.cs b/src/CodeIndex/Cli/DbCommandRunner.cs index 72dda58412..16617181a0 100644 --- a/src/CodeIndex/Cli/DbCommandRunner.cs +++ b/src/CodeIndex/Cli/DbCommandRunner.cs @@ -1,5 +1,6 @@ using System.Text.Json; using CodeIndex.Database; +using CodeIndex.Diagnostics; using CodeIndex.Indexer; using Microsoft.Data.Sqlite; @@ -17,6 +18,10 @@ public static class DbCommandRunner private const int CheckpointNameDiagnosticTextLimit = 80; internal const int CheckpointListEntryLimit = 100; internal const int CheckpointFileInspectLimit = 32; + internal const int RestoreBackupListEntryLimit = 100; + internal const int RestoreBackupPruneScanLimit = 1_000; + internal const int DefaultRestoreBackupKeepCount = 10; + internal const int MaxRestoreBackupKeepCount = 1_000; internal const int IntegrityCheckRowLimit = 100; internal const int IntegrityCheckTextLimit = 4096; internal const int SchemaEntryLimit = 200; @@ -25,6 +30,7 @@ public static class DbCommandRunner internal static Action? RestoreFailureAfterBackupForTesting { get; set; } internal static Action? DeleteTemporaryDirectoryForTesting { get; set; } internal static Func>? IntegrityCheckRowsForTesting { get; set; } + internal static Func>? EnumerateCheckpointFileNamesForTesting { get; set; } public static int Run(string[] cmdArgs, JsonSerializerOptions jsonOptions) { @@ -44,13 +50,13 @@ public static int Run(string[] cmdArgs, JsonSerializerOptions jsonOptions) "Run `cdidx db --integrity-check --help` to see the supported command shape.", CommandErrorCodes.UsageError); - if (!options.IntegrityCheck && !options.Schema && !options.Prune && !options.Checkpoint && !options.ListCheckpoints && !options.Restore) + if (!options.IntegrityCheck && !options.Schema && !options.Prune && !options.Checkpoint && !options.ListCheckpoints && !options.Restore && !options.RestoreBackups) return WriteCommandError( options.Json, jsonOptions, "db requires a mode flag", CommandExitCodes.UsageError, - "Pass `--integrity-check`, `schema`, `prune --dry-run|--apply`, `checkpoint [name]`, `checkpoints --list`, or `restore `.", + "Pass `--integrity-check`, `schema`, `prune --dry-run|--apply`, `checkpoint [name]`, `checkpoints --list`, `restore `, or `restore-backups --list|--prune --keep `.", CommandErrorCodes.UsageError); if ((options.IntegrityCheck ? 1 : 0) @@ -58,13 +64,14 @@ public static int Run(string[] cmdArgs, JsonSerializerOptions jsonOptions) + (options.Prune ? 1 : 0) + (options.Checkpoint ? 1 : 0) + (options.ListCheckpoints ? 1 : 0) - + (options.Restore ? 1 : 0) > 1) + + (options.Restore ? 1 : 0) + + (options.RestoreBackups ? 1 : 0) > 1) return WriteCommandError( options.Json, jsonOptions, "db accepts exactly one mode", CommandExitCodes.UsageError, - "Run one of `cdidx db --integrity-check`, `cdidx db schema`, `cdidx db prune --dry-run|--apply`, `cdidx db checkpoint [name]`, `cdidx db checkpoints --list`, or `cdidx db restore `.", + "Run one of `cdidx db --integrity-check`, `cdidx db schema`, `cdidx db prune --dry-run|--apply`, `cdidx db checkpoint [name]`, `cdidx db checkpoints --list`, `cdidx db restore `, or `cdidx db restore-backups --list|--prune --keep `.", CommandErrorCodes.UsageError); var dbPath = options.DbPath; @@ -102,6 +109,9 @@ public static int Run(string[] cmdArgs, JsonSerializerOptions jsonOptions) if (options.Restore) return RunRestore(options, jsonOptions); + if (options.RestoreBackups) + return RunRestoreBackups(options, jsonOptions); + return RunIntegrityCheck(options, jsonOptions, dbPath, isUri); } @@ -330,7 +340,8 @@ private static int RunCheckpoint(DbCommandOptions options, JsonSerializerOptions result.CheckpointPath, result.Files, result.FilesTruncated, - CheckpointFileInspectLimit), + CheckpointFileInspectLimit, + result.Diagnostics), CliJsonSerializerContextFactory.Create(jsonOptions).DbCheckpointJsonResult)); } else @@ -340,19 +351,25 @@ private static int RunCheckpoint(DbCommandOptions options, JsonSerializerOptions Console.WriteLine($" name : {result.Name}"); Console.WriteLine($" checkpoint: {result.CheckpointPath}"); Console.WriteLine($" files : {ConsoleUi.Counted(result.Files.Count, "file")}{(result.FilesTruncated ? " (truncated)" : string.Empty)}"); + foreach (var diagnostic in result.Diagnostics) + Console.Error.WriteLine($"Warning [{diagnostic.Code}]: {diagnostic.Message}"); } return CommandExitCodes.Success; } catch (Exception ex) { + var safeMessage = ex is ArgumentException + ? ex.Message + : $"failed to create database checkpoint: {CommandErrorWriter.FormatSanitizedException(ex)}"; return WriteCommandError( options.Json, jsonOptions, - $"failed to create database checkpoint: {ex.Message}", + safeMessage, CommandExitCodes.DatabaseError, "Ensure the database and checkpoint directory are writable, then retry `cdidx db checkpoint`.", - CommandErrorCodes.DbError); + CommandErrorCodes.DbError, + category: ex is ArgumentException ? null : DiagnosticRedactor.ClassifyException(ex)); } } @@ -404,9 +421,10 @@ private static int RunRestore(DbCommandOptions options, JsonSerializerOptions js if (!ValidateWritableFileDb(options, jsonOptions, "restore", out var fullDbPath, out var validationExitCode)) return validationExitCode; + var checkpointPath = string.Empty; try { - var checkpointPath = GetCheckpointPath(fullDbPath, options.Name); + checkpointPath = GetCheckpointPath(fullDbPath, options.Name); if (!Directory.Exists(checkpointPath)) return WriteCommandError(options.Json, jsonOptions, $"checkpoint not found: {FormatCheckpointNameForDiagnostic(options.Name)}", CommandExitCodes.NotFound, "Run `cdidx db checkpoints --list` to see available checkpoints.", CommandErrorCodes.DbNotFound); @@ -427,16 +445,152 @@ private static int RunRestore(DbCommandOptions options, JsonSerializerOptions js return CommandExitCodes.Success; } + catch (DbRestoreOperationException ex) + { + return WriteRestoreError(options, jsonOptions, fullDbPath, options.Name, checkpointPath, ex); + } catch (Exception ex) { return WriteCommandError( options.Json, jsonOptions, - $"failed to restore database checkpoint: {ex.Message}", + $"failed to restore database checkpoint: {CommandErrorWriter.FormatSanitizedException(ex)}", CommandExitCodes.DatabaseError, "Ensure no cdidx writer is running, then retry `cdidx db restore `.", - CommandErrorCodes.DbError); + CommandErrorCodes.DbError, + category: DiagnosticRedactor.ClassifyException(ex)); + } + } + + private static int WriteRestoreError( + DbCommandOptions options, + JsonSerializerOptions jsonOptions, + string fullDbPath, + string name, + string checkpointPath, + DbRestoreOperationException ex) + { + var primary = ex.InnerException ?? ex; + var message = $"failed to restore database checkpoint: {CommandErrorWriter.FormatSanitizedException(primary)}"; + const string hint = "Ensure no cdidx writer is running, then retry `cdidx db restore `."; + if (options.Json) + { + Console.WriteLine(JsonSerializer.Serialize( + new DbRestoreJsonResult( + "error", + fullDbPath, + name, + string.IsNullOrWhiteSpace(ex.CheckpointPath) ? checkpointPath : ex.CheckpointPath, + ex.BackupPath, + message, + CommandErrorCodes.DbError, + hint, + ex.RollbackFailure is not null, + ex.RollbackFailure), + CliJsonSerializerContextFactory.Create(jsonOptions).DbRestoreJsonResult)); + return CommandExitCodes.DatabaseError; + } + + return WriteCommandError( + false, + jsonOptions, + message, + CommandExitCodes.DatabaseError, + hint, + CommandErrorCodes.DbError, + category: DiagnosticRedactor.ClassifyException(primary)); + } + + private static int RunRestoreBackups(DbCommandOptions options, JsonSerializerOptions jsonOptions) + { + if (!options.RestoreBackupsList && !options.RestoreBackupsPrune) + return WriteCommandError( + options.Json, + jsonOptions, + "restore-backups requires --list or --prune", + CommandExitCodes.UsageError, + "Use `cdidx db restore-backups --list` or `cdidx db restore-backups --prune --keep `.", + CommandErrorCodes.UsageError); + + if (options.RestoreBackupsList && options.RestoreBackupsPrune) + return WriteCommandError( + options.Json, + jsonOptions, + "restore-backups accepts only one of --list or --prune", + CommandExitCodes.UsageError, + "Choose `--list` or `--prune`.", + CommandErrorCodes.UsageError); + + if (!TryResolveFileDb(options.DbPath, out var fullDbPath, out var error)) + return WriteCommandError(options.Json, jsonOptions, error, CommandExitCodes.DatabaseError, "Use a filesystem database path, not a SQLite URI.", CommandErrorCodes.DbError); + + if (options.RestoreBackupsList) + { + var result = ListRestoreBackups(fullDbPath, RestoreBackupListEntryLimit); + if (options.Json) + { + Console.WriteLine(JsonSerializer.Serialize( + new DbRestoreBackupListJsonResult( + fullDbPath, + result.Entries, + result.Truncated, + RestoreBackupListEntryLimit, + CheckpointFileInspectLimit, + result.Diagnostics), + CliJsonSerializerContextFactory.Create(jsonOptions).DbRestoreBackupListJsonResult)); + } + else + { + Console.WriteLine("Database restore backups"); + Console.WriteLine($" database: {fullDbPath}"); + if (result.Truncated) + Console.WriteLine($" truncated: yes (restore backup limit {RestoreBackupListEntryLimit:N0}, file limit {CheckpointFileInspectLimit:N0} per backup)"); + if (result.Entries.Count == 0) + { + Console.WriteLine(" backups: none"); + } + else + { + foreach (var entry in result.Entries) + Console.WriteLine($" {entry.Name} {entry.CreatedAtUtc} {entry.Bytes:N0} bytes{(entry.FilesTruncated ? " (files truncated)" : string.Empty)}"); + } + + foreach (var diagnostic in result.Diagnostics) + Console.Error.WriteLine($"Warning [{diagnostic.Code}]: {diagnostic.Message}"); + } + + return CommandExitCodes.Success; + } + + var pruneResult = PruneRestoreBackups(fullDbPath, options.RestoreBackupsKeep); + if (options.Json) + { + Console.WriteLine(JsonSerializer.Serialize( + new DbRestoreBackupPruneJsonResult( + "success", + fullDbPath, + options.RestoreBackupsKeep, + pruneResult.Deleted, + pruneResult.Retained, + pruneResult.Truncated, + RestoreBackupPruneScanLimit, + pruneResult.Diagnostics), + CliJsonSerializerContextFactory.Create(jsonOptions).DbRestoreBackupPruneJsonResult)); + } + else + { + Console.WriteLine("Pruned database restore backups."); + Console.WriteLine($" database: {fullDbPath}"); + Console.WriteLine($" keep : {options.RestoreBackupsKeep:N0}"); + Console.WriteLine($" deleted : {pruneResult.Deleted:N0}"); + Console.WriteLine($" retained: {pruneResult.Retained:N0}"); + if (pruneResult.Truncated) + Console.WriteLine($" truncated: yes (restore backup scan limit {RestoreBackupPruneScanLimit:N0})"); + foreach (var diagnostic in pruneResult.Diagnostics) + Console.Error.WriteLine($"Warning [{diagnostic.Code}]: {diagnostic.Message}"); } + + return CommandExitCodes.Success; } // PRAGMA integrity_check returns a single row `"ok"` when the file passes every consistency @@ -757,7 +911,7 @@ private static DbCheckpointOperationResult CreateCheckpoint(string fullDbPath, s CopyIfExists(fullDbPath, Path.Combine(tempPath, Path.GetFileName(fullDbPath)), privateDestination: true); CopyIfExists(fullDbPath + "-wal", Path.Combine(tempPath, Path.GetFileName(fullDbPath) + "-wal"), privateDestination: true); CopyIfExists(fullDbPath + "-shm", Path.Combine(tempPath, Path.GetFileName(fullDbPath) + "-shm"), privateDestination: true); - DataDirectorySecurity.WritePrivateText(Path.Combine(tempPath, "manifest.txt"), $"name={name}{Environment.NewLine}created_at_utc={DateTimeOffset.UtcNow:O}{Environment.NewLine}db={fullDbPath}{Environment.NewLine}"); + DataDirectorySecurity.WritePrivateText(Path.Combine(tempPath, "manifest.txt"), $"name={name}{Environment.NewLine}created_at_utc={DateTimeOffset.UtcNow:O}{Environment.NewLine}db_file={Path.GetFileName(fullDbPath)}{Environment.NewLine}"); Directory.Move(tempPath, checkpointPath); } catch @@ -770,8 +924,9 @@ private static DbCheckpointOperationResult CreateCheckpoint(string fullDbPath, s throw; } - var files = EnumerateCheckpointFileNames(checkpointPath); - return new DbCheckpointOperationResult(name, checkpointPath, files.Items, files.Truncated); + var diagnostics = new List(); + var files = EnumerateCheckpointFileNames(checkpointPath, diagnostics); + return new DbCheckpointOperationResult(name, checkpointPath, files.Items, files.Truncated, diagnostics); } private static DbCheckpointListReadResult ListCheckpoints(string fullDbPath) @@ -824,10 +979,159 @@ private static DbCheckpointListReadResult ListCheckpoints(string fullDbPath) bytes.Truncated)); } - entries.Sort((left, right) => string.Compare(left.Name, right.Name, StringComparison.Ordinal)); + entries.Sort((left, right) => + { + var createdCompare = string.Compare(right.CreatedAtUtc, left.CreatedAtUtc, StringComparison.Ordinal); + return createdCompare != 0 + ? createdCompare + : string.Compare(left.Name, right.Name, StringComparison.Ordinal); + }); return new DbCheckpointListReadResult(entries, checkpointsTruncated || entries.Any(entry => entry.FilesTruncated), diagnostics); } + private static DbRestoreBackupReadResult ListRestoreBackups(string fullDbPath, int limit) + { + var parent = Path.GetDirectoryName(fullDbPath) ?? Path.GetPathRoot(fullDbPath) ?? Path.GetFullPath("."); + var diagnostics = new List(); + if (!Directory.Exists(parent)) + return new DbRestoreBackupReadResult([], DirectoryEnumerationTruncated: false, FileInspectionTruncated: false, diagnostics); + + var dbFileName = Path.GetFileName(fullDbPath); + var prefix = GetRestoreBackupDirectoryPrefix(fullDbPath); + var entries = new List(); + var backupsTruncated = false; + var directoriesInspected = 0; + var directories = EnumerateRestoreBackupDirectories(parent, prefix, diagnostics, limit + 1); + backupsTruncated |= directories.Truncated; + foreach (var path in directories.Items) + { + if (directoriesInspected >= limit) + { + backupsTruncated = true; + break; + } + + directoriesInspected++; + if (!File.Exists(LongPath.EnsureWindowsPrefix(Path.Combine(path, dbFileName)))) + continue; + + DirectoryInfo info; + DateTime createdAtUtc; + try + { + info = new DirectoryInfo(path); + createdAtUtc = info.CreationTimeUtc; + } + catch (Exception ex) when (IsRecoverableFilesystemException(ex)) + { + diagnostics.Add(CreateCheckpointDiagnostic("restore_backup_directory_stat_failed", "Unable to inspect restore backup directory metadata.", path)); + backupsTruncated = true; + continue; + } + + var bytes = SumCheckpointBytes(path, diagnostics); + entries.Add(new DbRestoreBackupEntryJsonResult( + info.Name, + path, + createdAtUtc.ToString("O", System.Globalization.CultureInfo.InvariantCulture), + bytes.Bytes, + bytes.Truncated)); + } + + entries.Sort((left, right) => + { + var createdCompare = string.Compare(right.CreatedAtUtc, left.CreatedAtUtc, StringComparison.Ordinal); + return createdCompare != 0 + ? createdCompare + : string.Compare(right.Name, left.Name, StringComparison.Ordinal); + }); + return new DbRestoreBackupReadResult(entries, backupsTruncated, entries.Any(entry => entry.FilesTruncated), diagnostics); + } + + private static DbRestoreBackupPruneResult PruneRestoreBackups(string fullDbPath, int keep) + { + var result = ListRestoreBackups(fullDbPath, RestoreBackupPruneScanLimit); + var diagnostics = result.Diagnostics; + if (result.DirectoryEnumerationTruncated) + { + diagnostics.Add(new DbDiagnosticJsonResult( + "restore_backup_prune_truncated", + "Restore backup pruning was skipped because backup enumeration reached the scan limit.", + ConsoleUi.FormatBoundedValue(fullDbPath))); + return new DbRestoreBackupPruneResult(Deleted: 0, Retained: result.Entries.Count, Truncated: true, diagnostics); + } + + var deleted = 0; + foreach (var entry in result.Entries.Skip(keep)) + { + if (TryDeleteRestoreBackupDirectory(fullDbPath, entry.BackupPath, diagnostics)) + deleted++; + } + + var retained = result.Entries.Count - deleted; + return new DbRestoreBackupPruneResult(deleted, retained, result.Truncated, diagnostics); + } + + private static (List Items, bool Truncated) EnumerateRestoreBackupDirectories( + string parent, + string prefix, + List diagnostics, + int limit) + { + var directories = new List(); + try + { + foreach (var directory in Directory.EnumerateDirectories(parent, prefix + "*")) + { + if (directories.Count >= limit) + return (directories, Truncated: true); + if (Path.GetFileName(directory).StartsWith(prefix, StringComparison.Ordinal)) + directories.Add(directory); + } + + return (directories, Truncated: false); + } + catch (Exception ex) when (IsRecoverableFilesystemException(ex)) + { + diagnostics.Add(CreateCheckpointDiagnostic("restore_backup_directory_enumeration_failed", "Unable to enumerate every restore backup directory.", parent)); + return (directories, Truncated: true); + } + } + + private static bool TryDeleteRestoreBackupDirectory( + string fullDbPath, + string backupPath, + List diagnostics) + { + var parent = Path.GetDirectoryName(fullDbPath) ?? Path.GetPathRoot(fullDbPath) ?? Path.GetFullPath("."); + var prefix = GetRestoreBackupDirectoryPrefix(fullDbPath); + if (!TryValidateTemporaryDirectoryCleanupTarget(backupPath, parent, prefix, out var fullPath, out var validationFailure)) + { + diagnostics.Add(new DbDiagnosticJsonResult( + "restore_backup_delete_skipped", + $"Skipped deleting restore backup directory: {validationFailure}.", + ConsoleUi.FormatBoundedValue(backupPath))); + return false; + } + + try + { + if (!Directory.Exists(LongPath.EnsureWindowsPrefix(fullPath))) + return false; + + Directory.Delete(LongPath.EnsureWindowsPrefix(fullPath), recursive: true); + return true; + } + catch (Exception ex) when (IsRecoverableFilesystemException(ex)) + { + diagnostics.Add(new DbDiagnosticJsonResult( + "restore_backup_delete_failed", + $"Unable to delete restore backup directory ({CommandErrorWriter.FormatSanitizedException(ex)}).", + ConsoleUi.FormatBoundedValue(fullPath))); + return false; + } + } + private static (List Items, bool Truncated) EnumerateCheckpointDirectories( string root, List diagnostics, @@ -852,21 +1156,32 @@ private static (List Items, bool Truncated) EnumerateCheckpointDirectori } } - private static (List Items, bool Truncated) EnumerateCheckpointFileNames(string checkpointPath) + private static (List Items, bool Truncated) EnumerateCheckpointFileNames( + string checkpointPath, + List diagnostics) { var files = new List(); var truncated = false; - foreach (var file in Directory.EnumerateFiles(checkpointPath)) + try { - if (files.Count >= CheckpointFileInspectLimit) + IEnumerable fileNames = EnumerateCheckpointFileNamesForTesting?.Invoke(checkpointPath) + ?? Directory.EnumerateFiles(checkpointPath).Select(Path.GetFileName); + foreach (var name in fileNames) { - truncated = true; - break; - } + if (files.Count >= CheckpointFileInspectLimit) + { + truncated = true; + break; + } - var name = Path.GetFileName(file); - if (name is not null) - files.Add(name); + if (name is not null) + files.Add(name); + } + } + catch (Exception ex) when (IsRecoverableFilesystemException(ex)) + { + diagnostics.Add(CreateCheckpointDiagnostic("checkpoint_file_enumeration_failed", "Unable to enumerate every checkpoint file after creation.", checkpointPath)); + truncated = true; } files.Sort(StringComparer.Ordinal); @@ -954,18 +1269,23 @@ private static string RestoreCheckpoint(string fullDbPath, string name, string c MoveIfExists(Path.Combine(restoreTempPath, Path.GetFileName(fullDbPath) + "-wal"), fullDbPath + "-wal", privateDestination: true); MoveIfExists(Path.Combine(restoreTempPath, Path.GetFileName(fullDbPath) + "-shm"), fullDbPath + "-shm", privateDestination: true); } - catch + catch (Exception primaryEx) { + DbDiagnosticJsonResult? rollbackFailure = null; try { RestoreBackedUpFiles(fullDbPath, backupPath); } catch (Exception rollbackEx) when (IsRecoverableRestoreException(rollbackEx)) { - Console.Error.WriteLine($"Warning: failed to roll back database restore from backup {ConsoleUi.FormatBoundedValue(backupPath)} ({CommandErrorWriter.FormatSanitizedException(rollbackEx)})."); + rollbackFailure = new DbDiagnosticJsonResult( + "restore_rollback_failed", + $"Failed to roll back database restore from backup ({CommandErrorWriter.FormatSanitizedException(rollbackEx)}).", + ConsoleUi.FormatBoundedValue(backupPath)); + Console.Error.WriteLine($"Warning [{rollbackFailure.Code}]: {rollbackFailure.Message}"); } - throw; + throw new DbRestoreOperationException(primaryEx, checkpointPath, backupPath, rollbackFailure); } finally { @@ -1006,6 +1326,9 @@ private static string MakeRestorePathSuffix() private static string GetCheckpointRoot(string fullDbPath) => fullDbPath + CheckpointsDirectorySuffix; + private static string GetRestoreBackupDirectoryPrefix(string fullDbPath) + => Path.GetFileName(fullDbPath) + ".restore-backup-"; + private static string GetCheckpointPath(string fullDbPath, string name) { ValidateCheckpointName(name); @@ -1017,9 +1340,30 @@ private static void CopyIfExists(string source, string destination, bool private if (!TryGetRegularExistingFile(source, out var normalizedSource)) return; - File.Copy(normalizedSource, LongPath.EnsureWindowsPrefix(destination), overwrite: false); - if (privateDestination) - DataDirectorySecurity.ApplyPrivateFileMode(destination); + if (!privateDestination || OperatingSystem.IsWindows()) + { + File.Copy(normalizedSource, LongPath.EnsureWindowsPrefix(destination), overwrite: false); + if (privateDestination) + DataDirectorySecurity.ApplyPrivateFileMode(destination); + return; + } + + using (var input = new FileStream(normalizedSource, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (var output = new FileStream( + LongPath.EnsureWindowsPrefix(destination), + new FileStreamOptions + { + Mode = FileMode.CreateNew, + Access = FileAccess.Write, + Share = FileShare.None, + UnixCreateMode = DataDirectorySecurity.PrivateFileMode, + })) + { + input.CopyTo(output); + output.Flush(flushToDisk: true); + } + + DataDirectorySecurity.ApplyPrivateFileMode(destination); } private static void MoveIfExists(string source, string destination, bool privateDestination = false) @@ -1144,6 +1488,10 @@ internal static DbCommandOptions ParseArgs(string[] args) var checkpoint = false; var listCheckpoints = false; var restore = false; + var restoreBackups = false; + var restoreBackupsList = false; + var restoreBackupsPrune = false; + var restoreBackupsKeep = DefaultRestoreBackupKeepCount; string? name = null; string? parseError = null; @@ -1184,15 +1532,47 @@ internal static DbCommandOptions ParseArgs(string[] args) else parseError = "restore requires a checkpoint name"; break; + case "restore-backups": + restoreBackups = true; + break; case "--dry-run": pruneDryRun = true; break; case "--apply": pruneApply = true; break; + case "--prune": + if (restoreBackups) + restoreBackupsPrune = true; + else + parseError = "--prune is only valid with `cdidx db restore-backups --prune`"; + break; + case "--keep" when i + 1 < args.Length: + if (!restoreBackups) + { + parseError = "--keep is only valid with `cdidx db restore-backups --prune --keep `"; + break; + } + + if (!int.TryParse(args[++i], System.Globalization.NumberStyles.None, System.Globalization.CultureInfo.InvariantCulture, out restoreBackupsKeep) + || restoreBackupsKeep < 0 + || restoreBackupsKeep > MaxRestoreBackupKeepCount) + { + parseError = $"--keep must be an integer from 0 to {MaxRestoreBackupKeepCount}"; + } + break; + case "--keep": + parseError = "--keep requires a value"; + break; case "--list": if (listCheckpoints) break; + if (restoreBackups) + { + restoreBackupsList = true; + break; + } + parseError = "--list is only valid with `cdidx db checkpoints --list`"; break; case "--help" or "-h": @@ -1209,6 +1589,9 @@ internal static DbCommandOptions ParseArgs(string[] args) break; } + if (parseError is null && restoreBackups && (pruneDryRun || pruneApply)) + parseError = "--dry-run and --apply are not supported with `cdidx db restore-backups`; use `--prune --keep ` to delete retained backups."; + return new DbCommandOptions { DbPath = dbPath, @@ -1221,14 +1604,18 @@ internal static DbCommandOptions ParseArgs(string[] args) Checkpoint = checkpoint, ListCheckpoints = listCheckpoints, Restore = restore, + RestoreBackups = restoreBackups, + RestoreBackupsList = restoreBackupsList, + RestoreBackupsPrune = restoreBackupsPrune, + RestoreBackupsKeep = restoreBackupsKeep, Name = name, ParseError = parseError, }; } - private static int WriteCommandError(bool json, JsonSerializerOptions jsonOptions, string message, int exitCode, string? hint = null, string? errorCode = null) + private static int WriteCommandError(bool json, JsonSerializerOptions jsonOptions, string message, int exitCode, string? hint = null, string? errorCode = null, string? category = null) { - return CommandErrorWriter.WriteJsonOrHuman(json, jsonOptions, message, exitCode, hint, errorCode: errorCode); + return CommandErrorWriter.WriteJsonOrHuman(json, jsonOptions, message, exitCode, hint, errorCode: errorCode, category: category); } } @@ -1245,14 +1632,50 @@ internal sealed class DbCommandOptions public bool Checkpoint { get; init; } public bool ListCheckpoints { get; init; } public bool Restore { get; init; } + public bool RestoreBackups { get; init; } + public bool RestoreBackupsList { get; init; } + public bool RestoreBackupsPrune { get; init; } + public int RestoreBackupsKeep { get; init; } = DbCommandRunner.DefaultRestoreBackupKeepCount; public string? Name { get; init; } public string? ParseError { get; init; } } -internal sealed record DbCheckpointOperationResult(string Name, string CheckpointPath, List Files, bool FilesTruncated); +internal sealed record DbCheckpointOperationResult(string Name, string CheckpointPath, List Files, bool FilesTruncated, List Diagnostics); internal sealed record DbCheckpointListReadResult(List Entries, bool Truncated, List Diagnostics); +internal sealed record DbRestoreBackupReadResult( + List Entries, + bool DirectoryEnumerationTruncated, + bool FileInspectionTruncated, + List Diagnostics) +{ + public bool Truncated => DirectoryEnumerationTruncated || FileInspectionTruncated; +} + +internal sealed record DbRestoreBackupPruneResult(int Deleted, int Retained, bool Truncated, List Diagnostics); + +internal sealed class DbRestoreOperationException : Exception +{ + public DbRestoreOperationException( + Exception innerException, + string checkpointPath, + string backupPath, + DbDiagnosticJsonResult? rollbackFailure) + : base("database restore failed", innerException) + { + CheckpointPath = checkpointPath; + BackupPath = backupPath; + RollbackFailure = rollbackFailure; + } + + public string CheckpointPath { get; } + + public string BackupPath { get; } + + public DbDiagnosticJsonResult? RollbackFailure { get; } +} + internal sealed record DbIntegrityCheckReadResult(List Rows, bool RowsTruncated, bool TextTruncated) { public bool Truncated => RowsTruncated || TextTruncated; diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 8c22b64fd7..261f01c0e6 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -84,7 +84,8 @@ internal sealed record DbCheckpointJsonResult( [property: JsonPropertyName("checkpoint_path")] string CheckpointPath, [property: JsonPropertyName("files")] List Files, [property: JsonPropertyName("files_truncated")] bool FilesTruncated = false, - [property: JsonPropertyName("file_limit")] int FileLimit = 0); + [property: JsonPropertyName("file_limit")] int FileLimit = 0, + [property: JsonPropertyName("diagnostics")] List? Diagnostics = null); internal sealed record DbCheckpointListJsonResult( [property: JsonPropertyName("db_path")] string DbPath, @@ -106,7 +107,37 @@ internal sealed record DbRestoreJsonResult( [property: JsonPropertyName("db_path")] string DbPath, [property: JsonPropertyName("name")] string Name, [property: JsonPropertyName("checkpoint_path")] string CheckpointPath, - [property: JsonPropertyName("backup_path")] string BackupPath); + [property: JsonPropertyName("backup_path")] string BackupPath, + [property: JsonPropertyName("message")] string? Message = null, + [property: JsonPropertyName("error_code")] string? ErrorCode = null, + [property: JsonPropertyName("hint")] string? Hint = null, + [property: JsonPropertyName("rollback_failed")] bool RollbackFailed = false, + [property: JsonPropertyName("rollback_failure")] DbDiagnosticJsonResult? RollbackFailure = null); + +internal sealed record DbRestoreBackupEntryJsonResult( + [property: JsonPropertyName("name")] string Name, + [property: JsonPropertyName("backup_path")] string BackupPath, + [property: JsonPropertyName("created_at_utc")] string CreatedAtUtc, + [property: JsonPropertyName("bytes")] long Bytes, + [property: JsonPropertyName("files_truncated")] bool FilesTruncated = false); + +internal sealed record DbRestoreBackupListJsonResult( + [property: JsonPropertyName("db_path")] string DbPath, + [property: JsonPropertyName("backups")] List Backups, + [property: JsonPropertyName("truncated")] bool Truncated = false, + [property: JsonPropertyName("backup_limit")] int BackupLimit = 0, + [property: JsonPropertyName("file_limit")] int FileLimit = 0, + [property: JsonPropertyName("diagnostics")] List? Diagnostics = null); + +internal sealed record DbRestoreBackupPruneJsonResult( + [property: JsonPropertyName("status")] string Status, + [property: JsonPropertyName("db_path")] string DbPath, + [property: JsonPropertyName("keep")] int Keep, + [property: JsonPropertyName("deleted")] int Deleted, + [property: JsonPropertyName("retained")] int Retained, + [property: JsonPropertyName("truncated")] bool Truncated = false, + [property: JsonPropertyName("backup_limit")] int BackupLimit = 0, + [property: JsonPropertyName("diagnostics")] List? Diagnostics = null); internal sealed record DbSchemaEntryJsonResult( [property: JsonPropertyName("type")] string Type, @@ -492,6 +523,9 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(DbDiagnosticJsonResult))] [JsonSerializable(typeof(DbIntegrityCheckJsonResult))] [JsonSerializable(typeof(DbPruneJsonResult))] +[JsonSerializable(typeof(DbRestoreBackupEntryJsonResult))] +[JsonSerializable(typeof(DbRestoreBackupListJsonResult))] +[JsonSerializable(typeof(DbRestoreBackupPruneJsonResult))] [JsonSerializable(typeof(DbRestoreJsonResult))] [JsonSerializable(typeof(DbSchemaEntryJsonResult))] [JsonSerializable(typeof(DbSchemaJsonResult))] diff --git a/tests/CodeIndex.Tests/DbCommandRunnerTests.cs b/tests/CodeIndex.Tests/DbCommandRunnerTests.cs index a08b7904e6..be71feabf1 100644 --- a/tests/CodeIndex.Tests/DbCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/DbCommandRunnerTests.cs @@ -97,6 +97,17 @@ public void ParseArgs_RestoreRequiresName() Assert.Contains("requires", options.ParseError); } + [Fact] + public void ParseArgs_RestoreBackupsPruneSetsKeep_Issue3833() + { + var options = DbCommandRunner.ParseArgs(["restore-backups", "--prune", "--keep", "3"]); + + Assert.True(options.RestoreBackups); + Assert.True(options.RestoreBackupsPrune); + Assert.Equal(3, options.RestoreBackupsKeep); + Assert.Null(options.ParseError); + } + [Fact] public void Run_WithoutModeFlag_ReturnsUsageError() { @@ -530,6 +541,60 @@ public void Run_Checkpoint_OnPosix_WritesPrivateSnapshotPermissions() } } + [Fact] + public void Run_CheckpointManifestOmitsAbsoluteDbPath_Issue3833() + { + var root = Path.Combine(Path.GetTempPath(), $"cdidx_db_checkpoint_manifest_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(root, "codeindex.db"); + try + { + Directory.CreateDirectory(root); + File.WriteAllText(dbPath, "db"); + + var (checkpointExit, _, _) = RunAndCaptureStreams(["checkpoint", "manifest", "--db", dbPath]); + + Assert.Equal(CommandExitCodes.Success, checkpointExit); + var manifest = File.ReadAllText(Path.Combine(dbPath + ".checkpoints", "manifest", "manifest.txt")); + Assert.Contains("db_file=codeindex.db", manifest); + Assert.DoesNotContain(dbPath, manifest); + Assert.DoesNotContain(root, manifest); + } + finally + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + + [Fact] + public void Run_CheckpointJsonReportsRecoverableFileEnumerationFailure_Issue3833() + { + var root = Path.Combine(Path.GetTempPath(), $"cdidx_db_checkpoint_enum_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(root, "codeindex.db"); + try + { + Directory.CreateDirectory(root); + File.WriteAllText(dbPath, "db"); + DbCommandRunner.EnumerateCheckpointFileNamesForTesting = _ => throw new IOException("secret local enumeration path"); + + var (checkpointExit, json) = RunAndCaptureJson(["checkpoint", "enum", "--db", dbPath, "--json"]); + + Assert.Equal(CommandExitCodes.Success, checkpointExit); + Assert.True(json.GetProperty("files_truncated").GetBoolean()); + var diagnostic = Assert.Single(json.GetProperty("diagnostics").EnumerateArray()); + Assert.Equal("checkpoint_file_enumeration_failed", diagnostic.GetProperty("code").GetString()); + Assert.DoesNotContain("secret local enumeration path", json.ToString()); + } + finally + { + DbCommandRunner.EnumerateCheckpointFileNamesForTesting = null; + SqliteConnection.ClearAllPools(); + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + [Fact] public void Run_CheckpointTempCleanupFailurePreservesOriginalFailure_Issue3029() { @@ -695,7 +760,8 @@ public void Run_RestoreIncompleteCheckpoint_ReturnsErrorAndKeepsDatabase() var (restoreExit, _, stderr) = RunAndCaptureStreams(["restore", "bad", "--db", dbPath]); Assert.Equal(CommandExitCodes.DatabaseError, restoreExit); - Assert.Contains("incomplete", stderr); + Assert.Contains("InvalidOperationException", stderr); + Assert.DoesNotContain("checkpoint is incomplete", stderr); Assert.Equal(originalBytes, File.ReadAllBytes(dbPath)); Assert.Empty(Directory.GetDirectories(root, "codeindex.db.restore-backup-*")); } @@ -729,7 +795,8 @@ public void Run_RestoreFailureAfterBackup_RestoresOriginalDatabase() var (restoreExit, _, stderr) = RunAndCaptureStreams(["restore", "saved", "--db", dbPath]); Assert.Equal(CommandExitCodes.DatabaseError, restoreExit); - Assert.Contains("injected restore failure", stderr); + Assert.Contains("IOException", stderr); + Assert.DoesNotContain("injected restore failure", stderr); Assert.Equal("changed", File.ReadAllText(dbPath)); Assert.Single(Directory.GetDirectories(root, "codeindex.db.restore-backup-*")); Assert.Empty(Directory.GetDirectories(root, "codeindex.db.restore-tmp-*")); @@ -768,8 +835,52 @@ public void Run_RestoreRollbackFailurePreservesPrimaryFailure_Issue3514() var (restoreExit, _, stderr) = RunAndCaptureStreams(["restore", "saved", "--db", dbPath]); Assert.Equal(CommandExitCodes.DatabaseError, restoreExit); - Assert.Contains("primary restore failure", stderr); - Assert.Contains("failed to roll back database restore", stderr); + Assert.Contains("IOException", stderr); + Assert.Contains("restore_rollback_failed", stderr); + Assert.DoesNotContain("primary restore failure", stderr); + } + finally + { + DbCommandRunner.RestoreFailureAfterBackupForTesting = null; + SqliteConnection.ClearAllPools(); + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + + [Fact] + public void Run_RestoreRollbackFailureJsonIncludesStructuredMetadata_Issue3833() + { + var root = Path.Combine(Path.GetTempPath(), $"cdidx_db_checkpoint_rollback_json_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(root, "codeindex.db"); + Directory.CreateDirectory(root); + try + { + using (var db = new DbContext(dbPath)) + db.InitializeSchema(); + SqliteConnection.ClearAllPools(); + + var (checkpointExit, _, _) = RunAndCaptureStreams(["checkpoint", "saved", "--db", dbPath]); + Assert.Equal(CommandExitCodes.Success, checkpointExit); + + File.WriteAllText(dbPath, "changed"); + DbCommandRunner.RestoreFailureAfterBackupForTesting = () => + { + Directory.CreateDirectory(dbPath); + throw new IOException("primary restore failure token=secret"); + }; + + var (restoreExit, stdout, _) = RunAndCaptureStreams(["restore", "saved", "--db", dbPath, "--json"]); + using var document = JsonDocument.Parse(stdout); + var json = document.RootElement; + + Assert.Equal(CommandExitCodes.DatabaseError, restoreExit); + Assert.Equal("error", json.GetProperty("status").GetString()); + Assert.True(json.GetProperty("rollback_failed").GetBoolean()); + Assert.Equal("restore_rollback_failed", json.GetProperty("rollback_failure").GetProperty("code").GetString()); + Assert.Contains("IOException", json.GetProperty("message").GetString()); + Assert.DoesNotContain("primary restore failure", stdout); + Assert.DoesNotContain("token=secret", stdout); } finally { @@ -780,6 +891,153 @@ public void Run_RestoreRollbackFailurePreservesPrimaryFailure_Issue3514() } } + [Fact] + public void Run_RestoreBackupsListAndPruneOrdersByRecency_Issue3833() + { + var root = Path.Combine(Path.GetTempPath(), $"cdidx_db_restore_backups_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(root, "codeindex.db"); + Directory.CreateDirectory(root); + try + { + File.WriteAllText(dbPath, "current"); + var older = Path.Combine(root, "codeindex.db.restore-backup-20260101000000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + var newer = Path.Combine(root, "codeindex.db.restore-backup-20260102000000000-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + Directory.CreateDirectory(older); + Directory.CreateDirectory(newer); + File.WriteAllText(Path.Combine(older, "codeindex.db"), "older"); + File.WriteAllText(Path.Combine(newer, "codeindex.db"), "newer"); + var sameCreationTime = new DateTime(2026, 1, 3, 0, 0, 0, DateTimeKind.Utc); + Directory.SetCreationTimeUtc(older, sameCreationTime); + Directory.SetCreationTimeUtc(newer, sameCreationTime); + + var (listExit, listJson) = RunAndCaptureJson(["restore-backups", "--list", "--db", dbPath, "--json"]); + + Assert.Equal(CommandExitCodes.Success, listExit); + var backups = listJson.GetProperty("backups"); + Assert.Equal(2, backups.GetArrayLength()); + Assert.Equal(Path.GetFileName(newer), backups[0].GetProperty("name").GetString()); + + var (pruneExit, pruneJson) = RunAndCaptureJson(["restore-backups", "--prune", "--keep", "1", "--db", dbPath, "--json"]); + + Assert.Equal(CommandExitCodes.Success, pruneExit); + Assert.Equal(1, pruneJson.GetProperty("deleted").GetInt32()); + Assert.Equal(1, pruneJson.GetProperty("retained").GetInt32()); + Assert.False(Directory.Exists(older)); + Assert.True(Directory.Exists(newer)); + } + finally + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + + [Fact] + public void Run_RestoreBackupsRejectsDryRunWithoutDeleting_Issue3833() + { + var root = Path.Combine(Path.GetTempPath(), $"cdidx_db_restore_backups_dry_run_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(root, "codeindex.db"); + Directory.CreateDirectory(root); + try + { + File.WriteAllText(dbPath, "current"); + var backup = Path.Combine(root, "codeindex.db.restore-backup-20260101000000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + Directory.CreateDirectory(backup); + File.WriteAllText(Path.Combine(backup, "codeindex.db"), "backup"); + + var (exitCode, _, stderr) = RunAndCaptureStreams(["restore-backups", "--prune", "--dry-run", "--keep", "0", "--db", dbPath]); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Contains("--dry-run and --apply are not supported", stderr); + Assert.True(Directory.Exists(backup)); + } + finally + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + + [Fact] + public void Run_RestoreBackupsPruneSkipsDeletionWhenScanTruncated_Issue3833() + { + var root = Path.Combine(Path.GetTempPath(), $"cdidx_db_restore_backups_truncated_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(root, "codeindex.db"); + Directory.CreateDirectory(root); + try + { + File.WriteAllText(dbPath, "current"); + for (var i = 0; i < DbCommandRunner.RestoreBackupPruneScanLimit + 1; i++) + { + var backup = Path.Combine(root, $"codeindex.db.restore-backup-20260101000000000-{i:x32}"); + Directory.CreateDirectory(backup); + File.WriteAllText(Path.Combine(backup, "codeindex.db"), "backup"); + } + + var (exitCode, json) = RunAndCaptureJson(["restore-backups", "--prune", "--keep", "0", "--db", dbPath, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.True(json.GetProperty("truncated").GetBoolean()); + Assert.Equal(0, json.GetProperty("deleted").GetInt32()); + Assert.Equal(DbCommandRunner.RestoreBackupPruneScanLimit, json.GetProperty("retained").GetInt32()); + Assert.Contains( + json.GetProperty("diagnostics").EnumerateArray(), + diagnostic => diagnostic.GetProperty("code").GetString() == "restore_backup_prune_truncated"); + Assert.Equal( + DbCommandRunner.RestoreBackupPruneScanLimit + 1, + Directory.GetDirectories(root, "codeindex.db.restore-backup-*").Length); + } + finally + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + + [Fact] + public void Run_RestoreBackupsPruneDeletesWhenOnlyFileInspectionTruncated_Issue3833() + { + var root = Path.Combine(Path.GetTempPath(), $"cdidx_db_restore_backups_file_truncated_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(root, "codeindex.db"); + Directory.CreateDirectory(root); + try + { + File.WriteAllText(dbPath, "current"); + var older = Path.Combine(root, "codeindex.db.restore-backup-20260101000000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + var newer = Path.Combine(root, "codeindex.db.restore-backup-20260102000000000-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + Directory.CreateDirectory(older); + Directory.CreateDirectory(newer); + File.WriteAllText(Path.Combine(older, "codeindex.db"), "older"); + File.WriteAllText(Path.Combine(newer, "codeindex.db"), "newer"); + for (var i = 0; i < DbCommandRunner.CheckpointFileInspectLimit + 1; i++) + File.WriteAllText(Path.Combine(newer, $"extra-{i:D4}.txt"), "x"); + + var (exitCode, json) = RunAndCaptureJson(["restore-backups", "--prune", "--keep", "1", "--db", dbPath, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.True(json.GetProperty("truncated").GetBoolean()); + Assert.Equal(1, json.GetProperty("deleted").GetInt32()); + Assert.Equal(1, json.GetProperty("retained").GetInt32()); + Assert.False(Directory.Exists(older)); + Assert.True(Directory.Exists(newer)); + if (json.TryGetProperty("diagnostics", out var diagnostics)) + { + Assert.DoesNotContain( + diagnostics.EnumerateArray(), + diagnostic => diagnostic.GetProperty("code").GetString() == "restore_backup_prune_truncated"); + } + } + finally + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + [Fact] public void Run_RestoreRejectsSymlinkedCheckpointPayload_Issue3514() { @@ -807,7 +1065,9 @@ public void Run_RestoreRejectsSymlinkedCheckpointPayload_Issue3514() var (restoreExit, _, stderr) = RunAndCaptureStreams(["restore", "saved", "--db", dbPath]); Assert.Equal(CommandExitCodes.DatabaseError, restoreExit); - Assert.Contains("not a regular file", stderr); + Assert.Contains("InvalidOperationException", stderr); + Assert.DoesNotContain("not a regular file", stderr); + Assert.DoesNotContain(checkpointDbPath, stderr); Assert.Equal(originalBytes, File.ReadAllBytes(dbPath)); } finally From c449d3640010ca9f35bc534f9cb6bb46b6ba1cbe Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 23:58:58 +0900 Subject: [PATCH 4/4] Preserve restore rollback backup path in warning (#3833) --- src/CodeIndex/Cli/DbCommandRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CodeIndex/Cli/DbCommandRunner.cs b/src/CodeIndex/Cli/DbCommandRunner.cs index d20045d26c..31e8b4ae32 100644 --- a/src/CodeIndex/Cli/DbCommandRunner.cs +++ b/src/CodeIndex/Cli/DbCommandRunner.cs @@ -1282,7 +1282,7 @@ private static string RestoreCheckpoint(string fullDbPath, string name, string c "restore_rollback_failed", $"Failed to roll back database restore from backup ({CommandErrorWriter.FormatSanitizedException(rollbackEx)}).", ConsoleUi.FormatBoundedValue(backupPath)); - CommandErrorWriter.WriteStderr($"Warning [{rollbackFailure.Code}]: {rollbackFailure.Message}"); + CommandErrorWriter.WriteStderr($"Warning [{rollbackFailure.Code}]: {rollbackFailure.Message} Backup: {rollbackFailure.Path}"); } throw new DbRestoreOperationException(primaryEx, checkpointPath, backupPath, rollbackFailure);