From 624530018355c3cdc0355d8f35eb6491da5ea443 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Wed, 29 Jul 2026 13:19:10 +0900 Subject: [PATCH 1/3] Fix parallel batch session reuse (#4872) --- DEVELOPER_GUIDE.md | 14 +- TESTING_GUIDE.md | 4 + changelog.d/unreleased/4872.fixed.md | 18 ++ src/CodeIndex/Cli/QueryCommandRunner.Batch.cs | 174 +++++++++++++++-- .../QueryCommandRunnerBatchIssue4872Tests.cs | 183 ++++++++++++++++++ 5 files changed, 365 insertions(+), 28 deletions(-) create mode 100644 changelog.d/unreleased/4872.fixed.md create mode 100644 tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 9c347a6ce..aa3c99137 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -412,7 +412,7 @@ Interactive terminal controls are allowed only when stdout is not redirected or Query commands that accept path filters (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `map`, `inspect`, `deps`, `impact`, `unused`, `hotspots`, and `validate`) expand `--project` into the matching project directory glob before hitting `DbReader`, so all existing SQL path predicates keep working. When the indexed project root cannot be resolved and project expansion falls back to the process current directory, CLI query context and MCP structured payloads include `project_filter_root` and `project_filter_root_fallback_reason`. `index --project` expands to the files under the selected project directory and reuses the existing `--files` update path, but rejects expansions above 65,536 files for one project or 131,072 unique files across all requested projects with an explicit-files recovery hint. -`cdidx batch` is a CLI-side query loop for editor integrations and scripts that need several query commands against the same DB without spawning `cdidx` repeatedly. Each newline-delimited stdin record may use the established JSON string-array form or the validated `{"command": "...", "args": [...]}` object form. Object input rejects duplicate/unknown properties, missing or blank commands, non-array `args`, non-string values, and the same argument count/length violations as array input. Serial mode opens one `DbContext` / `DbReader`; `--parallel ` requires `--json-summary`, is capped at 16 workers, and opens one isolated query-only context per active worker. Every form dispatches only commands in the side-effect-free allowlist owned by `CliCommandCatalog`. That schema includes query and read-only discovery surfaces such as `goto` and `audit`; adding a top-level command or a dispatcher arm alone cannot cross the batch safety boundary. +`cdidx batch` is a CLI-side query loop for editor integrations and scripts that need several query commands against the same DB without spawning `cdidx` repeatedly. Each newline-delimited stdin record may use the established JSON string-array form or the validated `{"command": "...", "args": [...]}` object form. Object input rejects duplicate/unknown properties, missing or blank commands, non-array `args`, non-string values, and the same argument count/length violations as array input. Serial mode opens one `DbContext` / `DbReader`; `--parallel ` requires `--json-summary`, is capped at 16 workers, and lazily retains at most one isolated query-only context per active worker slot for reuse across that batch invocation. A retained context is leased to only one command at a time, so SQLite readers are never used concurrently while repeated database open, schema probe, and reader setup costs are avoided. Every form dispatches only commands in the side-effect-free allowlist owned by `CliCommandCatalog`. That schema includes query and read-only discovery surfaces such as `goto` and `audit`; adding a top-level command or a dispatcher arm alone cannot cross the batch safety boundary. The default input budget remains 1,024 lines and is configurable through `--max-input-lines ` up to 65,536. Each decoded string argument remains capped at 8,192 characters. The JSON-summary output budget defaults to 10,485,760 characters and `--max-output-chars ` accepts 4,096 through 67,108,864. Immediate EOF with no commands remains exit 0 with no output by default; `--json-summary` appends a final JSON object with `commands_processed`, `line_errors`, `command_failures`, and `exit_code` for non-interactive callers that need an explicit empty-input signal. By default, child query commands stream their normal stdout/stderr directly. In @@ -438,9 +438,9 @@ final `record: "batch_summary"` retains `commands_processed`, `line_errors`, `command_failures`, and `exit_code`, and publishes `output_chars`, `output_char_limit`, `input_line_limit`, `parallelism`, and input/output limit state for empty-input, failure, and budget accounting. Parallel workers route -stdout/stderr through per-command bounded writers, keep a separate read-only -SQLite connection and thread-local batch reader, and buffer only the active -worker window. `ScopedConsoleOutput` keeps nested JSON-envelope capture on the +stdout/stderr through per-command bounded writers, reuse a separate read-only +SQLite connection and thread-local batch reader per active worker slot, and +buffer only the active worker window. `ScopedConsoleOutput` keeps nested JSON-envelope capture on the current worker's routed stdout instead of replacing another worker's process-wide writer. Completed records are committed to the shared output writer in input order; an ordinary item failure remains isolated. Caller cancellation is @@ -3776,7 +3776,7 @@ override が文書化されていない限り ANSI/progress control を抑止す path filter を受け付ける query コマンド（`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `map`, `inspect`, `deps`, `impact`, `unused`, `hotspots`, `validate`）は、`--project` を対応する project directory glob に展開してから `DbReader` に渡す。これにより既存の SQL path predicate をそのまま利用できる。indexed project root を解決できず process current directory に fallback して project expansion する場合、CLI query context と MCP structured payload は `project_filter_root` と `project_filter_root_fallback_reason` を含める。`index --project` は選択された project directory 配下のファイルに展開し、既存の `--files` 更新経路を再利用する。ただし 1 project で 65,536 files、requested projects 全体で 131,072 unique files を超える展開は拒否し、明示的な `--files` を使う recovery hint を返す。 -`cdidx batch` は、同じ DB に複数の query command を投げる editor integration や script 向けの CLI 側 query loop である。newline-delimited な stdin record は従来の JSON 文字列配列 form、または検証済みの `{"command": "...", "args": [...]}` object form を使用できる。object input は重複/未知 property、欠落または空白 command、array でない `args`、文字列でない値、array input と同じ引数数/長さ違反を拒否する。serial mode は 1 つの `DbContext` / `DbReader` を開く。`--parallel ` は `--json-summary` を必須とし、最大 16 workers に制限し、active worker ごとに分離した query-only context を開く。すべての form は `CliCommandCatalog` が正本となる副作用なし allowlist の command だけを dispatch する。この schema には `goto` や `audit` などの query / read-only discovery surface が含まれ、top-level command や dispatcher arm を追加しただけでは batch の安全境界を越えられない。 +`cdidx batch` は、同じ DB に複数の query command を投げる editor integration や script 向けの CLI 側 query loop である。newline-delimited な stdin record は従来の JSON 文字列配列 form、または検証済みの `{"command": "...", "args": [...]}` object form を使用できる。object input は重複/未知 property、欠落または空白 command、array でない `args`、文字列でない値、array input と同じ引数数/長さ違反を拒否する。serial mode は 1 つの `DbContext` / `DbReader` を開く。`--parallel ` は `--json-summary` を必須とし、最大 16 workers に制限する。parallel mode は active worker slot ごとに分離した query-only context を最大 1 つまで遅延作成し、同じ batch invocation 内で再利用する。保持した context は一度に 1 command だけへ貸し出すため、SQLite reader を並行利用せず、database open、schema probe、reader setup の反復コストを避ける。すべての form は `CliCommandCatalog` が正本となる副作用なし allowlist の command だけを dispatch する。この schema には `goto` や `audit` などの query / read-only discovery surface が含まれ、top-level command や dispatcher arm を追加しただけでは batch の安全境界を越えられない。 既定の入力 budget は 1,024 行のままで、`--max-input-lines ` により最大 65,536 まで設定できる。デコード後の各文字列引数は引き続き 8,192 文字に制限する。JSON-summary 出力 budget は既定で 10,485,760 文字であり、`--max-output-chars ` は 4,096 から 67,108,864 までを受け付ける。command がない即時 EOF は既定で exit 0 かつ無出力のまま維持される。非対話の呼び出し元が空入力を明示的に判定したい場合は、`--json-summary` が `commands_processed`、`line_errors`、`command_failures`、`exit_code` を含む最終 JSON オブジェクトを追加する。 既定では child query command の通常の stdout / stderr を直接 stream する。`--json-summary` @@ -3798,8 +3798,8 @@ arguments、escape 展開、terminal error、final summary を含む serialized empty input、failure、budget accounting のために `commands_processed`、`line_errors`、 `command_failures`、`exit_code`、`output_chars`、`output_char_limit`、`input_line_limit`、 `parallelism` と input / output limit state を保持する。parallel worker は stdout / stderr を -command ごとの bounded writer へ route し、分離した read-only SQLite connection と thread-local -batch reader を使い、active worker window だけを buffer する。`ScopedConsoleOutput` は nested +command ごとの bounded writer へ route し、active worker slot ごとに分離した read-only SQLite +connection と thread-local batch reader を再利用し、active worker window だけを buffer する。`ScopedConsoleOutput` は nested JSON-envelope capture を現在の worker の routed stdout に保ち、他 worker の process-wide writer を 置き換えない。完了 record は入力順で共有 output writer へ commit する。通常の item failure は 他 item から隔離する。caller cancellation は、消費済み input item と final summary に diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 9f9c0a38b..b31ac80d9 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -663,6 +663,8 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Golden-file regression fixtures for the CLI `--json` output contracts (issue #1548). Each test runs one command (`status`, `search`, `references`, `impact`, `excerpt`) against a deterministic in-memory fixture, normalizes volatile fields (timestamps, absolute paths, commit SHAs, FTS5 scores, SQLite page counts), and diffs against the matching file under `tests/CodeIndex.Tests/golden/`. Renames, removals, reordered arrays, or new keys fail the snapshot so the contract change is forced to land alongside an intentional golden update. See "JSON `--json` output snapshots" below for the update procedure. - `QueryCommandRunnerBatchIssue4723Tests.cs` CLI batch coverage for structured command objects, configurable input/output budgets, bounded parallel read overlap, input-order result emission, per-item failure isolation, and cancellation/console restoration. The deterministic overlap test blocks the first worker until the second finishes through batch-only test seams; keep those seams reset in `finally` and do not replace the signal with timing assertions. +- `QueryCommandRunnerBatchIssue4872Tests.cs` + Parallel batch session-reuse coverage. Keep the exact worker-slot session bound, serial/parallel result and input-order parity, hot-WAL snapshot fixture, warmup, and the generous 12-item/3-item ratio guard together. The benchmark injects deterministic database-open/schema work through a test seam; its fixed delay models phase cost only, while worker coordination remains signal-driven. - `PropertyBasedParserTests.cs` FsCheck-driven property tests for parser-heavy paths called out in issue #1572: `ArgHelper.WantsHelp` and `ProgramRunner.IsProjectPathArg` never throw on arbitrary inputs; `FileIndexer.NormalizePathSeparators` is idempotent under double application; the literal-safe FTS5 sanitizer (`DbReader.SanitizeFtsQuery`) always emits a query that a real in-memory FTS5 virtual table can parse. They complement, not replace, the example-based tests in `ArgHelperTests.cs` / `QueryCommandRunnerTests.cs`. - `TestProjectHelper.cs`, `TestDeterminism.cs`, `RepositoryTestPaths.cs`, `TestConsoleLock.cs` @@ -1578,6 +1580,8 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" CLI の `--json` 出力契約に対するゴールデンファイル回帰フィクスチャ (issue #1548)。各テストは `status` / `search` / `references` / `impact` / `excerpt` を決定的なインメモリ fixture に対して実行し、揺らぐフィールド（timestamp、絶対パス、commit SHA、FTS5 score、SQLite page count など）を正規化したうえで `tests/CodeIndex.Tests/golden/` 配下のファイルと差分比較します。フィールドの rename / 削除 / 並び替え / 新規追加が起きると snapshot が失敗するため、契約変更は意図的な golden 更新と同じ PR で揃えざるを得ません。更新手順は下記「JSON `--json` 出力 snapshot」を参照してください。 - `QueryCommandRunnerBatchIssue4723Tests.cs` structured command object、設定可能な input / output budget、上限付き parallel read の重複実行、入力順の result 出力、item ごとの failure isolation、cancellation / console 復元を対象とする CLI batch test です。決定的な overlap test は batch 専用 test seam を通じて第 1 worker を第 2 worker の完了まで block します。seam は `finally` で必ず reset し、signal を timing assertion に置き換えないでください。 +- `QueryCommandRunnerBatchIssue4872Tests.cs` + parallel batch の session 再利用を検証します。worker slot 数と一致する厳密な session 上限、serial / parallel の result と入力順 parity、hot-WAL snapshot fixture、warmup、十分に余裕を持たせた 12-item / 3-item ratio guard を一緒に維持してください。benchmark は test seam から決定的な database-open / schema 作業を注入します。固定 delay は phase cost のモデル化だけに使い、worker coordination は引き続き signal で制御します。 - `PropertyBasedParserTests.cs` issue #1572 で挙げられたパーサー系経路に対する FsCheck 駆動の property テスト: `ArgHelper.WantsHelp` と `ProgramRunner.IsProjectPathArg` が任意入力で例外を投げないこと、`FileIndexer.NormalizePathSeparators` が二重適用で idempotent であること、literal-safe な FTS5 サニタイザ (`DbReader.SanitizeFtsQuery`) が常にインメモリ FTS5 仮想テーブルで parse 可能なクエリを出力すること。`ArgHelperTests.cs` / `QueryCommandRunnerTests.cs` の例ベーステストを置き換えるものではなく補完します。 - `TestProjectHelper.cs`、`TestDeterminism.cs`、`RepositoryTestPaths.cs`、`TestConsoleLock.cs` diff --git a/changelog.d/unreleased/4872.fixed.md b/changelog.d/unreleased/4872.fixed.md new file mode 100644 index 000000000..2bbaabe07 --- /dev/null +++ b/changelog.d/unreleased/4872.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 4872 +affected: + - src/CodeIndex/Cli/QueryCommandRunner.Batch.cs + - tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Parallel batch queries now reuse bounded read-only sessions (#4872)** — `cdidx batch --parallel` retains at most one isolated SQLite context per active worker slot instead of reopening and reproving the database for every item, while preserving input-order envelopes, cancellation, failure isolation, and the configured worker bound. + +## 日本語 + +- **parallel batch query が上限付き read-only session を再利用するようになりました (#4872)** — `cdidx batch --parallel` は item ごとに database を再open・再検証せず、active worker slot ごとに最大 1 つの分離した SQLite context を保持します。入力順 envelope、cancellation、failure isolation、設定された worker 上限は維持します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs index 796976d44..51448b80b 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs @@ -17,6 +17,7 @@ public static partial class QueryCommandRunner internal static Action? BatchParallelCommandCompletedForTesting { get; set; } internal static Action? BatchInputLineReadForTesting { get; set; } internal static Action? BatchParallelItemPreparedForTesting { get; set; } + internal static Action? BatchParallelSessionOpenedForTesting { get; set; } public static int RunBatch( string[] cmdArgs, @@ -156,31 +157,51 @@ public static int RunBatch( if (parallelism > 1) { + BatchParallelSession? firstSession = null; try { - using var validationDb = new DbContext(DbOpenIntent.QueryOnly, dbPath, cancellationToken); + var validationDb = new DbContext(DbOpenIntent.QueryOnly, dbPath, cancellationToken); if (!validationDb.TryValidateIsCodeIndexDb(out var validationReason)) + { + validationDb.Dispose(); return WriteInvalidCodeIndexDbError(dbPath, validationReason, json: false, jsonOptions); + } + firstSession = BatchParallelSession.FromValidated(validationDb); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { + firstSession?.Dispose(); return WriteBatchSetupCancellationSummary( maxInputLines, maxOutputChars, parallelism, jsonOptions); } + catch + { + firstSession?.Dispose(); + throw; + } - return RunBatchParallel( - dbPath, - dbPathExplicit, - maxInputLines, - maxOutputChars, - parallelism, - includeRawStreams, - jsonOptions, - appVersion, - cancellationToken); + try + { + return RunBatchParallel( + dbPath, + dbPathExplicit, + maxInputLines, + maxOutputChars, + parallelism, + includeRawStreams, + jsonOptions, + appVersion, + firstSession!, + cancellationToken); + } + catch + { + firstSession?.Dispose(); + throw; + } } try @@ -454,8 +475,14 @@ private static int RunBatchParallel( bool includeRawStreams, JsonSerializerOptions jsonOptions, string appVersion, + BatchParallelSession firstSession, CancellationToken cancellationToken) { + var sessions = new BatchParallelSession[parallelism]; + sessions[0] = firstSession; + for (var index = 1; index < sessions.Length; index++) + sessions[index] = new BatchParallelSession(dbPath); + var availableSessions = new Queue(sessions); using var consoleOwnership = ConsoleStreamOwnership.Enter(); var originalOut = Console.Out; var originalError = Console.Error; @@ -618,7 +645,8 @@ await input.Writer.WriteAsync(item, stopProducing.Token) }); var active = new Queue<( BatchPendingItem Item, - Task Result)>(); + Task Result, + BatchParallelSession? Session)>(); try { @@ -626,21 +654,31 @@ await input.Writer.WriteAsync(item, stopProducing.Token) { while (active.Count < parallelism && input.Reader.TryRead(out var item)) { - var result = item.Error is not null - ? Task.FromResult(null) - : Task.Run( + BatchParallelSession? session = null; + Task result; + if (item.Error is not null) + { + result = Task.FromResult(null); + } + else + { + session = availableSessions.Dequeue(); + var assignedSession = session; + result = Task.Run( () => RunBatchParallelCommand( item.LineNumber, item.CommandName!, item.Arguments, dbPath, dbPathExplicit, + assignedSession, stdoutRouter, stderrRouter, jsonOptions, appVersion, cancellationToken)); - active.Enqueue((item, result)); + } + active.Enqueue((item, result, session)); } if (active.Count > 0 @@ -648,8 +686,10 @@ await input.Writer.WriteAsync(item, stopProducing.Token) || active.Count == parallelism || input.Reader.Completion.IsCompleted)) { - var (item, resultTask) = active.Dequeue(); + var (item, resultTask, session) = active.Dequeue(); var result = resultTask.GetAwaiter().GetResult(); + if (session is not null) + availableSessions.Enqueue(session); if (item.Error is not null) { if (item.Error.ExitCode is CommandExitCodes.CancelledBySignal @@ -818,7 +858,15 @@ await input.Writer.WriteAsync(item, stopProducing.Token) } finally { - ConsoleStreamOwnership.Restore(originalOut, originalError); + try + { + foreach (var session in sessions) + session.Dispose(); + } + finally + { + ConsoleStreamOwnership.Restore(originalOut, originalError); + } } } @@ -828,6 +876,7 @@ private static BatchParallelCommandResult RunBatchParallelCommand( string[] subArgs, string dbPath, bool dbPathExplicit, + BatchParallelSession session, BatchConsoleRouter stdoutRouter, BatchConsoleRouter stderrRouter, JsonSerializerOptions jsonOptions, @@ -844,14 +893,13 @@ private static BatchParallelCommandResult RunBatchParallelCommand( try { cancellationToken.ThrowIfCancellationRequested(); - using var db = new DbContext(DbOpenIntent.QueryOnly, dbPath, cancellationToken); - if (!db.TryValidateIsCodeIndexDb(out var validationReason)) + if (!session.TryGetReader(cancellationToken, out var reader, out var validationReason)) { exitCode = WriteInvalidCodeIndexDbError(dbPath, validationReason, json: false, jsonOptions); } else { - s_batchReader = new DbReader(db); + s_batchReader = reader; s_batchDbPath = dbPath; s_batchDbPathExplicit = dbPathExplicit; BatchParallelCommandStartedForTesting?.Invoke(lineNumber); @@ -1942,6 +1990,90 @@ public void Dispose() } } + private sealed class BatchParallelSession : IDisposable + { + private readonly string? _dbPath; + private DbContext? _db; + private DbReader? _reader; + private bool _disposed; + + public BatchParallelSession(string dbPath) + { + _dbPath = dbPath; + } + + private BatchParallelSession(DbContext validatedDb) + { + _db = validatedDb; + _reader = new DbReader(validatedDb); + } + + public static BatchParallelSession FromValidated(DbContext validatedDb) + { + var session = new BatchParallelSession(validatedDb); + try + { + BatchParallelSessionOpenedForTesting?.Invoke(); + return session; + } + catch + { + session.Dispose(); + throw; + } + } + + public bool TryGetReader( + CancellationToken cancellationToken, + out DbReader reader, + out string? validationReason) + { + ObjectDisposedException.ThrowIf(_disposed, this); + cancellationToken.ThrowIfCancellationRequested(); + if (_reader is not null) + { + reader = _reader; + validationReason = null; + return true; + } + + var db = new DbContext( + DbOpenIntent.QueryOnly, + _dbPath ?? throw new InvalidOperationException("A lazy batch session requires a database path."), + cancellationToken); + if (!db.TryValidateIsCodeIndexDb(out validationReason)) + { + db.Dispose(); + reader = null!; + return false; + } + + _db = db; + _reader = new DbReader(db); + try + { + BatchParallelSessionOpenedForTesting?.Invoke(); + } + catch + { + Dispose(); + throw; + } + reader = _reader; + return true; + } + + public void Dispose() + { + if (_disposed) + return; + _disposed = true; + _reader = null; + _db?.Dispose(); + _db = null; + } + } + private sealed class BatchJsonOutputWriter( TextWriter output, int maxChars, diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs new file mode 100644 index 000000000..f78b972eb --- /dev/null +++ b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs @@ -0,0 +1,183 @@ +using System.Diagnostics; +using CodeIndex.Cli; +using Microsoft.Data.Sqlite; + +namespace CodeIndex.Tests; + +public partial class QueryCommandRunnerTests +{ + [Fact] + public void RunBatch_ParallelReusesBoundedSessionsAndMatchesSerialRecords_Issue4872() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_session_reuse_4872"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + var input = BuildIssue4872BatchInput(9); + var (serialExitCode, serialStdout, serialStderr) = CaptureConsoleWithInput( + input, + () => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary"], + _jsonOptions)); + + using var firstWaveStarted = new CountdownEvent(3); + var openedSessions = 0; + QueryCommandRunner.BatchParallelSessionOpenedForTesting = + () => Interlocked.Increment(ref openedSessions); + QueryCommandRunner.BatchParallelCommandStartedForTesting = lineNumber => + { + if (lineNumber > 3) + return; + firstWaveStarted.Signal(); + if (!firstWaveStarted.Wait(TimeSpan.FromSeconds(5))) + throw new TimeoutException("The first parallel batch wave did not start."); + }; + + string parallelStdout; + string parallelStderr; + int parallelExitCode; + try + { + (parallelExitCode, parallelStdout, parallelStderr) = CaptureConsoleWithInput( + input, + () => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "3"], + _jsonOptions)); + } + finally + { + QueryCommandRunner.BatchParallelSessionOpenedForTesting = null; + QueryCommandRunner.BatchParallelCommandStartedForTesting = null; + } + + Assert.Equal(CommandExitCodes.Success, serialExitCode); + Assert.Equal(CommandExitCodes.Success, parallelExitCode); + Assert.Equal(string.Empty, serialStderr); + Assert.Equal(string.Empty, parallelStderr); + Assert.Equal(3, Volatile.Read(ref openedSessions)); + + var serialLines = ParseJsonLines(serialStdout); + var parallelLines = ParseJsonLines(parallelStdout); + try + { + Assert.Equal(10, serialLines.Count); + Assert.Equal(serialLines.Count, parallelLines.Count); + for (var index = 0; index < 9; index++) + { + var serialRecord = serialLines[index].RootElement; + var parallelRecord = parallelLines[index].RootElement; + Assert.Equal(index + 1, parallelRecord.GetProperty("line").GetInt32()); + Assert.Equal( + serialRecord.GetProperty("command").GetString(), + parallelRecord.GetProperty("command").GetString()); + Assert.Equal( + serialRecord.GetProperty("arguments").GetRawText(), + parallelRecord.GetProperty("arguments").GetRawText()); + Assert.Equal( + serialRecord.GetProperty("exit_code").GetInt32(), + parallelRecord.GetProperty("exit_code").GetInt32()); + Assert.Equal( + serialRecord.GetProperty("result").GetRawText(), + parallelRecord.GetProperty("result").GetRawText()); + } + + Assert.Equal(1, serialLines[^1].RootElement.GetProperty("parallelism").GetInt32()); + Assert.Equal(3, parallelLines[^1].RootElement.GetProperty("parallelism").GetInt32()); + } + finally + { + foreach (var document in serialLines) + document.Dispose(); + foreach (var document in parallelLines) + document.Dispose(); + } + } + + [Fact] + public void RunBatch_ParallelSessionReuseKeepsTwelveItemCostWithinThreeItemRatio_Issue4872() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_ratio_4872"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var writer = new SqliteConnection( + $"Data Source={dbPath};Mode=ReadWrite;Pooling=False"); + writer.Open(); + using (var command = writer.CreateCommand()) + { + command.CommandText = """ + PRAGMA journal_mode=WAL; + PRAGMA wal_autocheckpoint=0; + CREATE TABLE batch_ratio_filler(payload BLOB NOT NULL); + INSERT INTO batch_ratio_filler(payload) VALUES (zeroblob(16777216)); + """; + command.ExecuteNonQuery(); + } + Assert.True(new FileInfo(dbPath + "-wal").Length >= 16_777_216); + + _ = MeasureIssue4872ParallelBatch(dbPath, commandCount: 3); + var shortRuns = new TimeSpan[3]; + var longRuns = new TimeSpan[3]; + for (var iteration = 0; iteration < shortRuns.Length; iteration++) + { + shortRuns[iteration] = MeasureIssue4872ParallelBatch(dbPath, commandCount: 3); + longRuns[iteration] = MeasureIssue4872ParallelBatch(dbPath, commandCount: 12); + } + + var shortBaseline = shortRuns.Min(); + var longBaseline = longRuns.Min(); + var ratio = longBaseline.TotalMilliseconds / shortBaseline.TotalMilliseconds; + Assert.True( + ratio <= 3.0, + $"Parallel batch session reuse regressed: 12-item/3-item ratio was {ratio:F2} " + + $"({longBaseline.TotalMilliseconds:F1} ms / {shortBaseline.TotalMilliseconds:F1} ms)."); + } + + private TimeSpan MeasureIssue4872ParallelBatch(string dbPath, int commandCount) + { + using var firstWaveStarted = new CountdownEvent(3); + QueryCommandRunner.BatchParallelSessionOpenedForTesting = static () => + { + // Model an expensive large-index open/schema phase. This is workload cost, + // not a synchronization delay; the countdown below owns worker coordination. + Thread.Sleep(TimeSpan.FromMilliseconds(100)); + }; + QueryCommandRunner.BatchParallelCommandStartedForTesting = lineNumber => + { + if (lineNumber > 3) + return; + firstWaveStarted.Signal(); + if (!firstWaveStarted.Wait(TimeSpan.FromSeconds(5))) + throw new TimeoutException("The parallel batch benchmark wave did not start."); + }; + + try + { + var stopwatch = Stopwatch.StartNew(); + var (exitCode, _, stderr) = CaptureConsoleWithInput( + BuildIssue4872RejectedBatchInput(commandCount), + () => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "3"], + _jsonOptions)); + stopwatch.Stop(); + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stderr); + return stopwatch.Elapsed; + } + finally + { + QueryCommandRunner.BatchParallelSessionOpenedForTesting = null; + QueryCommandRunner.BatchParallelCommandStartedForTesting = null; + } + } + + private static string BuildIssue4872BatchInput(int commandCount) + => string.Join( + '\n', + Enumerable.Repeat( + """{"command":"languages","args":["--format","count"]}""", + commandCount)) + "\n"; + + private static string BuildIssue4872RejectedBatchInput(int commandCount) + => string.Join( + '\n', + Enumerable.Repeat( + """{"command":"unknown"}""", + commandCount)) + "\n"; +} From 622c5d166cf13127a3f92754ab4088ffb7bcdf4b Mon Sep 17 00:00:00 2001 From: Widthdom Date: Wed, 29 Jul 2026 14:04:39 +0900 Subject: [PATCH 2/3] Address batch session review findings (#4872) --- DEVELOPER_GUIDE.md | 4 +- TESTING_GUIDE.md | 4 +- changelog.d/unreleased/4872.fixed.md | 4 +- src/CodeIndex/Cli/QueryCommandRunner.Batch.cs | 97 +++++++--- .../QueryCommandRunnerBatchIssue4872Tests.cs | 169 ++++++++++++++++++ 5 files changed, 244 insertions(+), 34 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index aa3c99137..bd350832e 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -412,7 +412,7 @@ Interactive terminal controls are allowed only when stdout is not redirected or Query commands that accept path filters (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `map`, `inspect`, `deps`, `impact`, `unused`, `hotspots`, and `validate`) expand `--project` into the matching project directory glob before hitting `DbReader`, so all existing SQL path predicates keep working. When the indexed project root cannot be resolved and project expansion falls back to the process current directory, CLI query context and MCP structured payloads include `project_filter_root` and `project_filter_root_fallback_reason`. `index --project` expands to the files under the selected project directory and reuses the existing `--files` update path, but rejects expansions above 65,536 files for one project or 131,072 unique files across all requested projects with an explicit-files recovery hint. -`cdidx batch` is a CLI-side query loop for editor integrations and scripts that need several query commands against the same DB without spawning `cdidx` repeatedly. Each newline-delimited stdin record may use the established JSON string-array form or the validated `{"command": "...", "args": [...]}` object form. Object input rejects duplicate/unknown properties, missing or blank commands, non-array `args`, non-string values, and the same argument count/length violations as array input. Serial mode opens one `DbContext` / `DbReader`; `--parallel ` requires `--json-summary`, is capped at 16 workers, and lazily retains at most one isolated query-only context per active worker slot for reuse across that batch invocation. A retained context is leased to only one command at a time, so SQLite readers are never used concurrently while repeated database open, schema probe, and reader setup costs are avoided. Every form dispatches only commands in the side-effect-free allowlist owned by `CliCommandCatalog`. That schema includes query and read-only discovery surfaces such as `goto` and `audit`; adding a top-level command or a dispatcher arm alone cannot cross the batch safety boundary. +`cdidx batch` is a CLI-side query loop for editor integrations and scripts that need several query commands against the same DB without spawning `cdidx` repeatedly. Each newline-delimited stdin record may use the established JSON string-array form or the validated `{"command": "...", "args": [...]}` object form. Object input rejects duplicate/unknown properties, missing or blank commands, non-array `args`, non-string values, and the same argument count/length violations as array input. Serial mode opens one `DbContext` / `DbReader`; `--parallel ` requires `--json-summary`, is capped at 16 workers, and lazily retains at most one isolated query-only context per active worker slot for reuse across that batch invocation. A retained context is leased to only one command at a time, so SQLite readers are never used concurrently while repeated database open, schema probe, and reader setup costs are avoided. Detached query-only snapshots are refreshed between items when the source generation changes, so long-lived batches observe later completed index updates. Every form dispatches only commands in the side-effect-free allowlist owned by `CliCommandCatalog`. That schema includes query and read-only discovery surfaces such as `goto` and `audit`; adding a top-level command or a dispatcher arm alone cannot cross the batch safety boundary. The default input budget remains 1,024 lines and is configurable through `--max-input-lines ` up to 65,536. Each decoded string argument remains capped at 8,192 characters. The JSON-summary output budget defaults to 10,485,760 characters and `--max-output-chars ` accepts 4,096 through 67,108,864. Immediate EOF with no commands remains exit 0 with no output by default; `--json-summary` appends a final JSON object with `commands_processed`, `line_errors`, `command_failures`, and `exit_code` for non-interactive callers that need an explicit empty-input signal. By default, child query commands stream their normal stdout/stderr directly. In @@ -3776,7 +3776,7 @@ override が文書化されていない限り ANSI/progress control を抑止す path filter を受け付ける query コマンド（`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `map`, `inspect`, `deps`, `impact`, `unused`, `hotspots`, `validate`）は、`--project` を対応する project directory glob に展開してから `DbReader` に渡す。これにより既存の SQL path predicate をそのまま利用できる。indexed project root を解決できず process current directory に fallback して project expansion する場合、CLI query context と MCP structured payload は `project_filter_root` と `project_filter_root_fallback_reason` を含める。`index --project` は選択された project directory 配下のファイルに展開し、既存の `--files` 更新経路を再利用する。ただし 1 project で 65,536 files、requested projects 全体で 131,072 unique files を超える展開は拒否し、明示的な `--files` を使う recovery hint を返す。 -`cdidx batch` は、同じ DB に複数の query command を投げる editor integration や script 向けの CLI 側 query loop である。newline-delimited な stdin record は従来の JSON 文字列配列 form、または検証済みの `{"command": "...", "args": [...]}` object form を使用できる。object input は重複/未知 property、欠落または空白 command、array でない `args`、文字列でない値、array input と同じ引数数/長さ違反を拒否する。serial mode は 1 つの `DbContext` / `DbReader` を開く。`--parallel ` は `--json-summary` を必須とし、最大 16 workers に制限する。parallel mode は active worker slot ごとに分離した query-only context を最大 1 つまで遅延作成し、同じ batch invocation 内で再利用する。保持した context は一度に 1 command だけへ貸し出すため、SQLite reader を並行利用せず、database open、schema probe、reader setup の反復コストを避ける。すべての form は `CliCommandCatalog` が正本となる副作用なし allowlist の command だけを dispatch する。この schema には `goto` や `audit` などの query / read-only discovery surface が含まれ、top-level command や dispatcher arm を追加しただけでは batch の安全境界を越えられない。 +`cdidx batch` は、同じ DB に複数の query command を投げる editor integration や script 向けの CLI 側 query loop である。newline-delimited な stdin record は従来の JSON 文字列配列 form、または検証済みの `{"command": "...", "args": [...]}` object form を使用できる。object input は重複/未知 property、欠落または空白 command、array でない `args`、文字列でない値、array input と同じ引数数/長さ違反を拒否する。serial mode は 1 つの `DbContext` / `DbReader` を開く。`--parallel ` は `--json-summary` を必須とし、最大 16 workers に制限する。parallel mode は active worker slot ごとに分離した query-only context を最大 1 つまで遅延作成し、同じ batch invocation 内で再利用する。保持した context は一度に 1 command だけへ貸し出すため、SQLite reader を並行利用せず、database open、schema probe、reader setup の反復コストを避ける。source generation が変わった場合は item 間で分離 snapshot を更新するため、長時間動作する batch も後から完了した index update を観測できる。すべての form は `CliCommandCatalog` が正本となる副作用なし allowlist の command だけを dispatch する。この schema には `goto` や `audit` などの query / read-only discovery surface が含まれ、top-level command や dispatcher arm を追加しただけでは batch の安全境界を越えられない。 既定の入力 budget は 1,024 行のままで、`--max-input-lines ` により最大 65,536 まで設定できる。デコード後の各文字列引数は引き続き 8,192 文字に制限する。JSON-summary 出力 budget は既定で 10,485,760 文字であり、`--max-output-chars ` は 4,096 から 67,108,864 までを受け付ける。command がない即時 EOF は既定で exit 0 かつ無出力のまま維持される。非対話の呼び出し元が空入力を明示的に判定したい場合は、`--json-summary` が `commands_processed`、`line_errors`、`command_failures`、`exit_code` を含む最終 JSON オブジェクトを追加する。 既定では child query command の通常の stdout / stderr を直接 stream する。`--json-summary` diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index b31ac80d9..d56277387 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -664,7 +664,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding - `QueryCommandRunnerBatchIssue4723Tests.cs` CLI batch coverage for structured command objects, configurable input/output budgets, bounded parallel read overlap, input-order result emission, per-item failure isolation, and cancellation/console restoration. The deterministic overlap test blocks the first worker until the second finishes through batch-only test seams; keep those seams reset in `finally` and do not replace the signal with timing assertions. - `QueryCommandRunnerBatchIssue4872Tests.cs` - Parallel batch session-reuse coverage. Keep the exact worker-slot session bound, serial/parallel result and input-order parity, hot-WAL snapshot fixture, warmup, and the generous 12-item/3-item ratio guard together. The benchmark injects deterministic database-open/schema work through a test seam; its fixed delay models phase cost only, while worker coordination remains signal-driven. + Parallel batch session-reuse coverage. Keep the exact worker-slot session bound, serial/parallel result and input-order parity, between-item detached-snapshot refresh, reader-construction failure cleanup, hot-WAL snapshot fixture, warmup, and the generous 12-item/3-item ratio guard together. The benchmark injects deterministic database-open/schema work through a test seam; its fixed delay models phase cost only, while worker coordination remains signal-driven. - `PropertyBasedParserTests.cs` FsCheck-driven property tests for parser-heavy paths called out in issue #1572: `ArgHelper.WantsHelp` and `ProgramRunner.IsProjectPathArg` never throw on arbitrary inputs; `FileIndexer.NormalizePathSeparators` is idempotent under double application; the literal-safe FTS5 sanitizer (`DbReader.SanitizeFtsQuery`) always emits a query that a real in-memory FTS5 virtual table can parse. They complement, not replace, the example-based tests in `ArgHelperTests.cs` / `QueryCommandRunnerTests.cs`. - `TestProjectHelper.cs`, `TestDeterminism.cs`, `RepositoryTestPaths.cs`, `TestConsoleLock.cs` @@ -1581,7 +1581,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - `QueryCommandRunnerBatchIssue4723Tests.cs` structured command object、設定可能な input / output budget、上限付き parallel read の重複実行、入力順の result 出力、item ごとの failure isolation、cancellation / console 復元を対象とする CLI batch test です。決定的な overlap test は batch 専用 test seam を通じて第 1 worker を第 2 worker の完了まで block します。seam は `finally` で必ず reset し、signal を timing assertion に置き換えないでください。 - `QueryCommandRunnerBatchIssue4872Tests.cs` - parallel batch の session 再利用を検証します。worker slot 数と一致する厳密な session 上限、serial / parallel の result と入力順 parity、hot-WAL snapshot fixture、warmup、十分に余裕を持たせた 12-item / 3-item ratio guard を一緒に維持してください。benchmark は test seam から決定的な database-open / schema 作業を注入します。固定 delay は phase cost のモデル化だけに使い、worker coordination は引き続き signal で制御します。 + parallel batch の session 再利用を検証します。worker slot 数と一致する厳密な session 上限、serial / parallel の result と入力順 parity、item 間の分離 snapshot 更新、reader 構築失敗時の cleanup、hot-WAL snapshot fixture、warmup、十分に余裕を持たせた 12-item / 3-item ratio guard を一緒に維持してください。benchmark は test seam から決定的な database-open / schema 作業を注入します。固定 delay は phase cost のモデル化だけに使い、worker coordination は引き続き signal で制御します。 - `PropertyBasedParserTests.cs` issue #1572 で挙げられたパーサー系経路に対する FsCheck 駆動の property テスト: `ArgHelper.WantsHelp` と `ProgramRunner.IsProjectPathArg` が任意入力で例外を投げないこと、`FileIndexer.NormalizePathSeparators` が二重適用で idempotent であること、literal-safe な FTS5 サニタイザ (`DbReader.SanitizeFtsQuery`) が常にインメモリ FTS5 仮想テーブルで parse 可能なクエリを出力すること。`ArgHelperTests.cs` / `QueryCommandRunnerTests.cs` の例ベーステストを置き換えるものではなく補完します。 - `TestProjectHelper.cs`、`TestDeterminism.cs`、`RepositoryTestPaths.cs`、`TestConsoleLock.cs` diff --git a/changelog.d/unreleased/4872.fixed.md b/changelog.d/unreleased/4872.fixed.md index 2bbaabe07..a31f5b698 100644 --- a/changelog.d/unreleased/4872.fixed.md +++ b/changelog.d/unreleased/4872.fixed.md @@ -11,8 +11,8 @@ affected: ## English -- **Parallel batch queries now reuse bounded read-only sessions (#4872)** — `cdidx batch --parallel` retains at most one isolated SQLite context per active worker slot instead of reopening and reproving the database for every item, while preserving input-order envelopes, cancellation, failure isolation, and the configured worker bound. +- **Parallel batch queries now reuse bounded read-only sessions (#4872)** — `cdidx batch --parallel` retains at most one isolated SQLite context per active worker slot instead of reopening and reproving the database for every item, refreshes detached snapshots after source-generation changes, and cleans up failed reader construction while preserving input-order envelopes, cancellation, failure isolation, and the configured worker bound. ## 日本語 -- **parallel batch query が上限付き read-only session を再利用するようになりました (#4872)** — `cdidx batch --parallel` は item ごとに database を再open・再検証せず、active worker slot ごとに最大 1 つの分離した SQLite context を保持します。入力順 envelope、cancellation、failure isolation、設定された worker 上限は維持します。 +- **parallel batch query が上限付き read-only session を再利用するようになりました (#4872)** — `cdidx batch --parallel` は item ごとに database を再open・再検証せず、active worker slot ごとに最大 1 つの分離した SQLite context を保持します。source generation 変更後の分離 snapshot 更新と reader 構築失敗時の cleanup を行い、入力順 envelope、cancellation、failure isolation、設定された worker 上限は維持します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs index 51448b80b..e37ef2845 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs @@ -18,6 +18,7 @@ public static partial class QueryCommandRunner internal static Action? BatchInputLineReadForTesting { get; set; } internal static Action? BatchParallelItemPreparedForTesting { get; set; } internal static Action? BatchParallelSessionOpenedForTesting { get; set; } + internal static Func? BatchParallelReaderFactoryForTesting { get; set; } public static int RunBatch( string[] cmdArgs, @@ -166,7 +167,7 @@ public static int RunBatch( validationDb.Dispose(); return WriteInvalidCodeIndexDbError(dbPath, validationReason, json: false, jsonOptions); } - firstSession = BatchParallelSession.FromValidated(validationDb); + firstSession = BatchParallelSession.FromValidated(dbPath, validationDb); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -1992,7 +1993,7 @@ public void Dispose() private sealed class BatchParallelSession : IDisposable { - private readonly string? _dbPath; + private readonly string _dbPath; private DbContext? _db; private DbReader? _reader; private bool _disposed; @@ -2002,23 +2003,31 @@ public BatchParallelSession(string dbPath) _dbPath = dbPath; } - private BatchParallelSession(DbContext validatedDb) + private BatchParallelSession( + string dbPath, + DbContext validatedDb, + DbReader reader) { + _dbPath = dbPath; _db = validatedDb; - _reader = new DbReader(validatedDb); + _reader = reader; } - public static BatchParallelSession FromValidated(DbContext validatedDb) + public static BatchParallelSession FromValidated( + string dbPath, + DbContext validatedDb) { - var session = new BatchParallelSession(validatedDb); + DbReader? reader = null; try { + reader = CreateReader(validatedDb); BatchParallelSessionOpenedForTesting?.Invoke(); - return session; + return new BatchParallelSession(dbPath, validatedDb, reader); } catch { - session.Dispose(); + reader?.Dispose(); + validatedDb.Dispose(); throw; } } @@ -2030,37 +2039,59 @@ public bool TryGetReader( { ObjectDisposedException.ThrowIf(_disposed, this); cancellationToken.ThrowIfCancellationRequested(); - if (_reader is not null) + if (_reader is not null + && _db is not null + && (!_db.QueryOnlySnapshotRequiresRefresh + || _db.IsQueryOnlySnapshotCurrent(cancellationToken))) { reader = _reader; validationReason = null; return true; } - var db = new DbContext( - DbOpenIntent.QueryOnly, - _dbPath ?? throw new InvalidOperationException("A lazy batch session requires a database path."), - cancellationToken); - if (!db.TryValidateIsCodeIndexDb(out validationReason)) - { - db.Dispose(); - reader = null!; - return false; - } - - _db = db; - _reader = new DbReader(db); + DbContext? replacementDb = null; + DbReader? replacementReader = null; try { + replacementDb = new DbContext( + DbOpenIntent.QueryOnly, + _dbPath, + cancellationToken); + if (!replacementDb.TryValidateIsCodeIndexDb(out validationReason)) + { + replacementDb.Dispose(); + replacementDb = null; + reader = null!; + return false; + } + + replacementReader = CreateReader(replacementDb); BatchParallelSessionOpenedForTesting?.Invoke(); + + var previousReader = _reader; + var previousDb = _db; + _reader = replacementReader; + _db = replacementDb; + replacementReader = null; + replacementDb = null; + try + { + previousReader?.Dispose(); + } + finally + { + previousDb?.Dispose(); + } + + reader = _reader; + return true; } catch { - Dispose(); + replacementReader?.Dispose(); + replacementDb?.Dispose(); throw; } - reader = _reader; - return true; } public void Dispose() @@ -2068,10 +2099,20 @@ public void Dispose() if (_disposed) return; _disposed = true; - _reader = null; - _db?.Dispose(); - _db = null; + try + { + _reader?.Dispose(); + _reader = null; + } + finally + { + _db?.Dispose(); + _db = null; + } } + + private static DbReader CreateReader(DbContext db) + => BatchParallelReaderFactoryForTesting?.Invoke(db) ?? new DbReader(db); } private sealed class BatchJsonOutputWriter( diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs index f78b972eb..847abb908 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs @@ -1,5 +1,7 @@ using System.Diagnostics; +using System.Collections.Concurrent; using CodeIndex.Cli; +using CodeIndex.Database; using Microsoft.Data.Sqlite; namespace CodeIndex.Tests; @@ -91,6 +93,173 @@ public void RunBatch_ParallelReusesBoundedSessionsAndMatchesSerialRecords_Issue4 } } + [Fact] + public async Task RunBatch_ParallelRefreshesDetachedSnapshotsBetweenItems_Issue4872() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_snapshot_refresh_4872"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var writer = new SqliteConnection( + $"Data Source={dbPath};Mode=ReadWrite;Pooling=False"); + writer.Open(); + using (var setup = writer.CreateCommand()) + { + setup.CommandText = """ + PRAGMA journal_mode=WAL; + PRAGMA wal_autocheckpoint=0; + INSERT INTO files(path, lang, size, lines, checksum, modified) + VALUES ('src/Initial.cs', 'csharp', 1, 1, 'initial', CURRENT_TIMESTAMP); + """; + setup.ExecuteNonQuery(); + } + Assert.True(new FileInfo(dbPath + "-wal").Length > 0); + + using var input = new InteractiveBatchTextReader(); + using var stdout = new StringWriter(); + using var stderr = new StringWriter(); + using var firstWaveCompleted = new CountdownEvent(2); + using var cancellation = new CancellationTokenSource(); + Task? runTask = null; + QueryCommandRunner.BatchParallelCommandCompletedForTesting = lineNumber => + { + if (lineNumber <= 2) + firstWaveCompleted.Signal(); + }; + + try + { + runTask = Task.Run(() => + { + using var capture = ConsoleCapture.Start(stdout, stderr, input); + return QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "2"], + _jsonOptions, + cancellationToken: cancellation.Token); + }); + input.WriteLine("""{"command":"status","args":["--json"]}"""); + input.WriteLine("""{"command":"status","args":["--json"]}"""); + Assert.True( + firstWaveCompleted.Wait(TimeSpan.FromSeconds(60)), + "The first parallel batch wave did not complete."); + + using (var update = writer.CreateCommand()) + { + update.CommandText = """ + INSERT INTO files(path, lang, size, lines, checksum, modified) + VALUES ('src/Updated.cs', 'csharp', 1, 1, 'updated', CURRENT_TIMESTAMP); + """; + Assert.Equal(1, update.ExecuteNonQuery()); + } + + input.WriteLine("""{"command":"status","args":["--json"]}"""); + input.WriteLine("""{"command":"status","args":["--json"]}"""); + input.Complete(); + + var exitCode = await runTask.WaitAsync(TimeSpan.FromSeconds(60)); + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr.ToString()); + + var lines = ParseJsonLines(stdout.ToString()); + try + { + Assert.Equal(5, lines.Count); + Assert.Equal(1, lines[0].RootElement.GetProperty("result").GetProperty("files").GetInt32()); + Assert.Equal(1, lines[1].RootElement.GetProperty("result").GetProperty("files").GetInt32()); + Assert.Equal(2, lines[2].RootElement.GetProperty("result").GetProperty("files").GetInt32()); + Assert.Equal(2, lines[3].RootElement.GetProperty("result").GetProperty("files").GetInt32()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + finally + { + QueryCommandRunner.BatchParallelCommandCompletedForTesting = null; + input.Complete(); + if (runTask is { IsCompleted: false }) + { + cancellation.Cancel(); + try + { + await runTask.WaitAsync(TimeSpan.FromSeconds(15)); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + } + } + } + } + + [Fact] + public void RunBatch_ReaderConstructionFailuresDisposeDetachedSnapshots_Issue4872() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_reader_failure_4872"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var writer = new SqliteConnection( + $"Data Source={dbPath};Mode=ReadWrite;Pooling=False"); + writer.Open(); + using (var setup = writer.CreateCommand()) + { + setup.CommandText = """ + PRAGMA journal_mode=WAL; + PRAGMA wal_autocheckpoint=0; + INSERT INTO files(path, lang, size, lines, checksum, modified) + VALUES ('src/Initial.cs', 'csharp', 1, 1, 'initial', CURRENT_TIMESTAMP); + """; + setup.ExecuteNonQuery(); + } + Assert.True(new FileInfo(dbPath + "-wal").Length > 0); + + var snapshotDirectories = new ConcurrentBag(); + var readerConstructionAttempts = 0; + var originalDirectoryHook = DbConnectionFactory.QueryOnlySnapshotDirectoryCreatedForTesting; + var originalReaderFactory = QueryCommandRunner.BatchParallelReaderFactoryForTesting; + DbConnectionFactory.QueryOnlySnapshotDirectoryCreatedForTesting = snapshotDirectories.Add; + QueryCommandRunner.BatchParallelReaderFactoryForTesting = db => + { + if (Interlocked.Increment(ref readerConstructionAttempts) == 1) + return new DbReader(db); + throw new InvalidDataException("Injected parallel batch reader construction failure."); + }; + + int exitCode; + string stdout; + string stderr; + try + { + (exitCode, stdout, stderr) = CaptureConsoleWithInput( + BuildIssue4872BatchInput(6), + () => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "2"], + _jsonOptions)); + } + finally + { + QueryCommandRunner.BatchParallelReaderFactoryForTesting = originalReaderFactory; + DbConnectionFactory.QueryOnlySnapshotDirectoryCreatedForTesting = originalDirectoryHook; + } + + Assert.Equal(CommandExitCodes.RuntimeError, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.True(Volatile.Read(ref readerConstructionAttempts) >= 3); + Assert.True(snapshotDirectories.Count >= 3); + Assert.All(snapshotDirectories, path => Assert.False(Directory.Exists(path), path)); + + var lines = ParseJsonLines(stdout); + try + { + Assert.Equal(7, lines.Count); + Assert.Equal(3, lines.Take(6).Count( + line => line.RootElement.GetProperty("exit_code").GetInt32() == CommandExitCodes.RuntimeError)); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + [Fact] public void RunBatch_ParallelSessionReuseKeepsTwelveItemCostWithinThreeItemRatio_Issue4872() { From 510ceced8b0dc0a435a9a0137d97e41bde232d45 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Wed, 29 Jul 2026 15:13:36 +0900 Subject: [PATCH 3/3] Harden batch session freshness checks (#4872) --- DEVELOPER_GUIDE.md | 8 +- TESTING_GUIDE.md | 6 +- changelog.d/unreleased/4872.fixed.md | 6 +- src/CodeIndex/Cli/QueryCommandRunner.Batch.cs | 60 ++++-- src/CodeIndex/Database/DbConnectionFactory.cs | 89 ++++++++- src/CodeIndex/Database/DbContext.cs | 2 + .../QueryCommandRunnerBatchIssue4872Tests.cs | 181 +++++++++++++++++- 7 files changed, 326 insertions(+), 26 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index bd350832e..8e1ad352e 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -412,7 +412,7 @@ Interactive terminal controls are allowed only when stdout is not redirected or Query commands that accept path filters (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `map`, `inspect`, `deps`, `impact`, `unused`, `hotspots`, and `validate`) expand `--project` into the matching project directory glob before hitting `DbReader`, so all existing SQL path predicates keep working. When the indexed project root cannot be resolved and project expansion falls back to the process current directory, CLI query context and MCP structured payloads include `project_filter_root` and `project_filter_root_fallback_reason`. `index --project` expands to the files under the selected project directory and reuses the existing `--files` update path, but rejects expansions above 65,536 files for one project or 131,072 unique files across all requested projects with an explicit-files recovery hint. -`cdidx batch` is a CLI-side query loop for editor integrations and scripts that need several query commands against the same DB without spawning `cdidx` repeatedly. Each newline-delimited stdin record may use the established JSON string-array form or the validated `{"command": "...", "args": [...]}` object form. Object input rejects duplicate/unknown properties, missing or blank commands, non-array `args`, non-string values, and the same argument count/length violations as array input. Serial mode opens one `DbContext` / `DbReader`; `--parallel ` requires `--json-summary`, is capped at 16 workers, and lazily retains at most one isolated query-only context per active worker slot for reuse across that batch invocation. A retained context is leased to only one command at a time, so SQLite readers are never used concurrently while repeated database open, schema probe, and reader setup costs are avoided. Detached query-only snapshots are refreshed between items when the source generation changes, so long-lived batches observe later completed index updates. Every form dispatches only commands in the side-effect-free allowlist owned by `CliCommandCatalog`. That schema includes query and read-only discovery surfaces such as `goto` and `audit`; adding a top-level command or a dispatcher arm alone cannot cross the batch safety boundary. +`cdidx batch` is a CLI-side query loop for editor integrations and scripts that need several query commands against the same DB without spawning `cdidx` repeatedly. Each newline-delimited stdin record may use the established JSON string-array form or the validated `{"command": "...", "args": [...]}` object form. Object input rejects duplicate/unknown properties, missing or blank commands, non-array `args`, non-string values, and the same argument count/length violations as array input. Serial mode opens one `DbContext` / `DbReader`; `--parallel ` requires `--json-summary`, is capped at 16 workers, and lazily retains at most one isolated query-only context per active worker slot for reuse across that batch invocation. A retained context is leased to only one command at a time, so SQLite readers are never used concurrently while repeated database open, schema probe, and reader setup costs are avoided. Between items, each retained direct connection or detached snapshot verifies the source SQLite header/WAL generation and file identity, replacing the session whenever freshness cannot be proved so long-lived batches observe later completed index updates. Every form dispatches only commands in the side-effect-free allowlist owned by `CliCommandCatalog`. That schema includes query and read-only discovery surfaces such as `goto` and `audit`; adding a top-level command or a dispatcher arm alone cannot cross the batch safety boundary. The default input budget remains 1,024 lines and is configurable through `--max-input-lines ` up to 65,536. Each decoded string argument remains capped at 8,192 characters. The JSON-summary output budget defaults to 10,485,760 characters and `--max-output-chars ` accepts 4,096 through 67,108,864. Immediate EOF with no commands remains exit 0 with no output by default; `--json-summary` appends a final JSON object with `commands_processed`, `line_errors`, `command_failures`, and `exit_code` for non-interactive callers that need an explicit empty-input signal. By default, child query commands stream their normal stdout/stderr directly. In @@ -1693,7 +1693,7 @@ access. | Recipe SARIF | `search --recipe --format sarif` emits one result per bounded recipe result. Rule IDs use `recipe/query`; standard `fingerprints.cdidx/v1` values are derived from the normalized source location; result properties preserve recipe/query identity, severity, confidence, and per-query truncation; run properties preserve scope, applied result limits, aggregate counts, and conservative omitted-result metadata. Bound SARIF with `--limit` / `--total-limit`; row selectors such as `--sample`, `--first-per-file`, and `--per-file-limit` are rejected instead of being silently ignored. Recipe severity maps `critical` / `high` to `error`, `medium` to `warning`, and `low` / `info` to `note`. | | Recipe classifier output | Recipe run JSON may add `audit_classifications` to individual `CompactSearchResult` rows when a recipe classifier can classify the hit, and query/count payloads may add `classifier_counts` when classified rows are present. These fields are additive; use them to separate triage domains such as DTO/result-wrapper `.Result` properties versus Task/ValueTask blocking waits without changing the raw search query. | | NDJSON terminal records | Default NDJSON for `search`, `symbols`, and `files` appends one final `terminal_record` after result rows; search also emits it for zero-result responses, while raw `symbols` and `files` keep zero-result NDJSON empty. Recipe/audit search row streams share the same writer. Terminals report returned and observed total counts, `total_count_authoritative` / `total_count_lower_bound`, selection or interruption reason, applied limits, omitted rows, and recovery guidance. `--max-json-bytes` covers the complete stdout stream, including newlines and this terminal record; when additive selector-accounting fields prevent the terminal from fitting, the writer omits those optional fields before declaring the terminal impossible. A cap that still cannot fit the terminal fails before stdout. Capped output rejects `--profile`, `--verbose`, and `--json-envelope`. Byte-cap partial output exits with `CommandExitCodes.PartialResult` (`11`) unless `--allow-partial` explicitly opts into exit `0`. `--results-only` is the explicit terminal-record opt-out for these NDJSON row streams and is rejected with array, compact, summary, or count output. | -| `outline` / `unused` cursor binding | `outline --json` accepts `--kind `, `--limit` / `--top`, opaque `--cursor `, and `--outline-fields ` for bounded machine output. Controlled outline responses keep the normal envelope and add `total_symbol_count`, `returned_symbol_count`, `cursor_offset`, `next_cursor`, `has_more`, and `result_stable_at`, plus `kind_filter` and `selected_fields` when active. `outline` and `unused` cursors bind their offset to the normalized path/scope, filters, ordering, and index generation; reuse after changing those inputs or refreshing the index fails with explicit restart-required guidance. Legacy `outline:` / `unused:` inputs remain accepted for transition, but every newly emitted cursor is opaque and bound. | +| `outline` / `unused` cursor binding | `outline --json` accepts `--kind `, `--limit` / `--top`, opaque `--cursor `, and `--outline-fields ` for bounded machine output. Controlled outline responses keep the normal envelope and add `total_symbol_count`, `returned_symbol_count`, `cursor_offset`, `next_cursor`, `has_more`, and `result_stable_at`, plus `kind_filter` and `selected_fields` when active. Projection parsing canonicalizes aliases and removes duplicates before validation; unknown field names are reported together as one terminal usage error with valid candidates, while the empty-selection error is reserved for deliberately empty CSV input. `outline` and `unused` cursors bind their offset to the normalized path/scope, filters, ordering, and index generation; reuse after changing those inputs or refreshing the index fails with explicit restart-required guidance. Legacy `outline:` / `unused:` inputs remain accepted for transition, but every newly emitted cursor is opaque and bound. | | `hotspots --json` grouping semantics | `hotspots` and MCP `symbol_hotspots` emit `grouped_by`, `grouping_unit`, `count_kind`, `limit_applies_to`, `score_fields`, `ranking_fields`, and matching `query_context` fields. `--limit` applies to returned symbols, files, name/kind groups, or SQL statements; `--count` ignores `--limit` and reports total groups. Explicit `statement` grouping is SQL-only (`--lang sql` / `lang: "sql"`). | | `--json-envelope` commands | Applies to `search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `languages`, `impact`, `deps`, `unused`, and `hotspots`. | | `--json-envelope` shape | Wraps the per-line `--json` stream into a single `{"metadata": {...}, "results": [...]}` document. Stream terminal records are excluded from `results` and preserved as `metadata.stream_terminal`, while zero-result prelude/control records are preserved as `metadata.stream_control_records`; therefore `result_count` counts result rows only. A `find --all --count` object is both the count result and terminal scan metadata, so it remains in `results` and is also copied to `metadata.stream_terminal`. `metadata` also carries `api_version`, `command`, `cdidx_version`, `elapsed_ms`, `db_path`, `exit_code`, and, when applicable, `query_normalized` and `indexed_at_head_sha`. `indexed_at_head_sha` maps to the persisted latest-successful `indexed_head_sha` used by status and MCP output after full, `--files`, `--commits`, and `--changed-between` refreshes; failed/rolled-back refreshes do not advance it, and legacy DBs without that key fall back to full-scan-only `indexed_head_commit`. The bounded high-volume commands above support `--json-envelope --max-json-bytes` by measuring the final serialized document; other envelope/byte-cap combinations remain rejected. | @@ -3776,7 +3776,7 @@ override が文書化されていない限り ANSI/progress control を抑止す path filter を受け付ける query コマンド（`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `map`, `inspect`, `deps`, `impact`, `unused`, `hotspots`, `validate`）は、`--project` を対応する project directory glob に展開してから `DbReader` に渡す。これにより既存の SQL path predicate をそのまま利用できる。indexed project root を解決できず process current directory に fallback して project expansion する場合、CLI query context と MCP structured payload は `project_filter_root` と `project_filter_root_fallback_reason` を含める。`index --project` は選択された project directory 配下のファイルに展開し、既存の `--files` 更新経路を再利用する。ただし 1 project で 65,536 files、requested projects 全体で 131,072 unique files を超える展開は拒否し、明示的な `--files` を使う recovery hint を返す。 -`cdidx batch` は、同じ DB に複数の query command を投げる editor integration や script 向けの CLI 側 query loop である。newline-delimited な stdin record は従来の JSON 文字列配列 form、または検証済みの `{"command": "...", "args": [...]}` object form を使用できる。object input は重複/未知 property、欠落または空白 command、array でない `args`、文字列でない値、array input と同じ引数数/長さ違反を拒否する。serial mode は 1 つの `DbContext` / `DbReader` を開く。`--parallel ` は `--json-summary` を必須とし、最大 16 workers に制限する。parallel mode は active worker slot ごとに分離した query-only context を最大 1 つまで遅延作成し、同じ batch invocation 内で再利用する。保持した context は一度に 1 command だけへ貸し出すため、SQLite reader を並行利用せず、database open、schema probe、reader setup の反復コストを避ける。source generation が変わった場合は item 間で分離 snapshot を更新するため、長時間動作する batch も後から完了した index update を観測できる。すべての form は `CliCommandCatalog` が正本となる副作用なし allowlist の command だけを dispatch する。この schema には `goto` や `audit` などの query / read-only discovery surface が含まれ、top-level command や dispatcher arm を追加しただけでは batch の安全境界を越えられない。 +`cdidx batch` は、同じ DB に複数の query command を投げる editor integration や script 向けの CLI 側 query loop である。newline-delimited な stdin record は従来の JSON 文字列配列 form、または検証済みの `{"command": "...", "args": [...]}` object form を使用できる。object input は重複/未知 property、欠落または空白 command、array でない `args`、文字列でない値、array input と同じ引数数/長さ違反を拒否する。serial mode は 1 つの `DbContext` / `DbReader` を開く。`--parallel ` は `--json-summary` を必須とし、最大 16 workers に制限する。parallel mode は active worker slot ごとに分離した query-only context を最大 1 つまで遅延作成し、同じ batch invocation 内で再利用する。保持した context は一度に 1 command だけへ貸し出すため、SQLite reader を並行利用せず、database open、schema probe、reader setup の反復コストを避ける。各 item の間では、保持中の direct connection と分離 snapshot の両方で source SQLite header / WAL generation と file identity を検証し、freshness を証明できない場合は session を置き換えるため、長時間動作する batch も後から完了した index update を観測できる。すべての form は `CliCommandCatalog` が正本となる副作用なし allowlist の command だけを dispatch する。この schema には `goto` や `audit` などの query / read-only discovery surface が含まれ、top-level command や dispatcher arm を追加しただけでは batch の安全境界を越えられない。 既定の入力 budget は 1,024 行のままで、`--max-input-lines ` により最大 65,536 まで設定できる。デコード後の各文字列引数は引き続き 8,192 文字に制限する。JSON-summary 出力 budget は既定で 10,485,760 文字であり、`--max-output-chars ` は 4,096 から 67,108,864 までを受け付ける。command がない即時 EOF は既定で exit 0 かつ無出力のまま維持される。非対話の呼び出し元が空入力を明示的に判定したい場合は、`--json-summary` が `commands_processed`、`line_errors`、`command_failures`、`exit_code` を含む最終 JSON オブジェクトを追加する。 既定では child query command の通常の stdout / stderr を直接 stream する。`--json-summary` @@ -5060,7 +5060,7 @@ help はすべてこのレジストリを参照します。field 名は大文字 | Recipe SARIF | `search --recipe --format sarif` は、上限付き recipe result ごとに result を1件出力します。rule ID は `recipe/query` を使い、標準の `fingerprints.cdidx/v1` は正規化済み source location から導出します。result properties は recipe/query identity、severity、confidence、query ごとの truncation を保持し、run properties は scope、適用済み result limit、集計 count、保守的な omitted-result metadata を保持します。SARIF の上限には `--limit` / `--total-limit` を使い、`--sample`、`--first-per-file`、`--per-file-limit` のような row selector は黙って無視せず拒否します。recipe severity は `critical` / `high` を `error`、`medium` を `warning`、`low` / `info` を `note` に対応付けます。 | | Recipe classifier output | recipe classifier が hit を分類できる場合、recipe run JSON は個別の `CompactSearchResult` row に `audit_classifications` を追加することがあり、分類済み row がある query / count payload は `classifier_counts` を追加することがあります。これらは additive field です。raw search query を変えずに、DTO / result-wrapper の `.Result` property と Task / ValueTask の blocking wait などの triage domain を分離するために使います。 | | NDJSON terminal record | `search`、`symbols`、`files` の既定 NDJSON は result row の後に最後の `terminal_record` を 1 件追加します。`search` は 0 件応答にも終端を出力しますが、raw `symbols` / `files` の 0 件 NDJSON は空のままです。recipe / audit search の row stream も同じ writer を使います。終端は返却件数と観測済み総件数、`total_count_authoritative` / `total_count_lower_bound`、selection または中断理由、適用上限、省略行数、復旧案内を報告します。`--max-json-bytes` は改行と終端レコードを含む stdout stream 全体を対象にし、追加 selector-accounting field が原因で終端が収まらない場合は、終端自体を不可能と判定する前にそれらの任意 field を省略します。それでも終端が収まらない cap は stdout 出力前に失敗します。上限付き出力は `--profile`、`--verbose`、`--json-envelope` を拒否します。byte cap による部分出力は、`--allow-partial` で終了コード `0` を明示許可しない限り `CommandExitCodes.PartialResult`（`11`）を返します。`--results-only` はこれらの NDJSON row stream から終端レコードを明示的に除外するための option であり、array / compact / summary / count 出力との組み合わせは拒否されます。 | -| `outline` / `unused` cursor の束縛 | `outline --json` は bounded な機械向け出力として `--kind `、`--limit` / `--top`、opaque な `--cursor `、`--outline-fields ` を受け付けます。制御付き outline 応答は通常の envelope を維持し、`total_symbol_count`、`returned_symbol_count`、`cursor_offset`、`next_cursor`、`has_more`、`result_stable_at` を追加し、該当時は `kind_filter` と `selected_fields` も返します。`outline` と `unused` の cursor は offset を正規化済み path/scope、filter、ordering、index generation に束縛するため、条件変更後または index 更新後の再利用は restart-required の明示案内付きで失敗します。移行用に legacy の `outline:` / `unused:` 入力は受理しますが、新しく出力する cursor はすべて opaque かつ束縛済みです。 | +| `outline` / `unused` cursor の束縛 | `outline --json` は bounded な機械向け出力として `--kind `、`--limit` / `--top`、opaque な `--cursor `、`--outline-fields ` を受け付けます。制御付き outline 応答は通常の envelope を維持し、`total_symbol_count`、`returned_symbol_count`、`cursor_offset`、`next_cursor`、`has_more`、`result_stable_at` を追加し、該当時は `kind_filter` と `selected_fields` も返します。projection parser は検証前に alias を canonicalize して重複を除きます。未知field名はvalid候補を伴う1つの終端usage errorにまとめ、empty-selection errorは意図的に空のCSV入力にだけ使います。`outline` と `unused` の cursor は offset を正規化済み path/scope、filter、ordering、index generation に束縛するため、条件変更後または index 更新後の再利用は restart-required の明示案内付きで失敗します。移行用に legacy の `outline:` / `unused:` 入力は受理しますが、新しく出力する cursor はすべて opaque かつ束縛済みです。 | | `hotspots --json` grouping semantics | `hotspots` と MCP `symbol_hotspots` は `grouped_by`、`grouping_unit`、`count_kind`、`limit_applies_to`、`score_fields`、`ranking_fields` と、対応する `query_context` field を返します。`--limit` は返却される symbol、file、name/kind group、SQL statement に適用されます。`--count` は `--limit` を無視し、total group 数を返します。明示的な `statement` grouping は SQL 専用です（`--lang sql` / `lang: "sql"`）。 | | `--json-envelope` 対象 command | `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`find`、`excerpt`、`map`、`inspect`、`outline`、`status`、`validate`、`languages`、`impact`、`deps`、`unused`、`hotspots`。 | | `--json-envelope` shape | per-line `--json` stream を単一の `{"metadata": {...}, "results": [...]}` document に包みます。stream 終端レコードは `results` から除外して `metadata.stream_terminal` に保持し、0 件時の prelude / control record は `metadata.stream_control_records` に保持するため、`result_count` は result row だけを数えます。`find --all --count` object は count result であると同時に終端 scan metadata でもあるため、`results` に残しつつ `metadata.stream_terminal` にも複製します。`metadata` は `api_version`、`command`、`cdidx_version`、`elapsed_ms`、`db_path`、`exit_code`、該当時は `query_normalized` と `indexed_at_head_sha` も持ちます。`indexed_at_head_sha` は full、`--files`、`--commits`、`--changed-between` refresh 後に status / MCP output が使う永続化済みの最新成功 `indexed_head_sha` に対応します。失敗または rollback された refresh では進まず、この key を持たない legacy DB では full-scan 限定 `indexed_head_commit` に fallback します。上記の bounded 高ボリューム command は最終 document を測定することで `--json-envelope --max-json-bytes` を許可し、それ以外の envelope / byte-cap 組み合わせは引き続き拒否します。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index d56277387..334374973 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -219,6 +219,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Outline `size` sorting and its `span` alias share one ranking fixture. Outline reference and complexity metric sorting share one derived-ranking fixture. Outline kind sorting and default source-order field projection share one ranking fixture. + Outline projection validation reuses one parser/JSON-error fixture across single and multiple unknown fields, deliberately empty lists, aliases, duplicates, and mixed valid/invalid input; assert that unknown fields form one terminal usage error and do not trigger the empty-selection diagnostic. Deps JSON and json-graph byte-limit failures share one SQL graph fixture. Deps JSON summary output and json-graph summary rejection share one SQL graph fixture. Dependency-cycle coverage must prove that the graph budget is independent of the display limit, SCC ranking remains stable when the page size grows, opaque cursors return the next ranked component, mismatched cursor filters fail closed, and graph-budget exhaustion marks totals as non-authoritative. @@ -664,7 +665,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding - `QueryCommandRunnerBatchIssue4723Tests.cs` CLI batch coverage for structured command objects, configurable input/output budgets, bounded parallel read overlap, input-order result emission, per-item failure isolation, and cancellation/console restoration. The deterministic overlap test blocks the first worker until the second finishes through batch-only test seams; keep those seams reset in `finally` and do not replace the signal with timing assertions. - `QueryCommandRunnerBatchIssue4872Tests.cs` - Parallel batch session-reuse coverage. Keep the exact worker-slot session bound, serial/parallel result and input-order parity, between-item detached-snapshot refresh, reader-construction failure cleanup, hot-WAL snapshot fixture, warmup, and the generous 12-item/3-item ratio guard together. The benchmark injects deterministic database-open/schema work through a test seam; its fixed delay models phase cost only, while worker coordination remains signal-driven. + Parallel batch session-reuse coverage. Keep the exact worker-slot session bound, serial/parallel result and input-order parity, between-item checkpointed-WAL snapshot and direct-session refresh, validation/reader-construction failure cleanup, hot-WAL snapshot fixture, warmup, and the generous 12-item/3-item ratio guard together. The benchmark injects deterministic database-open/schema work through a test seam; its fixed delay models phase cost only, while worker coordination remains signal-driven. - `PropertyBasedParserTests.cs` FsCheck-driven property tests for parser-heavy paths called out in issue #1572: `ArgHelper.WantsHelp` and `ProgramRunner.IsProjectPathArg` never throw on arbitrary inputs; `FileIndexer.NormalizePathSeparators` is idempotent under double application; the literal-safe FTS5 sanitizer (`DbReader.SanitizeFtsQuery`) always emits a query that a real in-memory FTS5 virtual table can parse. They complement, not replace, the example-based tests in `ArgHelperTests.cs` / `QueryCommandRunnerTests.cs`. - `TestProjectHelper.cs`, `TestDeterminism.cs`, `RepositoryTestPaths.cs`, `TestConsoleLock.cs` @@ -1144,6 +1145,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" outlineの`size` sortとその`span` aliasは1つのranking fixtureを共有してください。 outlineのreference metric sortとcomplexity metric sortは1つのderived-ranking fixtureを共有してください。 outlineのkind sortとdefault source-order field projectionは1つのranking fixtureを共有してください。 + outline projection validationは、単一・複数の未知field、意図的な空list、alias、重複、valid/invalid混在を1つのparser/JSON-error fixtureで共有してください。未知fieldが1つの終端usage errorを形成し、empty-selection diagnosticを追加で発生させないことを検証してください。 depsのJSONとjson-graphのbyte-limit failureは1つのSQL graph fixtureを共有してください。 depsのJSON summary outputとjson-graph summary rejectionは1つのSQL graph fixtureを共有してください。 dependency-cycle coverage では、graph budget が表示 limit から独立していること、page size を増やしても SCC 順位が安定すること、不透明 cursor が次の順位の component を返すこと、cursor と filter の不一致が fail-closed になること、graph-budget 枯渇時に総件数が non-authoritative と示されることを検証してください。 @@ -1581,7 +1583,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - `QueryCommandRunnerBatchIssue4723Tests.cs` structured command object、設定可能な input / output budget、上限付き parallel read の重複実行、入力順の result 出力、item ごとの failure isolation、cancellation / console 復元を対象とする CLI batch test です。決定的な overlap test は batch 専用 test seam を通じて第 1 worker を第 2 worker の完了まで block します。seam は `finally` で必ず reset し、signal を timing assertion に置き換えないでください。 - `QueryCommandRunnerBatchIssue4872Tests.cs` - parallel batch の session 再利用を検証します。worker slot 数と一致する厳密な session 上限、serial / parallel の result と入力順 parity、item 間の分離 snapshot 更新、reader 構築失敗時の cleanup、hot-WAL snapshot fixture、warmup、十分に余裕を持たせた 12-item / 3-item ratio guard を一緒に維持してください。benchmark は test seam から決定的な database-open / schema 作業を注入します。固定 delay は phase cost のモデル化だけに使い、worker coordination は引き続き signal で制御します。 + parallel batch の session 再利用を検証します。worker slot 数と一致する厳密な session 上限、serial / parallel の result と入力順 parity、item 間の checkpoint 済み WAL snapshot / direct session 更新、validation / reader 構築失敗時の cleanup、hot-WAL snapshot fixture、warmup、十分に余裕を持たせた 12-item / 3-item ratio guard を一緒に維持してください。benchmark は test seam から決定的な database-open / schema 作業を注入します。固定 delay は phase cost のモデル化だけに使い、worker coordination は引き続き signal で制御します。 - `PropertyBasedParserTests.cs` issue #1572 で挙げられたパーサー系経路に対する FsCheck 駆動の property テスト: `ArgHelper.WantsHelp` と `ProgramRunner.IsProjectPathArg` が任意入力で例外を投げないこと、`FileIndexer.NormalizePathSeparators` が二重適用で idempotent であること、literal-safe な FTS5 サニタイザ (`DbReader.SanitizeFtsQuery`) が常にインメモリ FTS5 仮想テーブルで parse 可能なクエリを出力すること。`ArgHelperTests.cs` / `QueryCommandRunnerTests.cs` の例ベーステストを置き換えるものではなく補完します。 - `TestProjectHelper.cs`、`TestDeterminism.cs`、`RepositoryTestPaths.cs`、`TestConsoleLock.cs` diff --git a/changelog.d/unreleased/4872.fixed.md b/changelog.d/unreleased/4872.fixed.md index a31f5b698..70fb48c7c 100644 --- a/changelog.d/unreleased/4872.fixed.md +++ b/changelog.d/unreleased/4872.fixed.md @@ -4,6 +4,8 @@ issues: - 4872 affected: - src/CodeIndex/Cli/QueryCommandRunner.Batch.cs + - src/CodeIndex/Database/DbConnectionFactory.cs + - src/CodeIndex/Database/DbContext.cs - tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs - DEVELOPER_GUIDE.md - TESTING_GUIDE.md @@ -11,8 +13,8 @@ affected: ## English -- **Parallel batch queries now reuse bounded read-only sessions (#4872)** — `cdidx batch --parallel` retains at most one isolated SQLite context per active worker slot instead of reopening and reproving the database for every item, refreshes detached snapshots after source-generation changes, and cleans up failed reader construction while preserving input-order envelopes, cancellation, failure isolation, and the configured worker bound. +- **Parallel batch queries now reuse bounded read-only sessions (#4872)** — `cdidx batch --parallel` retains at most one isolated SQLite context per active worker slot instead of reopening and reproving the database for every item, verifies source generation and file identity for direct connections and detached snapshots between items, and cleans up validation/reader failures while preserving input-order envelopes, cancellation, failure isolation, and the configured worker bound. ## 日本語 -- **parallel batch query が上限付き read-only session を再利用するようになりました (#4872)** — `cdidx batch --parallel` は item ごとに database を再open・再検証せず、active worker slot ごとに最大 1 つの分離した SQLite context を保持します。source generation 変更後の分離 snapshot 更新と reader 構築失敗時の cleanup を行い、入力順 envelope、cancellation、failure isolation、設定された worker 上限は維持します。 +- **parallel batch query が上限付き read-only session を再利用するようになりました (#4872)** — `cdidx batch --parallel` は item ごとに database を再open・再検証せず、active worker slot ごとに最大 1 つの分離した SQLite context を保持します。item 間で direct connection と分離 snapshot の source generation / file identity を検証し、validation / reader failure を cleanup しながら、入力順 envelope、cancellation、failure isolation、設定された worker 上限を維持します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs index e37ef2845..504be07b8 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs @@ -19,6 +19,7 @@ public static partial class QueryCommandRunner internal static Action? BatchParallelItemPreparedForTesting { get; set; } internal static Action? BatchParallelSessionOpenedForTesting { get; set; } internal static Func? BatchParallelReaderFactoryForTesting { get; set; } + internal static Action? BatchParallelDatabaseValidatingForTesting { get; set; } public static int RunBatch( string[] cmdArgs, @@ -159,15 +160,19 @@ public static int RunBatch( if (parallelism > 1) { BatchParallelSession? firstSession = null; + DbContext? validationDb = null; try { - var validationDb = new DbContext(DbOpenIntent.QueryOnly, dbPath, cancellationToken); + validationDb = new DbContext(DbOpenIntent.QueryOnly, dbPath, cancellationToken); + BatchParallelDatabaseValidatingForTesting?.Invoke(); if (!validationDb.TryValidateIsCodeIndexDb(out var validationReason)) - { - validationDb.Dispose(); return WriteInvalidCodeIndexDbError(dbPath, validationReason, json: false, jsonOptions); - } - firstSession = BatchParallelSession.FromValidated(dbPath, validationDb); + var transferredDb = validationDb; + validationDb = null; + firstSession = BatchParallelSession.FromValidated( + dbPath, + transferredDb, + cancellationToken); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -183,6 +188,10 @@ public static int RunBatch( firstSession?.Dispose(); throw; } + finally + { + validationDb?.Dispose(); + } try { @@ -1996,6 +2005,8 @@ private sealed class BatchParallelSession : IDisposable private readonly string _dbPath; private DbContext? _db; private DbReader? _reader; + private DbConnectionFactory.QueryOnlySnapshotSourceState? _sourceState; + private bool _readerUsed; private bool _disposed; public BatchParallelSession(string dbPath) @@ -2006,23 +2017,27 @@ public BatchParallelSession(string dbPath) private BatchParallelSession( string dbPath, DbContext validatedDb, - DbReader reader) + DbReader reader, + DbConnectionFactory.QueryOnlySnapshotSourceState? sourceState) { _dbPath = dbPath; _db = validatedDb; _reader = reader; + _sourceState = sourceState; } public static BatchParallelSession FromValidated( string dbPath, - DbContext validatedDb) + DbContext validatedDb, + CancellationToken cancellationToken) { DbReader? reader = null; try { reader = CreateReader(validatedDb); + var sourceState = CaptureSourceState(dbPath, validatedDb, cancellationToken); BatchParallelSessionOpenedForTesting?.Invoke(); - return new BatchParallelSession(dbPath, validatedDb, reader); + return new BatchParallelSession(dbPath, validatedDb, reader, sourceState); } catch { @@ -2041,9 +2056,14 @@ public bool TryGetReader( cancellationToken.ThrowIfCancellationRequested(); if (_reader is not null && _db is not null - && (!_db.QueryOnlySnapshotRequiresRefresh - || _db.IsQueryOnlySnapshotCurrent(cancellationToken))) - { + && (!_readerUsed + || (_sourceState is { } sourceState + && DbConnectionFactory.IsQuerySourceStateCurrent( + _dbPath, + sourceState, + cancellationToken)))) + { + _readerUsed = true; reader = _reader; validationReason = null; return true; @@ -2066,12 +2086,18 @@ public bool TryGetReader( } replacementReader = CreateReader(replacementDb); + var replacementSourceState = CaptureSourceState( + _dbPath, + replacementDb, + cancellationToken); BatchParallelSessionOpenedForTesting?.Invoke(); var previousReader = _reader; var previousDb = _db; _reader = replacementReader; _db = replacementDb; + _sourceState = replacementSourceState; + _readerUsed = true; replacementReader = null; replacementDb = null; try @@ -2111,6 +2137,18 @@ public void Dispose() } } + private static DbConnectionFactory.QueryOnlySnapshotSourceState? CaptureSourceState( + string dbPath, + DbContext db, + CancellationToken cancellationToken) + => db.QueryOnlySnapshotSourceState + ?? (DbConnectionFactory.TryCaptureQuerySourceState( + dbPath, + cancellationToken, + out var sourceState) + ? sourceState + : null); + private static DbReader CreateReader(DbContext db) => BatchParallelReaderFactoryForTesting?.Invoke(db) ?? new DbReader(db); } diff --git a/src/CodeIndex/Database/DbConnectionFactory.cs b/src/CodeIndex/Database/DbConnectionFactory.cs index ca828a39a..d24799ecf 100644 --- a/src/CodeIndex/Database/DbConnectionFactory.cs +++ b/src/CodeIndex/Database/DbConnectionFactory.cs @@ -316,6 +316,37 @@ internal static bool IsQueryOnlySnapshotCurrent( && currentState == snapshotSourceState; } + internal static bool TryCaptureQuerySourceState( + string dbPath, + CancellationToken cancellationToken, + out QueryOnlySnapshotSourceState state) + { + cancellationToken.ThrowIfCancellationRequested(); + var localDbPath = dbPath; + if (SqliteFileUri.StartsWithFileScheme(dbPath)) + { + if (!TryGetLocalPath(dbPath, out var parsedPath, out _) || parsedPath == null) + { + state = default; + return false; + } + localDbPath = parsedPath; + } + + return TryCaptureQueryOnlySnapshotState( + localDbPath, + cancellationToken, + out state, + requireWalMode: false); + } + + internal static bool IsQuerySourceStateCurrent( + string dbPath, + QueryOnlySnapshotSourceState sourceState, + CancellationToken cancellationToken = default) + => TryCaptureQuerySourceState(dbPath, cancellationToken, out var currentState) + && currentState == sourceState; + private static SqliteConnection CreateStableWalSnapshotConnection( string localDbPath, out bool copiedHotWal, @@ -365,7 +396,7 @@ private static SqliteConnection CreateStableWalSnapshotConnection( if (TryCaptureQueryOnlySnapshotState(snapshotDbPath, cancellationToken, out var copied) && TryCaptureQueryOnlySnapshotState(normalizedDbPath, cancellationToken, out var after) - && before == copied + && before.ContentEquals(copied) && before == after) { copiedHotWal = before.WalLength > 0; @@ -444,7 +475,8 @@ private static void CopySnapshotFile(string sourcePath, string destinationPath, private static bool TryCaptureQueryOnlySnapshotState( string localDbPath, CancellationToken cancellationToken, - out QueryOnlySnapshotSourceState state) + out QueryOnlySnapshotSourceState state, + bool requireWalMode = true) { try { @@ -465,8 +497,7 @@ private static bool TryCaptureQueryOnlySnapshotState( if (dbHeaderLength < 20 || !dbHeader[..16].SequenceEqual("SQLite format 3\0"u8) - || dbHeader[18] != 2 - || dbHeader[19] != 2) + || (requireWalMode && (dbHeader[18] != 2 || dbHeader[19] != 2))) { state = default; return false; @@ -513,12 +544,24 @@ private static bool TryCaptureQueryOnlySnapshotState( walLength = 0; } + var dbFile = TryReadSourceFileIdentity(normalizedDbPath); + var walFile = TryReadSourceFileIdentity(walPath); + if (dbFile is null + || dbFile.Value.Length != dbLength + || (walFile?.Length ?? 0) != walLength) + { + state = default; + return false; + } + state = new QueryOnlySnapshotSourceState( dbLength, Fingerprint(dbHeader[..dbHeaderLength]), walLength, walHeaderFingerprint, - walLastFrameFingerprint); + walLastFrameFingerprint, + dbFile.Value, + walFile); return true; } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or NotSupportedException) @@ -528,6 +571,25 @@ private static bool TryCaptureQueryOnlySnapshotState( } } + private static QuerySourceFileIdentity? TryReadSourceFileIdentity(string path) + { + try + { + var file = new FileInfo(path); + file.Refresh(); + return file.Exists + ? new QuerySourceFileIdentity( + file.Length, + file.CreationTimeUtc.Ticks, + file.LastWriteTimeUtc.Ticks) + : null; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or NotSupportedException) + { + return null; + } + } + private static int ReadAtMost(Stream stream, Span destination, CancellationToken cancellationToken) { var total = 0; @@ -621,7 +683,22 @@ internal readonly record struct QueryOnlySnapshotSourceState( string DbHeaderFingerprint, long WalLength, string? WalHeaderFingerprint, - string? WalLastFrameFingerprint); + string? WalLastFrameFingerprint, + QuerySourceFileIdentity DatabaseFile, + QuerySourceFileIdentity? WalFile) + { + internal bool ContentEquals(QueryOnlySnapshotSourceState other) + => DbLength == other.DbLength + && DbHeaderFingerprint == other.DbHeaderFingerprint + && WalLength == other.WalLength + && WalHeaderFingerprint == other.WalHeaderFingerprint + && WalLastFrameFingerprint == other.WalLastFrameFingerprint; + } + + internal readonly record struct QuerySourceFileIdentity( + long Length, + long CreationTimeUtcTicks, + long LastWriteTimeUtcTicks); private sealed class QueryOnlySnapshotSourceChangedException(Exception innerException) : IOException("The query-only snapshot source disappeared while it was being copied.", innerException); diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index 0c0f41b9f..4726abb34 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -361,6 +361,8 @@ internal static Func? ReadMigrationTransact internal bool ImmutableReadOnlyWalRisk => _immutableReadOnlyWalRisk; internal bool ConnectionPooling => _connectionPooling; internal bool QueryOnlySnapshotRequiresRefresh => _queryOnlySnapshotRequiresRefresh; + internal DbConnectionFactory.QueryOnlySnapshotSourceState? QueryOnlySnapshotSourceState + => _queryOnlySnapshotSourceState; internal bool IsQueryOnlySnapshotCurrent(CancellationToken cancellationToken = default) => !_queryOnlySnapshotRequiresRefresh || (_queryOnlySnapshotSourcePath != null diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs index 847abb908..08ddae46c 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4872Tests.cs @@ -1,5 +1,5 @@ -using System.Diagnostics; using System.Collections.Concurrent; +using System.Diagnostics; using CodeIndex.Cli; using CodeIndex.Database; using Microsoft.Data.Sqlite; @@ -108,10 +108,13 @@ public async Task RunBatch_ParallelRefreshesDetachedSnapshotsBetweenItems_Issue4 PRAGMA wal_autocheckpoint=0; INSERT INTO files(path, lang, size, lines, checksum, modified) VALUES ('src/Initial.cs', 'csharp', 1, 1, 'initial', CURRENT_TIMESTAMP); + PRAGMA wal_checkpoint(TRUNCATE); + UPDATE files SET checksum = 'initial-hot' WHERE path = 'src/Initial.cs'; """; setup.ExecuteNonQuery(); } Assert.True(new FileInfo(dbPath + "-wal").Length > 0); + var initialDbLength = new FileInfo(dbPath).Length; using var input = new InteractiveBatchTextReader(); using var stdout = new StringWriter(); @@ -119,6 +122,129 @@ INSERT INTO files(path, lang, size, lines, checksum, modified) using var firstWaveCompleted = new CountdownEvent(2); using var cancellation = new CancellationTokenSource(); Task? runTask = null; + var openedSessions = 0; + QueryCommandRunner.BatchParallelSessionOpenedForTesting = + () => Interlocked.Increment(ref openedSessions); + QueryCommandRunner.BatchParallelCommandCompletedForTesting = lineNumber => + { + if (lineNumber <= 2) + firstWaveCompleted.Signal(); + }; + + try + { + runTask = Task.Run(() => + { + using var capture = ConsoleCapture.Start(stdout, stderr, input); + return QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "2"], + _jsonOptions, + cancellationToken: cancellation.Token); + }); + input.WriteLine("""{"command":"status","args":["--json"]}"""); + input.WriteLine("""{"command":"status","args":["--json"]}"""); + Assert.True( + firstWaveCompleted.Wait(TimeSpan.FromSeconds(60)), + "The first parallel batch wave did not complete."); + + using (var update = writer.CreateCommand()) + { + update.CommandText = """ + INSERT INTO files(path, lang, size, lines, checksum, modified) + VALUES ('src/Updated.cs', 'csharp', 1, 1, 'updated', CURRENT_TIMESTAMP); + PRAGMA wal_checkpoint(TRUNCATE); + """; + update.ExecuteNonQuery(); + } + Assert.Equal(initialDbLength, new FileInfo(dbPath).Length); + Assert.Equal(0, new FileInfo(dbPath + "-wal").Length); + + input.WriteLine("""{"command":"status","args":["--json"]}"""); + input.WriteLine("""{"command":"status","args":["--json"]}"""); + input.Complete(); + + var exitCode = await runTask.WaitAsync(TimeSpan.FromSeconds(60)); + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr.ToString()); + + var lines = ParseJsonLines(stdout.ToString()); + try + { + Assert.Equal(5, lines.Count); + Assert.Equal(1, lines[0].RootElement.GetProperty("result").GetProperty("files").GetInt32()); + Assert.Equal(1, lines[1].RootElement.GetProperty("result").GetProperty("files").GetInt32()); + Assert.Equal(2, lines[2].RootElement.GetProperty("result").GetProperty("files").GetInt32()); + Assert.Equal(2, lines[3].RootElement.GetProperty("result").GetProperty("files").GetInt32()); + Assert.Equal(4, Volatile.Read(ref openedSessions)); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + finally + { + QueryCommandRunner.BatchParallelSessionOpenedForTesting = null; + QueryCommandRunner.BatchParallelCommandCompletedForTesting = null; + input.Complete(); + if (runTask is { IsCompleted: false }) + { + cancellation.Cancel(); + try + { + await runTask.WaitAsync(TimeSpan.FromSeconds(15)); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + } + } + } + } + + [Fact] + public async Task RunBatch_ParallelRefreshesDirectSessionsBetweenItems_Issue4872() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_direct_refresh_4872"); + var sourceDbPath = TestProjectHelper.CreateProjectDb(project.Root); + using (var source = new SqliteConnection( + $"Data Source={sourceDbPath};Mode=ReadWrite;Pooling=False")) + { + source.Open(); + using var checkpoint = source.CreateCommand(); + checkpoint.CommandText = "PRAGMA wal_checkpoint(TRUNCATE)"; + checkpoint.ExecuteNonQuery(); + } + + var dbPath = Path.Combine(project.Root, "direct.db"); + File.Copy(sourceDbPath, dbPath); + using var writer = new SqliteConnection( + $"Data Source={dbPath};Mode=ReadWrite;Pooling=False"); + writer.Open(); + using (var journalMode = writer.CreateCommand()) + { + journalMode.CommandText = "PRAGMA journal_mode=DELETE"; + Assert.Equal("delete", Assert.IsType(journalMode.ExecuteScalar())); + } + using (var setup = writer.CreateCommand()) + { + setup.CommandText = """ + INSERT INTO files(path, lang, size, lines, checksum, modified) + VALUES ('src/Initial.cs', 'csharp', 1, 1, 'initial', CURRENT_TIMESTAMP); + """; + setup.ExecuteNonQuery(); + } + Assert.Equal("delete", ReadJournalMode(writer)); + + using var input = new InteractiveBatchTextReader(); + using var stdout = new StringWriter(); + using var stderr = new StringWriter(); + using var firstWaveCompleted = new CountdownEvent(2); + using var cancellation = new CancellationTokenSource(); + Task? runTask = null; + var openedSessions = 0; + QueryCommandRunner.BatchParallelSessionOpenedForTesting = + () => Interlocked.Increment(ref openedSessions); QueryCommandRunner.BatchParallelCommandCompletedForTesting = lineNumber => { if (lineNumber <= 2) @@ -166,6 +292,7 @@ INSERT INTO files(path, lang, size, lines, checksum, modified) Assert.Equal(1, lines[1].RootElement.GetProperty("result").GetProperty("files").GetInt32()); Assert.Equal(2, lines[2].RootElement.GetProperty("result").GetProperty("files").GetInt32()); Assert.Equal(2, lines[3].RootElement.GetProperty("result").GetProperty("files").GetInt32()); + Assert.Equal(4, Volatile.Read(ref openedSessions)); } finally { @@ -175,6 +302,7 @@ INSERT INTO files(path, lang, size, lines, checksum, modified) } finally { + QueryCommandRunner.BatchParallelSessionOpenedForTesting = null; QueryCommandRunner.BatchParallelCommandCompletedForTesting = null; input.Complete(); if (runTask is { IsCompleted: false }) @@ -191,6 +319,50 @@ INSERT INTO files(path, lang, size, lines, checksum, modified) } } + [Fact] + public void RunBatch_InitialValidationFailuresDisposeDetachedSnapshots_Issue4872() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_validation_failure_4872"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var writer = new SqliteConnection( + $"Data Source={dbPath};Mode=ReadWrite;Pooling=False"); + writer.Open(); + using (var setup = writer.CreateCommand()) + { + setup.CommandText = """ + PRAGMA journal_mode=WAL; + PRAGMA wal_autocheckpoint=0; + INSERT INTO files(path, lang, size, lines, checksum, modified) + VALUES ('src/Initial.cs', 'csharp', 1, 1, 'initial', CURRENT_TIMESTAMP); + """; + setup.ExecuteNonQuery(); + } + + var snapshotDirectories = new ConcurrentBag(); + var originalDirectoryHook = DbConnectionFactory.QueryOnlySnapshotDirectoryCreatedForTesting; + var originalValidationHook = QueryCommandRunner.BatchParallelDatabaseValidatingForTesting; + DbConnectionFactory.QueryOnlySnapshotDirectoryCreatedForTesting = snapshotDirectories.Add; + QueryCommandRunner.BatchParallelDatabaseValidatingForTesting = + () => throw new InvalidDataException("Injected parallel batch validation failure."); + + try + { + Assert.Throws(() => CaptureConsoleWithInput( + BuildIssue4872BatchInput(1), + () => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "2"], + _jsonOptions))); + } + finally + { + QueryCommandRunner.BatchParallelDatabaseValidatingForTesting = originalValidationHook; + DbConnectionFactory.QueryOnlySnapshotDirectoryCreatedForTesting = originalDirectoryHook; + } + + Assert.NotEmpty(snapshotDirectories); + Assert.All(snapshotDirectories, path => Assert.False(Directory.Exists(path), path)); + } + [Fact] public void RunBatch_ReaderConstructionFailuresDisposeDetachedSnapshots_Issue4872() { @@ -349,4 +521,11 @@ private static string BuildIssue4872RejectedBatchInput(int commandCount) Enumerable.Repeat( """{"command":"unknown"}""", commandCount)) + "\n"; + + private static string ReadJournalMode(SqliteConnection connection) + { + using var command = connection.CreateCommand(); + command.CommandText = "PRAGMA journal_mode"; + return Assert.IsType(command.ExecuteScalar()); + } }