From 35b1749c4b4c4543f46270290e74be84921108be Mon Sep 17 00:00:00 2001 From: Widthdom Date: Wed, 29 Jul 2026 08:49:19 +0900 Subject: [PATCH 1/3] Standardize batch error envelopes (#4871) --- DEVELOPER_GUIDE.md | 35 +- TESTING_GUIDE.md | 4 +- changelog.d/unreleased/4871.fixed.md | 22 + src/CodeIndex/Cli/CliFlagSchema.cs | 1 + src/CodeIndex/Cli/ConsoleUi.cs | 2 +- src/CodeIndex/Cli/QueryCommandRunner.Batch.cs | 412 ++++++++++++++---- .../QueryCommandRunnerBatchIssue4723Tests.cs | 35 +- .../QueryCommandRunnerBatchIssue4871Tests.cs | 244 +++++++++++ .../QueryCommandRunnerTests.cs | 24 +- 9 files changed, 665 insertions(+), 114 deletions(-) create mode 100644 changelog.d/unreleased/4871.fixed.md create mode 100644 tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4871Tests.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 97fb15b52..c0a058b36 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -419,13 +419,17 @@ By default, child query commands stream their normal stdout/stderr directly. In `--json-summary` mode, every non-blank stdin line must instead emit one machine-readable batch envelope before the final summary: parsed commands use `record: "batch_result"` and include `line`, `command`, `arguments`, -`exit_code`, and captured child `stderr`. The requested command/output format, -rather than output-text sniffing, selects the projection: successful +and `exit_code`. The requested command/output format, rather than output-text +sniffing, selects the projection: successful single-document JSON is embedded as typed `result`, while successful NDJSON is -embedded as a stable typed `results` array even when it has one row. Text and failed -commands remain raw `stdout` text so diagnostics are not lost. Malformed or -over-limit input lines use `record: "batch_error"` and an `error` object. Child -output must not be written directly beside batch metadata in this mode. The entire +embedded as a stable typed `results` array even when it has one row. Successful +text remains `stdout`, while every failure uses one typed `error` object with a +stable `error_code`, `category`, safe `message` / `hint`, and `scope`. +Malformed or over-limit input lines use `record: "batch_error"` and the same +typed error serializer. Failed records omit captured child stdout/stderr by +default; `--include-raw-streams` explicitly adds them under a bounded +`raw_streams` object. Child output must not be written directly beside batch +metadata in this mode. The entire serialized stream—including envelopes, arguments, escaping expansion, terminal errors, and the final summary—uses the configured `--max-output-chars` budget (default 10,485,760; maximum 67,108,864). An item that exhausts it retains its @@ -439,8 +443,9 @@ SQLite connection and thread-local batch reader, and buffer only the active worker window. `ScopedConsoleOutput` keeps nested JSON-envelope capture on the current worker's routed stdout instead of replacing another worker's process-wide writer. Completed records are committed to the shared output writer in input -order; an ordinary item failure remains isolated, while caller cancellation -stops scheduling and propagates. +order; an ordinary item failure remains isolated. Caller cancellation is +serialized as `batch_cancelled` for a consumed input item and in the final +summary before batch processing stops. Editor integrations can request standard location shapes directly. `definition`, `references`, `search`, `find`, and `validate` accept `--format `; `lsp` emits LSP `Location` arrays, `qf` emits Vim quickfix lines, and `sarif` emits SARIF 2.1.0. `goto ` returns the single unambiguous definition as one LSP `Location`, while `goto --all ` returns all matching locations without applying the default or environment-provided query limit. An explicit `--limit` or `--top` still bounds the returned location array. @@ -3688,11 +3693,14 @@ path filter を受け付ける query コマンド（`search`, `definition`, `ref 既定では child query command の通常の stdout / stderr を直接 stream する。`--json-summary` mode では、空白でない stdin 行ごとに final summary より前へ 1 つの machine-readable batch envelope を出力しなければならない。parse 済み command は `record: "batch_result"` として -`line`、`command`、`arguments`、`exit_code`、捕捉した child `stderr` を含める。output text の -推測ではなく requested command / output format で projection を選び、成功した単一 document JSON +`line`、`command`、`arguments`、`exit_code` を含める。output text の推測ではなく requested +command / output format で projection を選び、成功した単一 document JSON は型付き `result`、NDJSON は 1 row の場合も安定した型付き `results` array として埋め込む。 -text と失敗 command の出力は診断を失わないよう raw `stdout` text のまま -保持する。malformed line や入力上限超過 line は `record: "batch_error"` と `error` object を使う。 +成功した text command は `stdout` のまま保持する一方、すべての失敗は安定した `error_code`、 +`category`、安全な `message` / `hint`、`scope` を持つ共通の型付き `error` object を使う。 +malformed line や入力上限超過 line は `record: "batch_error"` と同じ typed error serializer を +使う。失敗 record は既定で捕捉した child stdout / stderr を省略し、 +`--include-raw-streams` を明示した場合だけ上限付きの `raw_streams` object に追加する。 この mode では child output を batch metadata と並べて直接出力してはならない。envelope、 arguments、escape 展開、terminal error、final summary を含む serialized stream 全体には 設定された `--max-output-chars` budget（既定 10,485,760、最大 67,108,864）を適用し、 @@ -3705,7 +3713,8 @@ command ごとの bounded writer へ route し、分離した read-only SQLite c batch reader を使い、active worker window だけを buffer する。`ScopedConsoleOutput` は nested JSON-envelope capture を現在の worker の routed stdout に保ち、他 worker の process-wide writer を 置き換えない。完了 record は入力順で共有 output writer へ commit する。通常の item failure は -他 item から隔離し、caller cancellation は scheduling を停止して伝播する。 +他 item から隔離する。caller cancellation は、消費済み input item と final summary に +`batch_cancelled` を記録してから後続処理を停止する。 editor integration は標準的な location 形状を直接要求できる。`definition`、`references`、`search`、`find`、`validate` は `--format ` を受け付け、`lsp` は LSP `Location` 配列、`qf` は Vim quickfix 行、`sarif` は SARIF 2.1.0 を出力する。`goto ` は曖昧でない単一定義を 1 つの LSP `Location` として返し、`goto --all ` は既定または環境変数由来の query limit を適用せず、一致する全 location を返す。明示的な `--limit` または `--top` を指定した場合は location 配列をその件数に制限する。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 568ae9e83..485694d29 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -193,7 +193,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Doctor full-inventory coverage keeps composed filter selection, filtered summary counts, exact UTF-8 byte-budget boundaries, and structured overflow errors together in `ProgramRunnerTests`; license JSON remains a subprocess contract in `ProgramCliTests` so immediate-command dispatch and the published field names are both exercised. Ctags export JSON coverage reuses one seeded database for default and `--include-generated` variants, asserts the fixed skip-reason keys sum to `skipped_count`, and keeps the missing-`files.generated` degradation in a separate legacy-schema fixture. Dry-run JSON coverage for ambiguous `.h` files locks the bounded `language_detections` entries and their stable source/confidence codes without mutating the index. - Batch `--json-summary` coverage keeps side-effect-free dispatch plus successful JSON, single-row NDJSON, and JSON-looking text projection in one seeded fixture when the assertions stay distinct; include case-insensitive format values, command-specific JSON formats, and alias-injected JSON defaults so batch classification cannot drift from child parsers. Serialized-output exhaustion remains a separate boundary fixture because of its large escaped payload. Assert `result` / `results` versus raw `stdout` presence explicitly, compare `output_chars` with the actual captured stream length, and cover repeated malformed lines up to the input cap so neither syntax sniffing nor envelope overhead can bypass the transport bounds. + Batch `--json-summary` coverage keeps side-effect-free dispatch plus successful JSON, single-row NDJSON, and JSON-looking text projection in one seeded fixture when the assertions stay distinct; include case-insensitive format values, command-specific JSON formats, and alias-injected JSON defaults so batch classification cannot drift from child parsers. Serialized-output exhaustion remains a separate boundary fixture because of its large escaped payload. Assert `result` / `results` versus raw `stdout` presence explicitly, compare `output_chars` with the actual captured stream length, and cover repeated malformed lines up to the input cap so neither syntax sniffing nor envelope overhead can bypass the transport bounds. Typed-failure coverage reuses one mixed scalar/array/schema/policy/child-failure input across serial and parallel modes, asserts stable ordered categories, and keeps timeout, caller cancellation, and explicit `--include-raw-streams` behavior in focused fixtures. Argument-validation variants that only differ by invalid scalar input share one database fixture and iterate within a fact when no per-case state or discovery identity is required. Positional `files` glob coverage shares one indexed-file fixture and iterates `*`, `?`, and recursive `**` patterns in a fact, matching the exact tokens that a quoted shell argument passes to the CLI. Excerpt focus coverage reuses one indexed fixture for line-only leading-window behavior, the focus-length dependency, and focus-column range validation; zero and non-numeric focus-column values share one indexed Markdown fixture. @@ -1112,7 +1112,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" doctor full-inventory coverage では、合成 filter の選択、filtered summary 件数、UTF-8 byte budget の exact boundary、structured overflow error を `ProgramRunnerTests` にまとめます。license JSON は `ProgramCliTests` の subprocess contract として、immediate-command dispatch と公開 field 名を同時に検証します。 ctags export JSON coverage は1つの seeded database を既定と `--include-generated` variant で再利用し、固定された skip-reason key の合計が `skipped_count` と一致することを検証します。`files.generated` がない場合の縮退は別の legacy-schema fixture に保ってください。 曖昧な `.h` に対する dry-run JSON coverage は、index を変更せず、上限付き `language_detections` entry と安定した判定元・信頼度 code を固定します。 - batch `--json-summary` coverage は、assertion を明確に保てる場合、副作用なし dispatch と成功時の JSON / 1 row NDJSON / JSON に見える text projection を1つの seeded fixture で共有し、case-insensitive な format 値、command 固有の JSON format、alias が内部追加する JSON default も含めて child parser と batch classification の drift を防いでください。serialized output の枯渇は escape を多く含む大きな payload の境界 fixture として分離します。test 側で `result` / `results` と raw `stdout` の有無を明示し、`output_chars` を実際に捕捉した stream 長と比較し、input cap までの malformed line 反復も検証して、syntax sniffing や envelope overhead が transport 上限を迂回しないようにしてください。 + batch `--json-summary` coverage は、assertion を明確に保てる場合、副作用なし dispatch と成功時の JSON / 1 row NDJSON / JSON に見える text projection を1つの seeded fixture で共有し、case-insensitive な format 値、command 固有の JSON format、alias が内部追加する JSON default も含めて child parser と batch classification の drift を防いでください。serialized output の枯渇は escape を多く含む大きな payload の境界 fixture として分離します。test 側で `result` / `results` と raw `stdout` の有無を明示し、`output_chars` を実際に捕捉した stream 長と比較し、input cap までの malformed line 反復も検証して、syntax sniffing や envelope overhead が transport 上限を迂回しないようにしてください。typed failure coverage は scalar / array / schema / policy / child failure を混在させた1つの input を serial / parallel mode で共有し、安定した順序と category を検証してください。timeout、caller cancellation、明示的な `--include-raw-streams` の挙動は焦点を絞った fixture に分離します。 invalid scalar input だけが異なる argument-validation variant は、case ごとの state や discovery identity が不要なら1つの database fixture を共有し、fact 内で反復してください。 `files` の positional glob coverage は1つの indexed-file fixture を共有し、quote された shell 引数が CLI に渡す token と同じ `*`、`?`、recursive `**` pattern を fact 内で反復してください。 excerpt の focus coverage は、line-only 時の先頭側 window、focus-length の依存関係、focus-column の範囲検証を1つの indexed fixture で共有してください。focus-column の zero / non-numeric value も1つの indexed Markdown fixture を再利用してください。 diff --git a/changelog.d/unreleased/4871.fixed.md b/changelog.d/unreleased/4871.fixed.md new file mode 100644 index 000000000..f89510e4a --- /dev/null +++ b/changelog.d/unreleased/4871.fixed.md @@ -0,0 +1,22 @@ +--- +category: fixed +issues: + - 4871 +affected: + - src/CodeIndex/Cli/CliFlagSchema.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - src/CodeIndex/Cli/QueryCommandRunner.Batch.cs + - tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4723Tests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4871Tests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Standardized typed error envelopes for every batch failure (#4871)** — `batch --json-summary` now gives malformed input, policy rejection, child failure, timeout, and cancellation the same stable error-code/category contract in serial and parallel modes, while bounded raw child streams require explicit `--include-raw-streams`. + +## 日本語 + +- **すべての batch failure で型付き error envelope を統一しました (#4871)** — `batch --json-summary` は malformed input、policy rejection、child failure、timeout、cancellation に対して serial / parallel mode 共通の安定した error-code / category 契約を返し、上限付きの raw child stream は `--include-raw-streams` を明示した場合だけ含めるようになりました。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 83e45b56e..8aa49d5f8 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -403,6 +403,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--data-dir", ValuePlaceholder = "", Description = "Directory containing codeindex.db; overrides CDIDX_DATA_DIR/XDG/workspace defaults", PrimaryCommands = Set(DataDirCommands), Safety = CliOptionSafety.Scope }, new() { Name = "--json", Description = "JSON output; search/symbols/files/validate also accept --json=array for a single JSON array", PrimaryCommands = Set(JsonCommands.Concat(["hooks"]).ToArray()) }, new() { Name = "--json-summary", Description = "Batch: emit one typed result/error record per input plus a final summary", PrimaryCommands = Set("batch") }, + new() { Name = "--include-raw-streams", Description = "Batch JSON-summary: attach bounded child stdout/stderr to failed records", PrimaryCommands = Set("batch") }, new() { Name = "--max-input-lines", ValuePlaceholder = "", Description = $"Batch: input-line budget (default {QueryCommandRunner.BatchDefaultInputLines}, max {QueryCommandRunner.BatchMaxInputLines})", PrimaryCommands = Set("batch") }, new() { Name = "--max-output-chars", ValuePlaceholder = "", Description = $"Batch JSON-summary output budget (default {QueryCommandRunner.BatchDefaultTotalOutputChars}, max {QueryCommandRunner.BatchMaxTotalOutputChars})", PrimaryCommands = Set("batch") }, new() { Name = "--parallel", ValuePlaceholder = "", Description = $"Batch JSON-summary worker count (default 1, max {QueryCommandRunner.BatchMaxParallelism}); results retain input order", PrimaryCommands = Set("batch") }, diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index 7fd297c57..426896d71 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -146,7 +146,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = ("export-ctags", "cdidx export ctags [--output ] [--db ] [--json] [--lang ] [--path ] [--exclude-path ] [--exclude-tests] [--include-generated]"), ("import", "cdidx import [--db ] [--prune-paths] [--no-backup] [--dry-run|--check] [--limit ] [--offset ] [--json]"), ("languages", "cdidx languages [--db ] [--json] [--format ] [--summary-only] [--limit |--top ] [--cursor ] [--max-json-bytes ] [--indexed-only] [--language |--extension |--alias ] [--capability ]"), - ("batch", "cdidx batch [--db ] [--json-summary] [--max-input-lines ] [--max-output-chars ] [--parallel ] # stdin is JSON Lines; --json-summary embeds typed child JSON plus a final summary"), + ("batch", "cdidx batch [--db ] [--json-summary] [--include-raw-streams] [--max-input-lines ] [--max-output-chars ] [--parallel ] # stdin is JSON Lines; --json-summary embeds typed child JSON plus a final summary"), ("hooks-install", "cdidx hooks install [--project ] [--force] [--dry-run] [--json]"), ("hooks-uninstall", "cdidx hooks uninstall [--project ] [--force] [--json]"), ("hooks-status", "cdidx hooks status [--project ] [--json]"), diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs index 31856049f..b4732c8d7 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs @@ -28,6 +28,7 @@ public static int RunBatch( var maxOutputCharsSpecified = false; var parallelism = 1; var parallelismSpecified = false; + var includeRawStreams = false; for (var i = 0; i < cmdArgs.Length; i++) { var arg = cmdArgs[i]; @@ -37,6 +38,12 @@ public static int RunBatch( continue; } + if (arg == "--include-raw-streams") + { + includeRawStreams = true; + continue; + } + if (arg == "--db") { if (i + 1 >= cmdArgs.Length || string.IsNullOrWhiteSpace(cmdArgs[i + 1])) @@ -128,6 +135,12 @@ public static int RunBatch( CommandErrorWriter.WriteStderr($"Usage: {ConsoleUi.GetUsageLine("batch")}"); return CommandExitCodes.UsageError; } + if (includeRawStreams && !jsonSummary) + { + CommandErrorWriter.WriteStderr("Error: --include-raw-streams requires --json-summary."); + CommandErrorWriter.WriteStderr($"Usage: {ConsoleUi.GetUsageLine("batch")}"); + return CommandExitCodes.UsageError; + } var isUri = dbPath.StartsWith("file:", StringComparison.OrdinalIgnoreCase); if (!isUri && !File.Exists(dbPath)) @@ -151,6 +164,7 @@ public static int RunBatch( maxInputLines, maxOutputChars, parallelism, + includeRawStreams, jsonOptions, appVersion, cancellationToken); @@ -181,7 +195,6 @@ public static int RunBatch( var inputLimitReached = false; while (TryReadBatchLine(Console.In, out var line, out var lineExceededLimit)) { - cancellationToken.ThrowIfCancellationRequested(); lineNumber++; if (lineNumber > maxInputLines) { @@ -207,7 +220,9 @@ public static int RunBatch( var lineError = new BatchLineError( $"batch line {lineNumber} exceeds the {BatchMaxLineChars} character limit.", CommandExitCodes.UsageError, - ErrorCode: CommandErrorCodes.UsageError); + Hint: "Split the command across smaller arguments or reduce the input record.", + ErrorCode: CommandErrorCodes.UsageError, + Category: "batch_input_line_length_limit"); if (jsonSummary) { if (!WriteBatchLineErrorJson(lineNumber, lineError, jsonOutput!)) @@ -234,6 +249,28 @@ public static int RunBatch( if (string.IsNullOrWhiteSpace(line)) continue; + if (cancellationToken.IsCancellationRequested) + { + if (!jsonSummary) + cancellationToken.ThrowIfCancellationRequested(); + + var lineError = BuildBatchCancellationLineError(lineNumber); + if (!WriteBatchLineErrorJson(lineNumber, lineError, jsonOutput!)) + { + WriteBatchOutputLimitErrorJson( + lineNumber, + commandName: null, + CommandExitCodes.CancelledBySignal, + maxOutputChars, + jsonOutput!); + outputLimitReached = true; + } + lineErrors++; + if (firstFailure == CommandExitCodes.Success) + firstFailure = CommandExitCodes.CancelledBySignal; + break; + } + if (!TryParseBatchLine(line, lineNumber, jsonOptions, !jsonSummary, out var commandName, out var subArgs, out var parseExitCode, out var parseError)) { if (jsonSummary) @@ -267,13 +304,15 @@ public static int RunBatch( commandName, subArgs, maxOutputChars, + includeRawStreams, jsonOutput!, jsonOptions, appVersion, cancellationToken) : new BatchCommandRunResult( RunBatchQueryCommand(commandName, subArgs, jsonOptions, appVersion, cancellationToken), - OutputLimitReached: false); + OutputLimitReached: false, + CancellationObserved: false); var exitCode = batchResult.ExitCode; if (exitCode != CommandExitCodes.Success) { @@ -286,6 +325,8 @@ public static int RunBatch( outputLimitReached = true; break; } + if (batchResult.CancellationObserved) + break; } if (jsonSummary) @@ -360,6 +401,7 @@ private static int RunBatchParallel( int maxInputLines, int maxOutputChars, int parallelism, + bool includeRawStreams, JsonSerializerOptions jsonOptions, string appVersion, CancellationToken cancellationToken) @@ -381,10 +423,8 @@ private static int RunBatchParallel( var commandFailures = 0; var outputLimitReached = false; var inputLimitReached = false; + var cancellationObserved = false; using var stopProducing = new CancellationTokenSource(); - using var producerCancellation = CancellationTokenSource.CreateLinkedTokenSource( - cancellationToken, - stopProducing.Token); var input = Channel.CreateBounded( new BoundedChannelOptions(1) { @@ -403,11 +443,14 @@ private static int RunBatchParallel( { while (!stopProducing.IsCancellationRequested) { - cancellationToken.ThrowIfCancellationRequested(); + if (cancellationToken.IsCancellationRequested) + { + cancellationObserved = true; + break; + } if (!TryReadBatchLine(Console.In, out var line, out var lineExceededLimit)) break; - cancellationToken.ThrowIfCancellationRequested(); lineNumber++; if (lineNumber > maxInputLines) { @@ -419,7 +462,7 @@ private static int RunBatchParallel( Category: "batch_input_line_limit"); input.Writer.WriteAsync( new BatchPendingItem(lineNumber, null, [], lineError, Terminal: true), - producerCancellation.Token) + stopProducing.Token) .AsTask() .GetAwaiter() .GetResult(); @@ -433,10 +476,12 @@ private static int RunBatchParallel( var lineError = new BatchLineError( $"batch line {lineNumber} exceeds the {BatchMaxLineChars} character limit.", CommandExitCodes.UsageError, - ErrorCode: CommandErrorCodes.UsageError); + Hint: "Split the command across smaller arguments or reduce the input record.", + ErrorCode: CommandErrorCodes.UsageError, + Category: "batch_input_line_length_limit"); input.Writer.WriteAsync( new BatchPendingItem(lineNumber, null, [], lineError, Terminal: false), - producerCancellation.Token) + stopProducing.Token) .AsTask() .GetAwaiter() .GetResult(); @@ -447,6 +492,24 @@ private static int RunBatchParallel( if (string.IsNullOrWhiteSpace(line)) continue; + if (cancellationToken.IsCancellationRequested) + { + cancellationObserved = true; + input.Writer.WriteAsync( + new BatchPendingItem( + lineNumber, + null, + [], + BuildBatchCancellationLineError(lineNumber), + Terminal: true), + stopProducing.Token) + .AsTask() + .GetAwaiter() + .GetResult(); + lineErrors++; + break; + } + BatchPendingItem item; if (!TryParseBatchLine( line, @@ -477,7 +540,7 @@ private static int RunBatchParallel( commandsProcessed++; } - input.Writer.WriteAsync(item, producerCancellation.Token) + input.Writer.WriteAsync(item, stopProducing.Token) .AsTask() .GetAwaiter() .GetResult(); @@ -514,8 +577,7 @@ private static int RunBatchParallel( stderrRouter, jsonOptions, appVersion, - cancellationToken), - cancellationToken); + cancellationToken)); active.Enqueue((item, result)); } @@ -565,6 +627,7 @@ private static int RunBatchParallel( result.Stderr, result.Error, ClassifyBatchOutput(item.CommandName!, item.Arguments), + includeRawStreams, jsonOutput)) { if (result.ExitCode != CommandExitCodes.Success) @@ -596,8 +659,7 @@ private static int RunBatchParallel( if (active.Count > 0 && active.Peek().Result.IsCompleted) continue; - using var waitCancellation = CancellationTokenSource.CreateLinkedTokenSource( - cancellationToken); + using var waitCancellation = new CancellationTokenSource(); var waitForInput = input.Reader.WaitToReadAsync(waitCancellation.Token).AsTask(); if (active.Count == 0) { @@ -676,6 +738,9 @@ private static int RunBatchParallel( { } + if (cancellationObserved && firstFailure == CommandExitCodes.Success) + firstFailure = CommandExitCodes.CancelledBySignal; + WriteBatchSummaryJson( lineNumber, commandsProcessed, @@ -741,20 +806,26 @@ private static BatchParallelCommandResult RunBatchParallelCommand( } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - throw; + exitCode = CommandExitCodes.CancelledBySignal; + error = BuildBatchCancellationError(); + } + catch (TimeoutException) + { + exitCode = CommandExitCodes.RuntimeError; + error = BuildBatchTimeoutError(); } catch (Exception ex) { - exitCode = CommandExitCodes.DatabaseError; - error = new JsonObject - { - ["message"] = "batch command failed without affecting other batch items.", - ["error_code"] = CommandErrorCodes.DbError, - ["category"] = SafeDiagnosticFormatter.FormatCategoryType( + exitCode = CommandExitCodes.RuntimeError; + error = BuildBatchTypedError( + "batch command failed without affecting other batch items.", + exitCode, + "Retry the item directly if command-specific diagnostics are required.", + CommandErrorCodes.CommandFailed, + SafeDiagnosticFormatter.FormatCategoryType( "batch_command_failure", ex.GetType().Name), - ["scope"] = "command", - }; + "command"); } finally { @@ -806,6 +877,7 @@ private static BatchCommandRunResult RunBatchQueryCommandWithJsonRecord( string commandName, string[] subArgs, int outputCharLimit, + bool includeRawStreams, BatchJsonOutputWriter output, JsonSerializerOptions jsonOptions, string appVersion, @@ -814,6 +886,8 @@ private static BatchCommandRunResult RunBatchQueryCommandWithJsonRecord( using var capture = new BatchCommandOutputCapture(); int exitCode; BatchOutputCaptureLimitExceededException? captureLimitExceeded = null; + JsonObject? commandError = null; + var cancellationObserved = false; try { capture.Start(); @@ -824,14 +898,37 @@ private static BatchCommandRunResult RunBatchQueryCommandWithJsonRecord( captureLimitExceeded = ex; exitCode = CommandExitCodes.InvalidArgument; } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + exitCode = CommandExitCodes.CancelledBySignal; + commandError = BuildBatchCancellationError(); + cancellationObserved = true; + } + catch (TimeoutException) + { + exitCode = CommandExitCodes.RuntimeError; + commandError = BuildBatchTimeoutError(); + } + catch (Exception ex) + { + exitCode = CommandExitCodes.RuntimeError; + commandError = BuildBatchTypedError( + "batch command failed without affecting other batch items.", + exitCode, + "Retry the item directly if command-specific diagnostics are required.", + CommandErrorCodes.CommandFailed, + SafeDiagnosticFormatter.FormatCategoryType( + "batch_command_failure", + ex.GetType().Name), + "command"); + } finally { capture.Stop(); } - JsonObject? error = null; if (captureLimitExceeded is not null) - error = BuildBatchCaptureLimitError(captureLimitExceeded); + commandError = BuildBatchCaptureLimitError(captureLimitExceeded); var recordWritten = WriteBatchCommandRecordJson( lineNumber, @@ -840,27 +937,97 @@ private static BatchCommandRunResult RunBatchQueryCommandWithJsonRecord( exitCode, capture.Stdout, capture.Stderr, - error, + commandError, ClassifyBatchOutput(commandName, subArgs), + includeRawStreams, output); if (recordWritten) - return new BatchCommandRunResult(exitCode, OutputLimitReached: false); + return new BatchCommandRunResult( + exitCode, + OutputLimitReached: false, + CancellationObserved: cancellationObserved); WriteBatchOutputLimitErrorJson(lineNumber, commandName, exitCode, outputCharLimit, output); - return new BatchCommandRunResult(CommandExitCodes.InvalidArgument, OutputLimitReached: true); + return new BatchCommandRunResult( + CommandExitCodes.InvalidArgument, + OutputLimitReached: true, + CancellationObserved: cancellationObserved); } private static JsonObject BuildBatchCaptureLimitError(BatchOutputCaptureLimitExceededException exception) { - return new JsonObject + return BuildBatchTypedError( + $"batch command {exception.StreamName} exceeded {exception.MaxChars} captured characters.", + CommandExitCodes.InvalidArgument, + "Reduce the result set or run cdidx batch without --json-summary for streaming output.", + CommandErrorCodes.UsageError, + "batch_child_output_limit", + "command", + new JsonObject + { + ["max_chars"] = exception.MaxChars, + ["stream"] = exception.StreamName, + }); + } + + private static JsonObject BuildBatchCancellationError() + => BuildBatchTypedError( + "batch command was cancelled by the caller.", + CommandExitCodes.CancelledBySignal, + "Retry the batch when the caller cancellation token is not cancelled.", + CommandErrorCodes.Interrupted, + "batch_cancelled", + "command"); + + private static BatchLineError BuildBatchCancellationLineError(int lineNumber) + => new( + $"batch line {lineNumber} was not dispatched because the caller cancelled the batch.", + CommandExitCodes.CancelledBySignal, + Hint: "Retry the batch when the caller cancellation token is not cancelled.", + ErrorCode: CommandErrorCodes.Interrupted, + Category: "batch_cancelled"); + + private static JsonObject BuildBatchTimeoutError() + => BuildBatchTypedError( + "batch command exceeded its execution deadline.", + CommandExitCodes.RuntimeError, + "Reduce the query scope or retry the command directly.", + CommandErrorCodes.CommandFailed, + "batch_command_timeout", + "command"); + + private static JsonObject BuildBatchTypedError( + string message, + int exitCode, + string? hint, + string? errorCode, + string? category, + string scope, + JsonObject? additionalProperties = null) + { + var (resolvedErrorCode, resolvedCategory) = CommandErrorWriter.ResolveMachineContract( + exitCode, + errorCode, + category); + var payload = new JsonObject { - ["message"] = $"batch command {exception.StreamName} exceeded {exception.MaxChars} captured characters.", - ["hint"] = "Reduce the result set or run cdidx batch without --json-summary for streaming output.", - ["error_code"] = CommandErrorCodes.UsageError, - ["max_chars"] = exception.MaxChars, - ["stream"] = exception.StreamName, - ["scope"] = "command", + ["message"] = message, + ["hint"] = hint ?? BatchFailureHint(exitCode), + ["error_code"] = resolvedErrorCode, + ["category"] = resolvedCategory, + ["scope"] = scope, }; + + if (additionalProperties is not null) + { + foreach (var property in additionalProperties) + { + if (!payload.ContainsKey(property.Key)) + payload[property.Key] = property.Value?.DeepClone(); + } + } + + return payload; } private static bool WriteBatchCommandRecordJson( @@ -872,6 +1039,7 @@ private static bool WriteBatchCommandRecordJson( string stderr, JsonObject? error, BatchOutputKind outputKind, + bool includeRawStreams, BatchJsonOutputWriter output) { var payload = new JsonObject @@ -883,25 +1051,106 @@ private static bool WriteBatchCommandRecordJson( ["command"] = commandName, ["arguments"] = ToJsonStringArray(subArgs), ["exit_code"] = exitCode, - ["stderr"] = stderr, }; - if (exitCode == CommandExitCodes.Success - && TryParseBatchStructuredOutput(stdout, outputKind, out var resultField, out var structuredOutput)) + if (exitCode == CommandExitCodes.Success) { - payload[resultField] = structuredOutput; + payload["stderr"] = stderr; + if (TryParseBatchStructuredOutput(stdout, outputKind, out var resultField, out var structuredOutput)) + payload[resultField] = structuredOutput; + else + payload["stdout"] = stdout; } else { - payload["stdout"] = stdout; + payload["error"] = BuildBatchCommandFailureError(commandName, exitCode, error); + if (includeRawStreams) + { + payload["raw_streams"] = new JsonObject + { + ["stdout"] = stdout, + ["stderr"] = stderr, + }; + } } - if (error is not null) - payload["error"] = error; - return output.TryWrite(payload); } + private static JsonObject BuildBatchCommandFailureError( + string commandName, + int exitCode, + JsonObject? error) + { + if (error is null && !CliCommandCatalog.IsBatchReadOnlyCommand(commandName)) + { + return BuildBatchTypedError( + "batch command was rejected by the read-only dispatch policy.", + exitCode, + $"Use one of {string.Join(", ", CliCommandCatalog.BatchReadOnlyCommands)}.", + CommandErrorCodes.UsageError, + "batch_command_not_allowed", + "command"); + } + + if (error is null) + { + var (_, category) = CommandErrorWriter.ResolveMachineContract(exitCode); + return BuildBatchTypedError( + "batch child command returned a non-zero exit code.", + exitCode, + BatchFailureHint(exitCode), + errorCode: null, + category: $"batch_child_{category}", + scope: "command", + new JsonObject + { + ["child_exit_code"] = exitCode, + }); + } + + var message = GetBatchErrorString(error, "message") + ?? "batch child command returned a non-zero exit code."; + var hint = GetBatchErrorString(error, "hint") ?? BatchFailureHint(exitCode); + var errorCode = GetBatchErrorString(error, "error_code"); + var categoryValue = GetBatchErrorString(error, "category"); + var scope = GetBatchErrorString(error, "scope") ?? "command"; + return BuildBatchTypedError( + message, + exitCode, + hint, + errorCode, + categoryValue, + scope, + error); + } + + private static string? GetBatchErrorString(JsonObject error, string propertyName) + => error[propertyName] is JsonValue value + && value.TryGetValue(out var text) + ? text + : null; + + private static string BatchFailureHint(int exitCode) + => exitCode switch + { + CommandExitCodes.UsageError or CommandExitCodes.InvalidArgument or CommandExitCodes.ExUsage + => "Check the child command arguments with `cdidx --help`.", + CommandExitCodes.NotFound + => "Broaden the query or remove strict not-found handling before retrying.", + CommandExitCodes.DatabaseError or CommandExitCodes.TransientDatabaseError + => "Run `cdidx status --check --json` and follow its repair guidance.", + CommandExitCodes.FeatureUnavailable + => "Use a build that includes the requested feature or choose a supported output mode.", + CommandExitCodes.StaleIndex + => "Refresh the index and retry the child command.", + CommandExitCodes.CancelledBySignal or CommandExitCodes.LegacyInterrupted + => "Retry the batch when the caller cancellation token is not cancelled.", + CommandExitCodes.PartialResult + => "Inspect the child command directly before relying on the partial result.", + _ => "Retry the child command directly for command-specific diagnostics.", + }; + private static bool TryParseBatchStructuredOutput( string stdout, BatchOutputKind outputKind, @@ -1057,8 +1306,6 @@ private static JsonObject BuildBatchLineErrorJson(int lineNumber, BatchLineError ["status"] = "error", ["line"] = lineNumber, ["exit_code"] = error.ExitCode, - ["stdout"] = string.Empty, - ["stderr"] = string.Empty, ["error"] = ToBatchErrorJson(error), }; } @@ -1070,16 +1317,18 @@ private static void WriteBatchOutputLimitErrorJson( int outputCharLimit, BatchJsonOutputWriter output) { - var error = new JsonObject - { - ["message"] = $"batch serialized output reached the {outputCharLimit} character limit.", - ["hint"] = "Split the request into smaller batches or reduce child output with --limit/--top.", - ["error_code"] = CommandErrorCodes.UsageError, - ["category"] = "batch_output_limit", - ["scope"] = "batch", - ["max_chars"] = outputCharLimit, - ["attempted_exit_code"] = attemptedExitCode, - }; + var error = BuildBatchTypedError( + $"batch serialized output reached the {outputCharLimit} character limit.", + CommandExitCodes.InvalidArgument, + "Split the request into smaller batches or reduce child output with --limit/--top.", + CommandErrorCodes.UsageError, + "batch_output_limit", + "batch", + new JsonObject + { + ["max_chars"] = outputCharLimit, + ["attempted_exit_code"] = attemptedExitCode, + }); var payload = new JsonObject { ["api_version"] = JsonOutputContract.ApiVersion, @@ -1087,8 +1336,6 @@ private static void WriteBatchOutputLimitErrorJson( ["status"] = "error", ["line"] = lineNumber, ["exit_code"] = CommandExitCodes.InvalidArgument, - ["stdout"] = string.Empty, - ["stderr"] = string.Empty, ["error"] = error, }; if (commandName is not null) @@ -1101,21 +1348,13 @@ private static void WriteBatchOutputLimitErrorJson( } private static JsonObject ToBatchErrorJson(BatchLineError error) - { - var payload = new JsonObject - { - ["message"] = error.Message, - }; - - if (!string.IsNullOrWhiteSpace(error.Hint)) - payload["hint"] = error.Hint; - if (!string.IsNullOrWhiteSpace(error.ErrorCode)) - payload["error_code"] = error.ErrorCode; - if (!string.IsNullOrWhiteSpace(error.Category)) - payload["category"] = error.Category; - - return payload; - } + => BuildBatchTypedError( + error.Message, + error.ExitCode, + error.Hint, + error.ErrorCode, + error.Category, + "input"); private static JsonArray ToJsonStringArray(IEnumerable values) { @@ -1196,7 +1435,9 @@ private static bool TryParseBatchLine( error = new BatchLineError( $"batch line {lineNumber} must be a non-empty JSON string array or a command object.", CommandExitCodes.UsageError, - ErrorCode: CommandErrorCodes.UsageError); + Hint: "Use a non-empty JSON string array or a {\"command\",\"args\"} object.", + ErrorCode: CommandErrorCodes.UsageError, + Category: "invalid_batch_input_shape"); if (writeDiagnostics) WriteBatchLineErrorDiagnostic(error, jsonOptions); return false; @@ -1206,7 +1447,9 @@ private static bool TryParseBatchLine( error = new BatchLineError( $"batch line {lineNumber} must contain at most {BatchMaxArgumentCount} command arguments.", CommandExitCodes.UsageError, - ErrorCode: CommandErrorCodes.UsageError); + Hint: "Reduce the number of child command arguments.", + ErrorCode: CommandErrorCodes.UsageError, + Category: "batch_argument_count_limit"); if (writeDiagnostics) WriteBatchLineErrorDiagnostic(error, jsonOptions); return false; @@ -1220,7 +1463,9 @@ private static bool TryParseBatchLine( error = new BatchLineError( $"batch line {lineNumber} must contain only strings.", CommandExitCodes.UsageError, - ErrorCode: CommandErrorCodes.UsageError); + Hint: "Encode the command and every argument as JSON strings.", + ErrorCode: CommandErrorCodes.UsageError, + Category: "invalid_batch_argument_type"); if (writeDiagnostics) WriteBatchLineErrorDiagnostic(error, jsonOptions); return false; @@ -1231,7 +1476,9 @@ private static bool TryParseBatchLine( error = new BatchLineError( $"batch line {lineNumber} argument {values.Count + 1} exceeds the {BatchMaxArgumentChars} character limit.", CommandExitCodes.UsageError, - ErrorCode: CommandErrorCodes.UsageError); + Hint: "Reduce the child command argument length.", + ErrorCode: CommandErrorCodes.UsageError, + Category: "batch_argument_length_limit"); if (writeDiagnostics) WriteBatchLineErrorDiagnostic(error, jsonOptions); return false; @@ -1404,7 +1651,9 @@ private static BatchLineError BuildGenericBatchLineError(int lineNumber) => new( $"batch line {lineNumber} could not be parsed.", CommandExitCodes.UsageError, - ErrorCode: CommandErrorCodes.UsageError); + Hint: "Use a JSON string array or a {\"command\",\"args\"} object.", + ErrorCode: CommandErrorCodes.UsageError, + Category: "invalid_batch_input"); private static int RunBatchQueryCommand( string commandName, @@ -1463,7 +1712,10 @@ private sealed record BatchLineError( string? Category = null, bool WriteAsJson = false); - private sealed record BatchCommandRunResult(int ExitCode, bool OutputLimitReached); + private sealed record BatchCommandRunResult( + int ExitCode, + bool OutputLimitReached, + bool CancellationObserved); private sealed record BatchPendingItem( int LineNumber, diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4723Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4723Tests.cs index 2736de13e..35ce01a9e 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4723Tests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4723Tests.cs @@ -174,12 +174,12 @@ public void RunBatch_ParallelReadsOverlapButEmitInInputOrderAndIsolateFailures_I {"command":"languages","args":["--format","count"]} """; - var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + var (exitCode, batchStdout, stderr) = CaptureConsoleWithInput( input, () => QueryCommandRunner.RunBatch( ["--db", dbPath, "--json-summary", "--parallel", "3"], _jsonOptions)); - var lines = ParseJsonLines(stdout); + var lines = ParseJsonLines(batchStdout); try { Assert.Equal(CommandExitCodes.UsageError, exitCode); @@ -314,7 +314,7 @@ public void RunBatch_ParallelFailureExitCodeFollowsInputOrder_Issue4723() } [Fact] - public void RunBatch_ParallelReadsPropagateCancellationAndRestoreConsole_Issue4723() + public void RunBatch_ParallelReadsSerializeCancellationAndRestoreConsole_Issues4723_4871() { using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_cancel_4723"); var dbPath = TestProjectHelper.CreateProjectDb(project.Root); @@ -327,18 +327,31 @@ public void RunBatch_ParallelReadsPropagateCancellationAndRestoreConsole_Issue47 try { - var exception = Record.Exception(() => CaptureConsoleWithInput( - """ - {"command":"recipes","args":["--json"]} - {"command":"languages","args":["--format","count"]} - - """, + var (exitCode, batchStdout, stderr) = CaptureConsoleWithInput( + """{"command":"recipes","args":["--json"]}""" + "\n", () => QueryCommandRunner.RunBatch( ["--db", dbPath, "--json-summary", "--parallel", "2"], _jsonOptions, - cancellationToken: cancellation.Token))); + cancellationToken: cancellation.Token)); Assert.True(cancellation.IsCancellationRequested); - Assert.IsAssignableFrom(exception); + Assert.Equal(CommandExitCodes.CancelledBySignal, exitCode); + Assert.Equal(string.Empty, stderr); + var lines = ParseJsonLines(batchStdout); + try + { + Assert.Equal(2, lines.Count); + Assert.Equal( + "batch_cancelled", + lines[0].RootElement.GetProperty("error").GetProperty("category").GetString()); + Assert.Equal( + CommandExitCodes.CancelledBySignal, + lines[1].RootElement.GetProperty("exit_code").GetInt32()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } } finally { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4871Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4871Tests.cs new file mode 100644 index 000000000..d97322949 --- /dev/null +++ b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4871Tests.cs @@ -0,0 +1,244 @@ +using CodeIndex.Cli; + +namespace CodeIndex.Tests; + +public partial class QueryCommandRunnerTests +{ + [Fact] + public void RunBatch_AllJsonSummaryFailuresUseTypedEnvelopesInSerialAndParallel_Issue4871() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_typed_errors_4871"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + var input = """ + 42 + [1] + [] + {"command":"status","args":"--json"} + ["unknown"] + ["search","missing-symbol-4871","--json","--strict-not-found"] + + """; + var expectedCategories = new[] + { + "invalid_batch_input_shape", + "invalid_batch_argument_type", + "invalid_batch_input_shape", + "invalid_batch_command_object", + "batch_command_not_allowed", + "batch_child_not_found", + }; + + foreach (var batchArgs in new[] + { + new[] { "--db", dbPath, "--json-summary" }, + new[] { "--db", dbPath, "--json-summary", "--parallel", "3" }, + }) + { + var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + input, + () => QueryCommandRunner.RunBatch(batchArgs, _jsonOptions)); + var lines = ParseJsonLines(stdout); + try + { + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal(expectedCategories.Length + 1, lines.Count); + + for (var index = 0; index < expectedCategories.Length; index++) + { + var record = lines[index].RootElement; + Assert.Equal("1", record.GetProperty("api_version").GetString()); + Assert.Equal("error", record.GetProperty("status").GetString()); + Assert.Equal(index + 1, record.GetProperty("line").GetInt32()); + Assert.False(record.TryGetProperty("stdout", out _)); + Assert.False(record.TryGetProperty("stderr", out _)); + Assert.False(record.TryGetProperty("raw_streams", out _)); + + var error = record.GetProperty("error"); + Assert.False(string.IsNullOrWhiteSpace(error.GetProperty("message").GetString())); + Assert.False(string.IsNullOrWhiteSpace(error.GetProperty("hint").GetString())); + Assert.False(string.IsNullOrWhiteSpace(error.GetProperty("error_code").GetString())); + Assert.Equal(expectedCategories[index], error.GetProperty("category").GetString()); + } + + var summary = lines[^1].RootElement; + Assert.Equal("batch_summary", summary.GetProperty("record").GetString()); + Assert.Equal(expectedCategories.Length, summary.GetProperty("input_lines_read").GetInt32()); + Assert.Equal(2, summary.GetProperty("commands_processed").GetInt32()); + Assert.Equal(4, summary.GetProperty("line_errors").GetInt32()); + Assert.Equal(2, summary.GetProperty("command_failures").GetInt32()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + } + + [Fact] + public void RunBatch_RawFailureStreamsRequireExplicitJsonSummaryOptIn_Issue4871() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_raw_streams_4871"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + + var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + "[\"unknown\"]\n", + () => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--include-raw-streams"], + _jsonOptions)); + var lines = ParseJsonLines(stdout); + try + { + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal(2, lines.Count); + var record = lines[0].RootElement; + Assert.Equal( + "batch_command_not_allowed", + record.GetProperty("error").GetProperty("category").GetString()); + var rawStreams = record.GetProperty("raw_streams"); + Assert.Equal(string.Empty, rawStreams.GetProperty("stdout").GetString()); + Assert.Contains( + "batch only supports query and read-only discovery commands", + rawStreams.GetProperty("stderr").GetString()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + + var (invalidExitCode, _, invalidStderr) = CaptureConsole( + () => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--include-raw-streams"], + _jsonOptions)); + Assert.Equal(CommandExitCodes.UsageError, invalidExitCode); + Assert.Contains("--include-raw-streams requires --json-summary", invalidStderr); + } + + [Fact] + public void RunBatch_ParallelTimeoutUsesTypedErrorAndPreservesInputOrder_Issue4871() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_timeout_4871"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + QueryCommandRunner.BatchParallelCommandStartedForTesting = lineNumber => + { + if (lineNumber == 1) + throw new TimeoutException("untrusted timeout detail"); + }; + + try + { + var input = """ + {"command":"status","args":["--json"]} + {"command":"languages","args":["--format","count"]} + + """; + var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + input, + () => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "2"], + _jsonOptions)); + var lines = ParseJsonLines(stdout); + try + { + Assert.Equal(CommandExitCodes.RuntimeError, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal(3, lines.Count); + Assert.Equal(1, lines[0].RootElement.GetProperty("line").GetInt32()); + Assert.Equal( + "batch_command_timeout", + lines[0].RootElement.GetProperty("error").GetProperty("category").GetString()); + Assert.DoesNotContain("untrusted timeout detail", stdout); + Assert.Equal(2, lines[1].RootElement.GetProperty("line").GetInt32()); + Assert.Equal("ok", lines[1].RootElement.GetProperty("status").GetString()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + finally + { + QueryCommandRunner.BatchParallelCommandStartedForTesting = null; + } + } + + [Fact] + public void RunBatch_ParallelCancellationUsesTypedErrorsAndRestoresConsole_Issue4871() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_cancel_4871"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var cancellation = new CancellationTokenSource(); + using var bothStarted = new CountdownEvent(2); + using var cancellationTriggered = new ManualResetEventSlim(); + QueryCommandRunner.BatchParallelCommandStartedForTesting = lineNumber => + { + bothStarted.Signal(); + if (!bothStarted.Wait(TimeSpan.FromSeconds(5))) + throw new TimeoutException("Both batch commands did not start."); + if (lineNumber == 1) + { + cancellation.Cancel(); + cancellationTriggered.Set(); + } + else if (!cancellationTriggered.Wait(TimeSpan.FromSeconds(5))) + { + throw new TimeoutException("The first batch command did not cancel the batch."); + } + }; + + try + { + var input = """ + {"command":"status","args":["--json"]} + {"command":"languages","args":["--format","count"]} + + """; + var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + input, + () => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "2"], + _jsonOptions, + cancellationToken: cancellation.Token)); + var lines = ParseJsonLines(stdout); + try + { + Assert.True(cancellation.IsCancellationRequested); + Assert.Equal(CommandExitCodes.CancelledBySignal, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal(3, lines.Count); + for (var index = 0; index < 2; index++) + { + var record = lines[index].RootElement; + Assert.Equal(index + 1, record.GetProperty("line").GetInt32()); + Assert.Equal("error", record.GetProperty("status").GetString()); + var error = record.GetProperty("error"); + Assert.Equal(CommandErrorCodes.Interrupted, error.GetProperty("error_code").GetString()); + Assert.Equal("batch_cancelled", error.GetProperty("category").GetString()); + } + Assert.Equal( + CommandExitCodes.CancelledBySignal, + lines[^1].RootElement.GetProperty("exit_code").GetInt32()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + finally + { + QueryCommandRunner.BatchParallelCommandStartedForTesting = null; + } + + var (_, restoredStdout, _) = CaptureConsole(() => + { + Console.Write("restored"); + return 0; + }); + Assert.Equal("restored", restoredStdout); + } +} diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index f144a6add..fd69e46b4 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -1178,9 +1178,12 @@ public void RunBatch_JsonSummaryReportsProcessedCommandsAndFailures_Issue3906_Is Assert.Equal("error", lineError.GetProperty("status").GetString()); Assert.Equal(2, lineError.GetProperty("line").GetInt32()); Assert.Equal(CommandExitCodes.UsageError, lineError.GetProperty("exit_code").GetInt32()); - Assert.Equal(string.Empty, lineError.GetProperty("stdout").GetString()); - Assert.Equal(string.Empty, lineError.GetProperty("stderr").GetString()); + Assert.False(lineError.TryGetProperty("stdout", out _)); + Assert.False(lineError.TryGetProperty("stderr", out _)); Assert.Contains("batch line 2 must be a non-empty JSON string array or a command object", lineError.GetProperty("error").GetProperty("message").GetString()); + Assert.Equal( + "invalid_batch_input_shape", + lineError.GetProperty("error").GetProperty("category").GetString()); var unsupportedRecord = unsupportedRecordDocument.RootElement; Assert.Equal("batch_result", unsupportedRecord.GetProperty("record").GetString()); @@ -1188,8 +1191,11 @@ public void RunBatch_JsonSummaryReportsProcessedCommandsAndFailures_Issue3906_Is Assert.Equal(3, unsupportedRecord.GetProperty("line").GetInt32()); Assert.Equal("unknown", unsupportedRecord.GetProperty("command").GetString()); Assert.Equal(CommandExitCodes.UsageError, unsupportedRecord.GetProperty("exit_code").GetInt32()); - Assert.Equal(string.Empty, unsupportedRecord.GetProperty("stdout").GetString()); - Assert.Contains("batch only supports query and read-only discovery commands", unsupportedRecord.GetProperty("stderr").GetString()); + Assert.False(unsupportedRecord.TryGetProperty("stdout", out _)); + Assert.False(unsupportedRecord.TryGetProperty("stderr", out _)); + Assert.Equal( + "batch_command_not_allowed", + unsupportedRecord.GetProperty("error").GetProperty("category").GetString()); var summary = summaryDocument.RootElement; Assert.Equal("batch_summary", summary.GetProperty("record").GetString()); @@ -1244,8 +1250,11 @@ public void Authenticate() { } Assert.Equal("batch_result", unsupportedRecord.GetProperty("record").GetString()); Assert.Equal("error", unsupportedRecord.GetProperty("status").GetString()); Assert.Equal("unknown", unsupportedRecord.GetProperty("command").GetString()); - Assert.Equal(string.Empty, unsupportedRecord.GetProperty("stdout").GetString()); - Assert.Contains("batch only supports query and read-only discovery commands", unsupportedRecord.GetProperty("stderr").GetString()); + Assert.False(unsupportedRecord.TryGetProperty("stdout", out _)); + Assert.False(unsupportedRecord.TryGetProperty("stderr", out _)); + Assert.Equal( + "batch_command_not_allowed", + unsupportedRecord.GetProperty("error").GetProperty("category").GetString()); var summary = summaryDocument.RootElement; Assert.Equal("batch_summary", summary.GetProperty("record").GetString()); @@ -1388,10 +1397,11 @@ public void RunBatch_JsonSummaryReportsCaptureLimitFromWithDb_Issue4142() Assert.Equal("error", excerptRecord.GetProperty("status").GetString()); Assert.Equal("excerpt", excerptRecord.GetProperty("command").GetString()); Assert.Equal(CommandExitCodes.InvalidArgument, excerptRecord.GetProperty("exit_code").GetInt32()); - Assert.DoesNotContain("database", excerptRecord.GetProperty("stderr").GetString()!, StringComparison.OrdinalIgnoreCase); + Assert.False(excerptRecord.TryGetProperty("stderr", out _)); var error = excerptRecord.GetProperty("error"); Assert.Contains("stdout exceeded", error.GetProperty("message").GetString()); Assert.Equal(CommandErrorCodes.UsageError, error.GetProperty("error_code").GetString()); + Assert.Equal("batch_child_output_limit", error.GetProperty("category").GetString()); Assert.Equal(JsonEnvelopeWrapper.MaxCapturedOutputChars, error.GetProperty("max_chars").GetInt32()); Assert.Equal("stdout", error.GetProperty("stream").GetString()); Assert.Equal("command", error.GetProperty("scope").GetString()); From 4b5ebd297e45caaef5a47c85f413512135a95ab9 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Wed, 29 Jul 2026 09:07:56 +0900 Subject: [PATCH 2/3] Handle batch cancellation boundaries (#4871) --- DEVELOPER_GUIDE.md | 8 +- TESTING_GUIDE.md | 4 +- src/CodeIndex/Cli/QueryCommandRunner.Batch.cs | 124 +++++++++++++-- .../QueryCommandRunnerBatchIssue4871Tests.cs | 147 ++++++++++++++++++ 4 files changed, 270 insertions(+), 13 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index c0a058b36..84e81b3ba 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -445,7 +445,9 @@ current worker's routed stdout instead of replacing another worker's process-wid writer. Completed records are committed to the shared output writer in input order; an ordinary item failure remains isolated. Caller cancellation is serialized as `batch_cancelled` for a consumed input item and in the final -summary before batch processing stops. +summary before batch processing stops. Parallel input waits remain +cancellation-aware even while stdin is blocked, and cancellation during +database setup still emits the typed final summary. Editor integrations can request standard location shapes directly. `definition`, `references`, `search`, `find`, and `validate` accept `--format `; `lsp` emits LSP `Location` arrays, `qf` emits Vim quickfix lines, and `sarif` emits SARIF 2.1.0. `goto ` returns the single unambiguous definition as one LSP `Location`, while `goto --all ` returns all matching locations without applying the default or environment-provided query limit. An explicit `--limit` or `--top` still bounds the returned location array. @@ -3714,7 +3716,9 @@ batch reader を使い、active worker window だけを buffer する。`ScopedC JSON-envelope capture を現在の worker の routed stdout に保ち、他 worker の process-wide writer を 置き換えない。完了 record は入力順で共有 output writer へ commit する。通常の item failure は 他 item から隔離する。caller cancellation は、消費済み input item と final summary に -`batch_cancelled` を記録してから後続処理を停止する。 +`batch_cancelled` を記録してから後続処理を停止する。parallel input wait は stdin が +block 中でも cancellation を検知し、database setup 中の cancellation でも型付き final +summary を出力する。 editor integration は標準的な location 形状を直接要求できる。`definition`、`references`、`search`、`find`、`validate` は `--format ` を受け付け、`lsp` は LSP `Location` 配列、`qf` は Vim quickfix 行、`sarif` は SARIF 2.1.0 を出力する。`goto ` は曖昧でない単一定義を 1 つの LSP `Location` として返し、`goto --all ` は既定または環境変数由来の query limit を適用せず、一致する全 location を返す。明示的な `--limit` または `--top` を指定した場合は location 配列をその件数に制限する。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 485694d29..22f1ee28d 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -193,7 +193,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Doctor full-inventory coverage keeps composed filter selection, filtered summary counts, exact UTF-8 byte-budget boundaries, and structured overflow errors together in `ProgramRunnerTests`; license JSON remains a subprocess contract in `ProgramCliTests` so immediate-command dispatch and the published field names are both exercised. Ctags export JSON coverage reuses one seeded database for default and `--include-generated` variants, asserts the fixed skip-reason keys sum to `skipped_count`, and keeps the missing-`files.generated` degradation in a separate legacy-schema fixture. Dry-run JSON coverage for ambiguous `.h` files locks the bounded `language_detections` entries and their stable source/confidence codes without mutating the index. - Batch `--json-summary` coverage keeps side-effect-free dispatch plus successful JSON, single-row NDJSON, and JSON-looking text projection in one seeded fixture when the assertions stay distinct; include case-insensitive format values, command-specific JSON formats, and alias-injected JSON defaults so batch classification cannot drift from child parsers. Serialized-output exhaustion remains a separate boundary fixture because of its large escaped payload. Assert `result` / `results` versus raw `stdout` presence explicitly, compare `output_chars` with the actual captured stream length, and cover repeated malformed lines up to the input cap so neither syntax sniffing nor envelope overhead can bypass the transport bounds. Typed-failure coverage reuses one mixed scalar/array/schema/policy/child-failure input across serial and parallel modes, asserts stable ordered categories, and keeps timeout, caller cancellation, and explicit `--include-raw-streams` behavior in focused fixtures. + Batch `--json-summary` coverage keeps side-effect-free dispatch plus successful JSON, single-row NDJSON, and JSON-looking text projection in one seeded fixture when the assertions stay distinct; include case-insensitive format values, command-specific JSON formats, and alias-injected JSON defaults so batch classification cannot drift from child parsers. Serialized-output exhaustion remains a separate boundary fixture because of its large escaped payload. Assert `result` / `results` versus raw `stdout` presence explicitly, compare `output_chars` with the actual captured stream length, and cover repeated malformed lines up to the input cap so neither syntax sniffing nor envelope overhead can bypass the transport bounds. Typed-failure coverage reuses one mixed scalar/array/schema/policy/child-failure input across serial and parallel modes, asserts stable ordered categories, and keeps timeout, caller cancellation, and explicit `--include-raw-streams` behavior in focused fixtures. Cancellation coverage includes pre-cancelled setup and a blocking stdin reader so the typed summary and prompt parallel wake-up remain deterministic. Argument-validation variants that only differ by invalid scalar input share one database fixture and iterate within a fact when no per-case state or discovery identity is required. Positional `files` glob coverage shares one indexed-file fixture and iterates `*`, `?`, and recursive `**` patterns in a fact, matching the exact tokens that a quoted shell argument passes to the CLI. Excerpt focus coverage reuses one indexed fixture for line-only leading-window behavior, the focus-length dependency, and focus-column range validation; zero and non-numeric focus-column values share one indexed Markdown fixture. @@ -1112,7 +1112,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" doctor full-inventory coverage では、合成 filter の選択、filtered summary 件数、UTF-8 byte budget の exact boundary、structured overflow error を `ProgramRunnerTests` にまとめます。license JSON は `ProgramCliTests` の subprocess contract として、immediate-command dispatch と公開 field 名を同時に検証します。 ctags export JSON coverage は1つの seeded database を既定と `--include-generated` variant で再利用し、固定された skip-reason key の合計が `skipped_count` と一致することを検証します。`files.generated` がない場合の縮退は別の legacy-schema fixture に保ってください。 曖昧な `.h` に対する dry-run JSON coverage は、index を変更せず、上限付き `language_detections` entry と安定した判定元・信頼度 code を固定します。 - batch `--json-summary` coverage は、assertion を明確に保てる場合、副作用なし dispatch と成功時の JSON / 1 row NDJSON / JSON に見える text projection を1つの seeded fixture で共有し、case-insensitive な format 値、command 固有の JSON format、alias が内部追加する JSON default も含めて child parser と batch classification の drift を防いでください。serialized output の枯渇は escape を多く含む大きな payload の境界 fixture として分離します。test 側で `result` / `results` と raw `stdout` の有無を明示し、`output_chars` を実際に捕捉した stream 長と比較し、input cap までの malformed line 反復も検証して、syntax sniffing や envelope overhead が transport 上限を迂回しないようにしてください。typed failure coverage は scalar / array / schema / policy / child failure を混在させた1つの input を serial / parallel mode で共有し、安定した順序と category を検証してください。timeout、caller cancellation、明示的な `--include-raw-streams` の挙動は焦点を絞った fixture に分離します。 + batch `--json-summary` coverage は、assertion を明確に保てる場合、副作用なし dispatch と成功時の JSON / 1 row NDJSON / JSON に見える text projection を1つの seeded fixture で共有し、case-insensitive な format 値、command 固有の JSON format、alias が内部追加する JSON default も含めて child parser と batch classification の drift を防いでください。serialized output の枯渇は escape を多く含む大きな payload の境界 fixture として分離します。test 側で `result` / `results` と raw `stdout` の有無を明示し、`output_chars` を実際に捕捉した stream 長と比較し、input cap までの malformed line 反復も検証して、syntax sniffing や envelope overhead が transport 上限を迂回しないようにしてください。typed failure coverage は scalar / array / schema / policy / child failure を混在させた1つの input を serial / parallel mode で共有し、安定した順序と category を検証してください。timeout、caller cancellation、明示的な `--include-raw-streams` の挙動は焦点を絞った fixture に分離します。cancellation coverage には事前 cancel 済み setup と blocking stdin reader を含め、型付き summary と parallel wait の迅速な解除を決定的に検証します。 invalid scalar input だけが異なる argument-validation variant は、case ごとの state や discovery identity が不要なら1つの database fixture を共有し、fact 内で反復してください。 `files` の positional glob coverage は1つの indexed-file fixture を共有し、quote された shell 引数が CLI に渡す token と同じ `*`、`?`、recursive `**` pattern を fact 内で反復してください。 excerpt の focus coverage は、line-only 時の先頭側 window、focus-length の依存関係、focus-column の範囲検証を1つの indexed fixture で共有してください。focus-column の zero / non-numeric value も1つの indexed Markdown fixture を再利用してください。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs index b4732c8d7..8e0ad80f0 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs @@ -152,11 +152,20 @@ public static int RunBatch( if (parallelism > 1) { - using (var validationDb = new DbContext(DbOpenIntent.QueryOnly, dbPath, cancellationToken)) + try { + using var validationDb = new DbContext(DbOpenIntent.QueryOnly, dbPath, cancellationToken); if (!validationDb.TryValidateIsCodeIndexDb(out var validationReason)) return WriteInvalidCodeIndexDbError(dbPath, validationReason, json: false, jsonOptions); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return WriteBatchSetupCancellationSummary( + maxInputLines, + maxOutputChars, + parallelism, + jsonOptions); + } return RunBatchParallel( dbPath, @@ -345,6 +354,16 @@ public static int RunBatch( return firstFailure; } + catch (OperationCanceledException) when ( + jsonSummary + && cancellationToken.IsCancellationRequested) + { + return WriteBatchSetupCancellationSummary( + maxInputLines, + maxOutputChars, + parallelism, + jsonOptions); + } finally { s_batchReader = null; @@ -424,6 +443,7 @@ private static int RunBatchParallel( var outputLimitReached = false; var inputLimitReached = false; var cancellationObserved = false; + var producerReadInProgress = 0; using var stopProducing = new CancellationTokenSource(); var input = Channel.CreateBounded( new BoundedChannelOptions(1) @@ -448,8 +468,23 @@ private static int RunBatchParallel( cancellationObserved = true; break; } - if (!TryReadBatchLine(Console.In, out var line, out var lineExceededLimit)) + string? line; + bool lineExceededLimit; + Volatile.Write(ref producerReadInProgress, 1); + try + { + if (!TryReadBatchLine(Console.In, out line, out lineExceededLimit)) + break; + } + finally + { + Volatile.Write(ref producerReadInProgress, 0); + } + if (cancellationToken.IsCancellationRequested) + { + cancellationObserved = true; break; + } lineNumber++; if (lineNumber > maxInputLines) @@ -659,12 +694,21 @@ private static int RunBatchParallel( if (active.Count > 0 && active.Peek().Result.IsCompleted) continue; - using var waitCancellation = new CancellationTokenSource(); + using var waitCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken); var waitForInput = input.Reader.WaitToReadAsync(waitCancellation.Token).AsTask(); if (active.Count == 0) { - if (!waitForInput.GetAwaiter().GetResult()) + try + { + if (!waitForInput.GetAwaiter().GetResult()) + break; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + cancellationObserved = true; break; + } continue; } @@ -682,6 +726,18 @@ private static int RunBatchParallel( { } } + else + { + try + { + _ = waitForInput.GetAwaiter().GetResult(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + cancellationObserved = true; + active.Peek().Result.GetAwaiter().GetResult(); + } + } } } catch @@ -728,14 +784,28 @@ private static int RunBatchParallel( active.Dequeue().Result.GetAwaiter().GetResult(); } - try + var producerBlockedOnInput = cancellationObserved + && Volatile.Read(ref producerReadInProgress) != 0 + && !producer.IsCompleted; + if (producerBlockedOnInput) { - producer.GetAwaiter().GetResult(); + _ = producer.ContinueWith( + static completedProducer => _ = completedProducer.Exception, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnFaulted, + TaskScheduler.Default); } - catch (OperationCanceledException) when ( - outputLimitReached - && !cancellationToken.IsCancellationRequested) + else { + try + { + producer.GetAwaiter().GetResult(); + } + catch (OperationCanceledException) when ( + outputLimitReached + && !cancellationToken.IsCancellationRequested) + { + } } if (cancellationObserved && firstFailure == CommandExitCodes.Success) @@ -868,10 +938,46 @@ private static void WriteBatchSummaryJson( ["input_limit_reached"] = inputLimitReached, ["parallelism"] = parallelism, }; + if (exitCode is CommandExitCodes.CancelledBySignal or CommandExitCodes.LegacyInterrupted) + { + payload["error"] = BuildBatchTypedError( + "batch processing was cancelled by the caller.", + exitCode, + "Retry the batch when the caller cancellation token is not cancelled.", + CommandErrorCodes.Interrupted, + "batch_cancelled", + "batch"); + } output.WriteSummary(payload); } + private static int WriteBatchSetupCancellationSummary( + int maxInputLines, + int maxOutputChars, + int parallelism, + JsonSerializerOptions jsonOptions) + { + var output = new BatchJsonOutputWriter( + Console.Out, + maxOutputChars, + BatchTerminalOutputReserveChars, + jsonOptions); + WriteBatchSummaryJson( + inputLinesRead: 0, + commandsProcessed: 0, + lineErrors: 0, + commandFailures: 0, + CommandExitCodes.CancelledBySignal, + outputLimitReached: false, + inputLimitReached: false, + maxInputLines, + maxOutputChars, + parallelism, + output); + return CommandExitCodes.CancelledBySignal; + } + private static BatchCommandRunResult RunBatchQueryCommandWithJsonRecord( int lineNumber, string commandName, diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4871Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4871Tests.cs index d97322949..1e391b5f7 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4871Tests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4871Tests.cs @@ -241,4 +241,151 @@ public void RunBatch_ParallelCancellationUsesTypedErrorsAndRestoresConsole_Issue }); Assert.Equal("restored", restoredStdout); } + + [Fact] + public void RunBatch_SetupCancellationUsesTypedSummaryInSerialAndParallel_Issue4871() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_setup_cancel_4871"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + foreach (var batchArgs in new[] + { + new[] { "--db", dbPath, "--json-summary" }, + new[] { "--db", dbPath, "--json-summary", "--parallel", "2" }, + }) + { + var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + "[\"status\",\"--json\"]\n", + () => QueryCommandRunner.RunBatch( + batchArgs, + _jsonOptions, + cancellationToken: cancellation.Token)); + var lines = ParseJsonLines(stdout); + try + { + Assert.Equal(CommandExitCodes.CancelledBySignal, exitCode); + Assert.Equal(string.Empty, stderr); + var summary = Assert.Single(lines).RootElement; + Assert.Equal("1", summary.GetProperty("api_version").GetString()); + Assert.Equal("batch_summary", summary.GetProperty("record").GetString()); + Assert.Equal(0, summary.GetProperty("input_lines_read").GetInt32()); + Assert.Equal( + "batch_cancelled", + summary.GetProperty("error").GetProperty("category").GetString()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + } + + [Fact] + public void RunBatch_ParallelCancellationInterruptsBlockedInputWait_Issue4871() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_blocked_input_cancel_4871"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var reader = new BlockingBatchReader(); + using var cancellation = new CancellationTokenSource(); + using var batchCompleted = new ManualResetEventSlim(); + using var cancellationTaskCompleted = new ManualResetEventSlim(); + var forcedReaderRelease = 0; + Exception? cancellationFailure = null; + using var capture = ConsoleCapture.StartWithInput( + reader, + captureOut: true, + captureError: true); + _ = Task.Run(() => + { + try + { + if (!reader.WaitUntilRead(TimeSpan.FromSeconds(5))) + throw new TimeoutException("The parallel batch producer did not start reading input."); + cancellation.Cancel(); + if (!batchCompleted.Wait(TimeSpan.FromSeconds(5))) + { + Interlocked.Exchange(ref forcedReaderRelease, 1); + reader.Release(); + } + } + catch (Exception ex) + { + cancellationFailure = ex; + } + finally + { + cancellationTaskCompleted.Set(); + } + }); + + int exitCode; + try + { + exitCode = QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "2"], + _jsonOptions, + cancellationToken: cancellation.Token); + batchCompleted.Set(); + Assert.True(cancellationTaskCompleted.Wait(TimeSpan.FromSeconds(5))); + } + finally + { + batchCompleted.Set(); + reader.Release(); + Assert.True(cancellationTaskCompleted.Wait(TimeSpan.FromSeconds(5))); + } + + Assert.Null(cancellationFailure); + Assert.Equal(0, Volatile.Read(ref forcedReaderRelease)); + Assert.Equal(CommandExitCodes.CancelledBySignal, exitCode); + Assert.Equal(string.Empty, capture.Error!.ToString()); + var lines = ParseJsonLines(capture.Out!.ToString() ?? string.Empty); + try + { + var summary = Assert.Single(lines).RootElement; + Assert.Equal("batch_summary", summary.GetProperty("record").GetString()); + Assert.Equal(CommandExitCodes.CancelledBySignal, summary.GetProperty("exit_code").GetInt32()); + Assert.Equal( + "batch_cancelled", + summary.GetProperty("error").GetProperty("category").GetString()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + + private sealed class BlockingBatchReader : TextReader + { + private readonly ManualResetEventSlim _entered = new(); + private readonly ManualResetEventSlim _released = new(); + + public override int Read() + { + _entered.Set(); + _released.Wait(); + return -1; + } + + public bool WaitUntilRead(TimeSpan timeout) + => _entered.Wait(timeout); + + public void Release() + => _released.Set(); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _released.Set(); + _entered.Dispose(); + _released.Dispose(); + } + base.Dispose(disposing); + } + } } From 19cb1e2dd1d29292ec55c2e618b1979454cd1054 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Wed, 29 Jul 2026 10:00:56 +0900 Subject: [PATCH 3/3] Coordinate cancellable batch input (#4871) --- DEVELOPER_GUIDE.md | 15 +- TESTING_GUIDE.md | 4 +- src/CodeIndex/Cli/QueryCommandRunner.Batch.cs | 303 ++++++++++-------- .../QueryCommandRunnerBatchIssue4871Tests.cs | 139 +++++++- 4 files changed, 318 insertions(+), 143 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 84e81b3ba..d5fc2ff69 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -445,9 +445,11 @@ current worker's routed stdout instead of replacing another worker's process-wid writer. Completed records are committed to the shared output writer in input order; an ordinary item failure remains isolated. Caller cancellation is serialized as `batch_cancelled` for a consumed input item and in the final -summary before batch processing stops. Parallel input waits remain -cancellation-aware even while stdin is blocked, and cancellation during -database setup still emits the typed final summary. +summary before batch processing stops. Serial and parallel input waits share a +bounded pump for each input reader, so cancellation remains prompt while stdin +is blocked and any line completed in flight stays buffered for a subsequent +batch invocation. Cancellation during database setup still emits the typed +final summary. Editor integrations can request standard location shapes directly. `definition`, `references`, `search`, `find`, and `validate` accept `--format `; `lsp` emits LSP `Location` arrays, `qf` emits Vim quickfix lines, and `sarif` emits SARIF 2.1.0. `goto ` returns the single unambiguous definition as one LSP `Location`, while `goto --all ` returns all matching locations without applying the default or environment-provided query limit. An explicit `--limit` or `--top` still bounds the returned location array. @@ -3716,9 +3718,10 @@ batch reader を使い、active worker window だけを buffer する。`ScopedC JSON-envelope capture を現在の worker の routed stdout に保ち、他 worker の process-wide writer を 置き換えない。完了 record は入力順で共有 output writer へ commit する。通常の item failure は 他 item から隔離する。caller cancellation は、消費済み input item と final summary に -`batch_cancelled` を記録してから後続処理を停止する。parallel input wait は stdin が -block 中でも cancellation を検知し、database setup 中の cancellation でも型付き final -summary を出力する。 +`batch_cancelled` を記録してから後続処理を停止する。serial / parallel の input wait は input +reader ごとの bounded pump を共有するため、stdin が block 中でも cancellation を迅速に検知し、 +同時に完成した line は後続の batch invocation 用に buffer したまま保持する。database setup +中の cancellation でも型付き final summary を出力する。 editor integration は標準的な location 形状を直接要求できる。`definition`、`references`、`search`、`find`、`validate` は `--format ` を受け付け、`lsp` は LSP `Location` 配列、`qf` は Vim quickfix 行、`sarif` は SARIF 2.1.0 を出力する。`goto ` は曖昧でない単一定義を 1 つの LSP `Location` として返し、`goto --all ` は既定または環境変数由来の query limit を適用せず、一致する全 location を返す。明示的な `--limit` または `--top` を指定した場合は location 配列をその件数に制限する。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 22f1ee28d..d9357cefe 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -193,7 +193,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Doctor full-inventory coverage keeps composed filter selection, filtered summary counts, exact UTF-8 byte-budget boundaries, and structured overflow errors together in `ProgramRunnerTests`; license JSON remains a subprocess contract in `ProgramCliTests` so immediate-command dispatch and the published field names are both exercised. Ctags export JSON coverage reuses one seeded database for default and `--include-generated` variants, asserts the fixed skip-reason keys sum to `skipped_count`, and keeps the missing-`files.generated` degradation in a separate legacy-schema fixture. Dry-run JSON coverage for ambiguous `.h` files locks the bounded `language_detections` entries and their stable source/confidence codes without mutating the index. - Batch `--json-summary` coverage keeps side-effect-free dispatch plus successful JSON, single-row NDJSON, and JSON-looking text projection in one seeded fixture when the assertions stay distinct; include case-insensitive format values, command-specific JSON formats, and alias-injected JSON defaults so batch classification cannot drift from child parsers. Serialized-output exhaustion remains a separate boundary fixture because of its large escaped payload. Assert `result` / `results` versus raw `stdout` presence explicitly, compare `output_chars` with the actual captured stream length, and cover repeated malformed lines up to the input cap so neither syntax sniffing nor envelope overhead can bypass the transport bounds. Typed-failure coverage reuses one mixed scalar/array/schema/policy/child-failure input across serial and parallel modes, asserts stable ordered categories, and keeps timeout, caller cancellation, and explicit `--include-raw-streams` behavior in focused fixtures. Cancellation coverage includes pre-cancelled setup and a blocking stdin reader so the typed summary and prompt parallel wake-up remain deterministic. + Batch `--json-summary` coverage keeps side-effect-free dispatch plus successful JSON, single-row NDJSON, and JSON-looking text projection in one seeded fixture when the assertions stay distinct; include case-insensitive format values, command-specific JSON formats, and alias-injected JSON defaults so batch classification cannot drift from child parsers. Serialized-output exhaustion remains a separate boundary fixture because of its large escaped payload. Assert `result` / `results` versus raw `stdout` presence explicitly, compare `output_chars` with the actual captured stream length, and cover repeated malformed lines up to the input cap so neither syntax sniffing nor envelope overhead can bypass the transport bounds. Typed-failure coverage reuses one mixed scalar/array/schema/policy/child-failure input across serial and parallel modes, asserts stable ordered categories, and keeps timeout, caller cancellation, and explicit `--include-raw-streams` behavior in focused fixtures. Cancellation coverage includes pre-cancelled setup, cancellation after an earlier failure and after parallel item preparation, and a blocking stdin reader that proves both prompt wake-up and preservation of an in-flight line for the next batch invocation. Argument-validation variants that only differ by invalid scalar input share one database fixture and iterate within a fact when no per-case state or discovery identity is required. Positional `files` glob coverage shares one indexed-file fixture and iterates `*`, `?`, and recursive `**` patterns in a fact, matching the exact tokens that a quoted shell argument passes to the CLI. Excerpt focus coverage reuses one indexed fixture for line-only leading-window behavior, the focus-length dependency, and focus-column range validation; zero and non-numeric focus-column values share one indexed Markdown fixture. @@ -1112,7 +1112,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" doctor full-inventory coverage では、合成 filter の選択、filtered summary 件数、UTF-8 byte budget の exact boundary、structured overflow error を `ProgramRunnerTests` にまとめます。license JSON は `ProgramCliTests` の subprocess contract として、immediate-command dispatch と公開 field 名を同時に検証します。 ctags export JSON coverage は1つの seeded database を既定と `--include-generated` variant で再利用し、固定された skip-reason key の合計が `skipped_count` と一致することを検証します。`files.generated` がない場合の縮退は別の legacy-schema fixture に保ってください。 曖昧な `.h` に対する dry-run JSON coverage は、index を変更せず、上限付き `language_detections` entry と安定した判定元・信頼度 code を固定します。 - batch `--json-summary` coverage は、assertion を明確に保てる場合、副作用なし dispatch と成功時の JSON / 1 row NDJSON / JSON に見える text projection を1つの seeded fixture で共有し、case-insensitive な format 値、command 固有の JSON format、alias が内部追加する JSON default も含めて child parser と batch classification の drift を防いでください。serialized output の枯渇は escape を多く含む大きな payload の境界 fixture として分離します。test 側で `result` / `results` と raw `stdout` の有無を明示し、`output_chars` を実際に捕捉した stream 長と比較し、input cap までの malformed line 反復も検証して、syntax sniffing や envelope overhead が transport 上限を迂回しないようにしてください。typed failure coverage は scalar / array / schema / policy / child failure を混在させた1つの input を serial / parallel mode で共有し、安定した順序と category を検証してください。timeout、caller cancellation、明示的な `--include-raw-streams` の挙動は焦点を絞った fixture に分離します。cancellation coverage には事前 cancel 済み setup と blocking stdin reader を含め、型付き summary と parallel wait の迅速な解除を決定的に検証します。 + batch `--json-summary` coverage は、assertion を明確に保てる場合、副作用なし dispatch と成功時の JSON / 1 row NDJSON / JSON に見える text projection を1つの seeded fixture で共有し、case-insensitive な format 値、command 固有の JSON format、alias が内部追加する JSON default も含めて child parser と batch classification の drift を防いでください。serialized output の枯渇は escape を多く含む大きな payload の境界 fixture として分離します。test 側で `result` / `results` と raw `stdout` の有無を明示し、`output_chars` を実際に捕捉した stream 長と比較し、input cap までの malformed line 反復も検証して、syntax sniffing や envelope overhead が transport 上限を迂回しないようにしてください。typed failure coverage は scalar / array / schema / policy / child failure を混在させた1つの input を serial / parallel mode で共有し、安定した順序と category を検証してください。timeout、caller cancellation、明示的な `--include-raw-streams` の挙動は焦点を絞った fixture に分離します。cancellation coverage には事前 cancel 済み setup、先行 failure 後と parallel item 準備後の cancellation、blocking stdin reader を含め、型付き summary、迅速な wait 解除、同時に読み取った line が次の batch invocation 用に保持されることを決定的に検証します。 invalid scalar input だけが異なる argument-validation variant は、case ごとの state や discovery identity が不要なら1つの database fixture を共有し、fact 内で反復してください。 `files` の positional glob coverage は1つの indexed-file fixture を共有し、quote された shell 引数が CLI に渡す token と同じ `*`、`?`、recursive `**` pattern を fact 内で反復してください。 excerpt の focus coverage は、line-only 時の先頭側 window、focus-length の依存関係、focus-column の範囲検証を1つの indexed fixture で共有してください。focus-column の zero / non-numeric value も1つの indexed Markdown fixture を再利用してください。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs index 8e0ad80f0..796976d44 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; using System.Text.Json.Nodes; @@ -11,8 +12,11 @@ namespace CodeIndex.Cli; public static partial class QueryCommandRunner { private const int BatchMaxCapturedOutputChars = JsonEnvelopeWrapper.MaxCapturedOutputChars; + private static readonly ConditionalWeakTable s_batchInputPumps = new(); internal static Action? BatchParallelCommandStartedForTesting { get; set; } internal static Action? BatchParallelCommandCompletedForTesting { get; set; } + internal static Action? BatchInputLineReadForTesting { get; set; } + internal static Action? BatchParallelItemPreparedForTesting { get; set; } public static int RunBatch( string[] cmdArgs, @@ -202,9 +206,55 @@ public static int RunBatch( var commandFailures = 0; var outputLimitReached = false; var inputLimitReached = false; - while (TryReadBatchLine(Console.In, out var line, out var lineExceededLimit)) + var batchInput = GetBatchInputPump(Console.In); + while (true) { + BatchPumpedLine? pumpedLine; + try + { + pumpedLine = batchInput.ReadAsync(cancellationToken) + .AsTask() + .GetAwaiter() + .GetResult(); + } + catch (OperationCanceledException) when ( + jsonSummary + && cancellationToken.IsCancellationRequested) + { + firstFailure = CommandExitCodes.CancelledBySignal; + break; + } + if (pumpedLine is null) + break; + + var line = pumpedLine.Value.Line; + var lineExceededLimit = pumpedLine.Value.ExceededLimit; lineNumber++; + BatchInputLineReadForTesting?.Invoke(lineNumber); + if (cancellationToken.IsCancellationRequested) + { + if (!jsonSummary) + cancellationToken.ThrowIfCancellationRequested(); + + firstFailure = CommandExitCodes.CancelledBySignal; + if (lineExceededLimit || !string.IsNullOrWhiteSpace(line)) + { + var lineError = BuildBatchCancellationLineError(lineNumber); + if (!WriteBatchLineErrorJson(lineNumber, lineError, jsonOutput!)) + { + WriteBatchOutputLimitErrorJson( + lineNumber, + commandName: null, + CommandExitCodes.CancelledBySignal, + maxOutputChars, + jsonOutput!); + outputLimitReached = true; + } + lineErrors++; + } + break; + } + if (lineNumber > maxInputLines) { var lineError = new BatchLineError( @@ -258,28 +308,6 @@ public static int RunBatch( if (string.IsNullOrWhiteSpace(line)) continue; - if (cancellationToken.IsCancellationRequested) - { - if (!jsonSummary) - cancellationToken.ThrowIfCancellationRequested(); - - var lineError = BuildBatchCancellationLineError(lineNumber); - if (!WriteBatchLineErrorJson(lineNumber, lineError, jsonOutput!)) - { - WriteBatchOutputLimitErrorJson( - lineNumber, - commandName: null, - CommandExitCodes.CancelledBySignal, - maxOutputChars, - jsonOutput!); - outputLimitReached = true; - } - lineErrors++; - if (firstFailure == CommandExitCodes.Success) - firstFailure = CommandExitCodes.CancelledBySignal; - break; - } - if (!TryParseBatchLine(line, lineNumber, jsonOptions, !jsonSummary, out var commandName, out var subArgs, out var parseExitCode, out var parseError)) { if (jsonSummary) @@ -335,7 +363,10 @@ public static int RunBatch( break; } if (batchResult.CancellationObserved) + { + firstFailure = CommandExitCodes.CancelledBySignal; break; + } } if (jsonSummary) @@ -443,8 +474,11 @@ private static int RunBatchParallel( var outputLimitReached = false; var inputLimitReached = false; var cancellationObserved = false; - var producerReadInProgress = 0; using var stopProducing = new CancellationTokenSource(); + using var producerCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + stopProducing.Token); + var batchInput = GetBatchInputPump(Console.In); var input = Channel.CreateBounded( new BoundedChannelOptions(1) { @@ -457,7 +491,7 @@ private static int RunBatchParallel( Console.SetError(stderrRouter); try { - var producer = Task.Run(() => + var producer = Task.Run(async () => { try { @@ -468,25 +502,34 @@ private static int RunBatchParallel( cancellationObserved = true; break; } - string? line; - bool lineExceededLimit; - Volatile.Write(ref producerReadInProgress, 1); - try - { - if (!TryReadBatchLine(Console.In, out line, out lineExceededLimit)) - break; - } - finally - { - Volatile.Write(ref producerReadInProgress, 0); - } + + var pumpedLine = await batchInput.ReadAsync(producerCancellation.Token) + .ConfigureAwait(false); + if (pumpedLine is null) + break; + + lineNumber++; + BatchInputLineReadForTesting?.Invoke(lineNumber); if (cancellationToken.IsCancellationRequested) { cancellationObserved = true; + if (pumpedLine.Value.ExceededLimit + || !string.IsNullOrWhiteSpace(pumpedLine.Value.Line)) + { + await input.Writer.WriteAsync( + new BatchPendingItem( + lineNumber, + null, + [], + BuildBatchCancellationLineError(lineNumber), + Terminal: true), + stopProducing.Token) + .ConfigureAwait(false); + lineErrors++; + } break; } - lineNumber++; if (lineNumber > maxInputLines) { var lineError = new BatchLineError( @@ -495,18 +538,16 @@ private static int RunBatchParallel( Hint: "Split the request into smaller batch invocations.", ErrorCode: CommandErrorCodes.UsageError, Category: "batch_input_line_limit"); - input.Writer.WriteAsync( + await input.Writer.WriteAsync( new BatchPendingItem(lineNumber, null, [], lineError, Terminal: true), stopProducing.Token) - .AsTask() - .GetAwaiter() - .GetResult(); + .ConfigureAwait(false); lineErrors++; inputLimitReached = true; break; } - if (lineExceededLimit) + if (pumpedLine.Value.ExceededLimit) { var lineError = new BatchLineError( $"batch line {lineNumber} exceeds the {BatchMaxLineChars} character limit.", @@ -514,37 +555,18 @@ private static int RunBatchParallel( Hint: "Split the command across smaller arguments or reduce the input record.", ErrorCode: CommandErrorCodes.UsageError, Category: "batch_input_line_length_limit"); - input.Writer.WriteAsync( + await input.Writer.WriteAsync( new BatchPendingItem(lineNumber, null, [], lineError, Terminal: false), stopProducing.Token) - .AsTask() - .GetAwaiter() - .GetResult(); + .ConfigureAwait(false); lineErrors++; continue; } + var line = pumpedLine.Value.Line; if (string.IsNullOrWhiteSpace(line)) continue; - if (cancellationToken.IsCancellationRequested) - { - cancellationObserved = true; - input.Writer.WriteAsync( - new BatchPendingItem( - lineNumber, - null, - [], - BuildBatchCancellationLineError(lineNumber), - Terminal: true), - stopProducing.Token) - .AsTask() - .GetAwaiter() - .GetResult(); - lineErrors++; - break; - } - BatchPendingItem item; if (!TryParseBatchLine( line, @@ -575,14 +597,19 @@ private static int RunBatchParallel( commandsProcessed++; } - input.Writer.WriteAsync(item, stopProducing.Token) - .AsTask() - .GetAwaiter() - .GetResult(); + BatchParallelItemPreparedForTesting?.Invoke(lineNumber); + await input.Writer.WriteAsync(item, stopProducing.Token) + .ConfigureAwait(false); } input.Writer.TryComplete(); } + catch (OperationCanceledException) when (producerCancellation.IsCancellationRequested) + { + if (cancellationToken.IsCancellationRequested) + cancellationObserved = true; + input.Writer.TryComplete(); + } catch (Exception ex) { input.Writer.TryComplete(ex); @@ -625,6 +652,11 @@ private static int RunBatchParallel( var result = resultTask.GetAwaiter().GetResult(); if (item.Error is not null) { + if (item.Error.ExitCode is CommandExitCodes.CancelledBySignal + or CommandExitCodes.LegacyInterrupted) + { + cancellationObserved = true; + } if (firstFailure == CommandExitCodes.Success) firstFailure = item.Error.ExitCode; @@ -667,6 +699,11 @@ private static int RunBatchParallel( { if (result.ExitCode != CommandExitCodes.Success) { + if (result.ExitCode is CommandExitCodes.CancelledBySignal + or CommandExitCodes.LegacyInterrupted) + { + cancellationObserved = true; + } commandFailures++; if (firstFailure == CommandExitCodes.Success) firstFailure = result.ExitCode; @@ -694,21 +731,12 @@ private static int RunBatchParallel( if (active.Count > 0 && active.Peek().Result.IsCompleted) continue; - using var waitCancellation = CancellationTokenSource.CreateLinkedTokenSource( - cancellationToken); + using var waitCancellation = new CancellationTokenSource(); var waitForInput = input.Reader.WaitToReadAsync(waitCancellation.Token).AsTask(); if (active.Count == 0) { - try - { - if (!waitForInput.GetAwaiter().GetResult()) - break; - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - cancellationObserved = true; + if (!waitForInput.GetAwaiter().GetResult()) break; - } continue; } @@ -726,18 +754,6 @@ private static int RunBatchParallel( { } } - else - { - try - { - _ = waitForInput.GetAwaiter().GetResult(); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - cancellationObserved = true; - active.Peek().Result.GetAwaiter().GetResult(); - } - } } } catch @@ -755,24 +771,13 @@ private static int RunBatchParallel( } } - if (producer.IsCompleted) + try { - try - { - producer.GetAwaiter().GetResult(); - } - catch - { - // Preserve the first failure from the consumer or ordered worker. - } + producer.GetAwaiter().GetResult(); } - else + catch { - _ = producer.ContinueWith( - static completedProducer => _ = completedProducer.Exception, - CancellationToken.None, - TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnFaulted, - TaskScheduler.Default); + // Preserve the first failure from the consumer or ordered worker. } throw; } @@ -784,31 +789,17 @@ private static int RunBatchParallel( active.Dequeue().Result.GetAwaiter().GetResult(); } - var producerBlockedOnInput = cancellationObserved - && Volatile.Read(ref producerReadInProgress) != 0 - && !producer.IsCompleted; - if (producerBlockedOnInput) + try { - _ = producer.ContinueWith( - static completedProducer => _ = completedProducer.Exception, - CancellationToken.None, - TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnFaulted, - TaskScheduler.Default); + producer.GetAwaiter().GetResult(); } - else + catch (OperationCanceledException) when ( + outputLimitReached + && !cancellationToken.IsCancellationRequested) { - try - { - producer.GetAwaiter().GetResult(); - } - catch (OperationCanceledException) when ( - outputLimitReached - && !cancellationToken.IsCancellationRequested) - { - } } - if (cancellationObserved && firstFailure == CommandExitCodes.Success) + if (cancellationObserved) firstFailure = CommandExitCodes.CancelledBySignal; WriteBatchSummaryJson( @@ -1830,6 +1821,64 @@ private sealed record BatchPendingItem( BatchLineError? Error, bool Terminal); + private static BatchInputPump GetBatchInputPump(TextReader reader) + => s_batchInputPumps.GetValue(reader, static value => new BatchInputPump(value)); + + private sealed class BatchInputPump + { + private readonly Channel _lines; + + public BatchInputPump(TextReader reader) + { + _lines = Channel.CreateBounded( + new BoundedChannelOptions(1) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = false, + SingleWriter = true, + }); + + // The pump is the sole owner of this reader. A cancelled batch only + // cancels its channel read, so an in-flight line remains available + // to the next batch invocation instead of being consumed by an + // orphaned per-invocation producer. + _ = Task.Run(() => Pump(reader)); + } + + public async ValueTask ReadAsync(CancellationToken cancellationToken) + { + try + { + return await _lines.Reader.ReadAsync(cancellationToken).ConfigureAwait(false); + } + catch (ChannelClosedException) when (_lines.Reader.Completion.IsCompletedSuccessfully) + { + return null; + } + } + + private void Pump(TextReader reader) + { + try + { + while (TryReadBatchLine(reader, out var line, out var exceededLimit)) + { + _lines.Writer.WriteAsync(new BatchPumpedLine(line, exceededLimit)) + .AsTask() + .GetAwaiter() + .GetResult(); + } + _lines.Writer.TryComplete(); + } + catch (Exception ex) + { + _lines.Writer.TryComplete(ex); + } + } + } + + private readonly record struct BatchPumpedLine(string? Line, bool ExceededLimit); + private sealed record BatchParallelCommandResult( int ExitCode, string Stdout, diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4871Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4871Tests.cs index 1e391b5f7..6b42a361a 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4871Tests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4871Tests.cs @@ -284,7 +284,109 @@ public void RunBatch_SetupCancellationUsesTypedSummaryInSerialAndParallel_Issue4 } [Fact] - public void RunBatch_ParallelCancellationInterruptsBlockedInputWait_Issue4871() + public void RunBatch_CancellationAfterPriorFailureUsesLineEnvelopeAndCancelledSummary_Issue4871() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_cancel_after_failure_4871"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + + foreach (var batchArgs in new[] + { + new[] { "--db", dbPath, "--json-summary" }, + new[] { "--db", dbPath, "--json-summary", "--parallel", "2" }, + }) + { + using var cancellation = new CancellationTokenSource(); + QueryCommandRunner.BatchInputLineReadForTesting = lineNumber => + { + if (lineNumber == 2) + cancellation.Cancel(); + }; + + try + { + var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + "42\n[1]\n", + () => QueryCommandRunner.RunBatch( + batchArgs, + _jsonOptions, + cancellationToken: cancellation.Token)); + var lines = ParseJsonLines(stdout); + try + { + Assert.Equal(CommandExitCodes.CancelledBySignal, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal(3, lines.Count); + Assert.Equal( + "invalid_batch_input_shape", + lines[0].RootElement.GetProperty("error").GetProperty("category").GetString()); + Assert.Equal( + "batch_cancelled", + lines[1].RootElement.GetProperty("error").GetProperty("category").GetString()); + Assert.Equal( + CommandExitCodes.CancelledBySignal, + lines[2].RootElement.GetProperty("exit_code").GetInt32()); + Assert.Equal( + "batch_cancelled", + lines[2].RootElement.GetProperty("error").GetProperty("category").GetString()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + finally + { + QueryCommandRunner.BatchInputLineReadForTesting = null; + } + } + } + + [Fact] + public void RunBatch_ParallelCancellationDrainsItemPreparedDuringCancellation_Issue4871() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_cancel_prepared_item_4871"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var cancellation = new CancellationTokenSource(); + QueryCommandRunner.BatchParallelItemPreparedForTesting = _ => cancellation.Cancel(); + + try + { + var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + "[\"status\",\"--json\"]\n", + () => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "2"], + _jsonOptions, + cancellationToken: cancellation.Token)); + var lines = ParseJsonLines(stdout); + try + { + Assert.Equal(CommandExitCodes.CancelledBySignal, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal(2, lines.Count); + Assert.Equal(1, lines[0].RootElement.GetProperty("line").GetInt32()); + Assert.Equal( + "batch_cancelled", + lines[0].RootElement.GetProperty("error").GetProperty("category").GetString()); + Assert.Equal( + CommandExitCodes.CancelledBySignal, + lines[1].RootElement.GetProperty("exit_code").GetInt32()); + Assert.Equal(1, lines[1].RootElement.GetProperty("commands_processed").GetInt32()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + finally + { + QueryCommandRunner.BatchParallelItemPreparedForTesting = null; + } + } + + [Fact] + public void RunBatch_ParallelCancellationInterruptsBlockedInputAndPreservesNextLine_Issue4871() { using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_blocked_input_cancel_4871"); var dbPath = TestProjectHelper.CreateProjectDb(project.Root); @@ -322,6 +424,7 @@ public void RunBatch_ParallelCancellationInterruptsBlockedInputWait_Issue4871() }); int exitCode; + int nextExitCode; try { exitCode = QueryCommandRunner.RunBatch( @@ -330,6 +433,10 @@ public void RunBatch_ParallelCancellationInterruptsBlockedInputWait_Issue4871() cancellationToken: cancellation.Token); batchCompleted.Set(); Assert.True(cancellationTaskCompleted.Wait(TimeSpan.FromSeconds(5))); + reader.Release(); + nextExitCode = QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary"], + _jsonOptions); } finally { @@ -341,16 +448,25 @@ public void RunBatch_ParallelCancellationInterruptsBlockedInputWait_Issue4871() Assert.Null(cancellationFailure); Assert.Equal(0, Volatile.Read(ref forcedReaderRelease)); Assert.Equal(CommandExitCodes.CancelledBySignal, exitCode); + Assert.Equal(CommandExitCodes.UsageError, nextExitCode); Assert.Equal(string.Empty, capture.Error!.ToString()); var lines = ParseJsonLines(capture.Out!.ToString() ?? string.Empty); try { - var summary = Assert.Single(lines).RootElement; - Assert.Equal("batch_summary", summary.GetProperty("record").GetString()); - Assert.Equal(CommandExitCodes.CancelledBySignal, summary.GetProperty("exit_code").GetInt32()); + Assert.Equal(3, lines.Count); + var cancelledSummary = lines[0].RootElement; + Assert.Equal("batch_summary", cancelledSummary.GetProperty("record").GetString()); + Assert.Equal( + CommandExitCodes.CancelledBySignal, + cancelledSummary.GetProperty("exit_code").GetInt32()); Assert.Equal( "batch_cancelled", - summary.GetProperty("error").GetProperty("category").GetString()); + cancelledSummary.GetProperty("error").GetProperty("category").GetString()); + Assert.Equal("batch_error", lines[1].RootElement.GetProperty("record").GetString()); + Assert.Equal( + "invalid_batch_input_shape", + lines[1].RootElement.GetProperty("error").GetProperty("category").GetString()); + Assert.Equal("batch_summary", lines[2].RootElement.GetProperty("record").GetString()); } finally { @@ -361,14 +477,21 @@ public void RunBatch_ParallelCancellationInterruptsBlockedInputWait_Issue4871() private sealed class BlockingBatchReader : TextReader { + private const string DeferredLine = "42\n"; private readonly ManualResetEventSlim _entered = new(); private readonly ManualResetEventSlim _released = new(); + private int _position; public override int Read() { - _entered.Set(); - _released.Wait(); - return -1; + if (_position == 0) + { + _entered.Set(); + _released.Wait(); + } + if (_position >= DeferredLine.Length) + return -1; + return DeferredLine[_position++]; } public bool WaitUntilRead(TimeSpan timeout)