From 0b0c1a5e48ed7553c70156381657939482561959 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 6 Jun 2026 22:29:40 +0900 Subject: [PATCH 1/3] Fix batch argument length limit for #3231 --- DEVELOPER_GUIDE.md | 4 +- USER_GUIDE.md | 8 +-- changelog.d/unreleased/3231.fixed.md | 18 +++++++ src/CodeIndex/Cli/QueryCommandRunner.cs | 9 +++- .../QueryCommandRunnerTests.cs | 50 +++++++++++++++++++ 5 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 changelog.d/unreleased/3231.fixed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index ea2d029333..76b90e457e 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -217,7 +217,7 @@ Interactive terminal controls are allowed only when stdout is not redirected or Query commands that accept path filters (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `map`, `inspect`, `deps`, `impact`, `unused`, `hotspots`, and `validate`) expand `--project` into the matching project directory glob before hitting `DbReader`, so all existing SQL path predicates keep working. `index --project` expands to the files under the selected project directory and reuses the existing `--files` update path, but rejects expansions above 65,536 files for one project or 131,072 unique files across all requested projects with an explicit-files recovery hint. -`cdidx batch` is a CLI-side query loop for editor integrations and scripts that need several query commands against the same DB without spawning `cdidx` repeatedly. It opens one `DbContext` / `DbReader`, reads newline-delimited JSON string arrays from stdin, and dispatches only query commands through the existing `QueryCommandRunner` paths so output and validation stay identical to the standalone command shape. +`cdidx batch` is a CLI-side query loop for editor integrations and scripts that need several query commands against the same DB without spawning `cdidx` repeatedly. It opens one `DbContext` / `DbReader`, reads newline-delimited JSON string arrays from stdin, caps each decoded string argument at 8,192 characters, and dispatches only query commands through the existing `QueryCommandRunner` paths so output and validation stay identical to the standalone command shape. Editor integrations can request standard location shapes directly. `definition`, `references`, `search`, `find`, and `validate` accept `--format `; `lsp` emits LSP `Location` arrays, `qf` emits Vim quickfix lines, and `sarif` emits SARIF 2.1.0. `goto ` returns the single unambiguous definition as one LSP `Location`, while `goto --all ` returns all matching locations. @@ -2355,7 +2355,7 @@ override が文書化されていない限り ANSI/progress control を抑止す path filter を受け付ける query コマンド（`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `map`, `inspect`, `deps`, `impact`, `unused`, `hotspots`, `validate`）は、`--project` を対応する project directory glob に展開してから `DbReader` に渡す。これにより既存の SQL path predicate をそのまま利用できる。`index --project` は選択された project directory 配下のファイルに展開し、既存の `--files` 更新経路を再利用する。ただし 1 project で 65,536 files、requested projects 全体で 131,072 unique files を超える展開は拒否し、明示的な `--files` を使う recovery hint を返す。 -`cdidx batch` は、同じ DB に複数の query command を投げる editor integration や script 向けの CLI 側 query loop である。1 つの `DbContext` / `DbReader` を開き、stdin から newline-delimited JSON 文字列配列を読み、query command だけを既存の `QueryCommandRunner` 経路へ dispatch するため、出力と validation は単発コマンドと同じ形を保つ。 +`cdidx batch` は、同じ DB に複数の query command を投げる editor integration や script 向けの CLI 側 query loop である。1 つの `DbContext` / `DbReader` を開き、stdin から newline-delimited JSON 文字列配列を読み、デコード後の各文字列引数を 8,192 文字に制限し、query command だけを既存の `QueryCommandRunner` 経路へ dispatch するため、出力と validation は単発コマンドと同じ形を保つ。 editor integration は標準的な location 形状を直接要求できる。`definition`、`references`、`search`、`find`、`validate` は `--format ` を受け付け、`lsp` は LSP `Location` 配列、`qf` は Vim quickfix 行、`sarif` は SARIF 2.1.0 を出力する。`goto ` は曖昧でない単一定義を 1 つの LSP `Location` として返し、`goto --all ` は一致する全 location を返す。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 623ade2668..8871600c9b 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -927,8 +927,9 @@ For scripts or editor integrations that need several queries against the same index, `cdidx batch --db ` keeps one SQLite connection open and reads one JSON string array per stdin line. Each array starts with a query command name, followed by that command's normal arguments. Each stdin line is capped at -1,048,576 characters, and each command can carry at most 256 arguments after -the command name: +1,048,576 characters, each decoded string argument is capped at 8,192 +characters, and each command can carry at most 256 arguments after the command +name: ```bash printf '%s\n' \ @@ -3221,7 +3222,8 @@ cdidx search "authenticate" --json --verbose 同じインデックスに対して複数の query を投げる script や editor integration では、 `cdidx batch --db ` を使うと 1 つの SQLite connection を開いたまま処理できます。 stdin の各行は JSON 文字列配列で、先頭に query command 名、その後ろに通常の引数を並べます。 -各 stdin 行は 1,048,576 文字まで、各 command は command 名の後ろに最大 256 引数までです: +各 stdin 行は 1,048,576 文字まで、デコード後の各文字列引数は 8,192 文字まで、 +各 command は command 名の後ろに最大 256 引数までです: ```bash printf '%s\n' \ diff --git a/changelog.d/unreleased/3231.fixed.md b/changelog.d/unreleased/3231.fixed.md new file mode 100644 index 0000000000..e09ce19e97 --- /dev/null +++ b/changelog.d/unreleased/3231.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3231 +affected: + - src/CodeIndex/Cli/QueryCommandRunner.cs + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs + - USER_GUIDE.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Batch query arguments are now individually capped (#3231)** — `cdidx batch` rejects any decoded JSON string argument longer than 8,192 characters before dispatching the query command. + +## 日本語 + +- **batch query の各引数に個別上限を追加しました (#3231)** — `cdidx batch` は、デコード後の JSON 文字列引数が 8,192 文字を超える場合、query command へ dispatch する前に拒否します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index fa49fe9ccb..2c637fd52d 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -28,6 +28,7 @@ public static class QueryCommandRunner internal const int MaxWorkspaceDependencyDatabasePairCount = MaxWorkspaceDependencyDatabaseCount * (MaxWorkspaceDependencyDatabaseCount - 1); internal const int BatchMaxLineChars = 1024 * 1024; internal const int BatchMaxArgumentCount = 256; + internal const int BatchMaxArgumentChars = 8192; internal const int BatchMaxJsonDepth = 32; internal const string DefaultLimitEnvironmentVariable = "CDIDX_DEFAULT_LIMIT"; internal const string DefaultSnippetLinesEnvironmentVariable = "CDIDX_DEFAULT_SNIPPET_LINES"; @@ -429,7 +430,13 @@ private static bool TryParseBatchLine(string line, int lineNumber, out string co Console.Error.WriteLine($"Error: batch line {lineNumber} must contain only strings."); return false; } - values.Add(element.GetString() ?? string.Empty); + var value = element.GetString() ?? string.Empty; + if (value.Length > BatchMaxArgumentChars) + { + Console.Error.WriteLine($"Error: batch line {lineNumber} argument {values.Count + 1} exceeds the {BatchMaxArgumentChars} character limit."); + return false; + } + values.Add(value); } commandName = values[0]; diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index 0d91d9dda5..c878537b9d 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -778,6 +778,56 @@ public void RunBatch_ArgumentCountExceedsLimit_ReturnsUsageError_Issue2891() } } + [Fact] + public void RunBatch_ArgumentAtLimitParsesBeforeDispatch_Issue3231() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_batch_arg_at_limit"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var commandName = new string('x', QueryCommandRunner.BatchMaxArgumentChars); + var input = JsonSerializer.Serialize(new[] { commandName }) + "\n"; + + var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + input, + () => QueryCommandRunner.RunBatch(["--db", dbPath], _jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stdout); + Assert.Contains("batch only supports query commands", stderr); + Assert.DoesNotContain("character limit", stderr); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void RunBatch_ArgumentExceedsLimitReturnsUsageError_Issue3231() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_batch_arg_too_long"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var commandName = new string('x', QueryCommandRunner.BatchMaxArgumentChars + 1); + var input = JsonSerializer.Serialize(new[] { commandName }) + "\n"; + + var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + input, + () => QueryCommandRunner.RunBatch(["--db", dbPath], _jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stdout); + Assert.Contains($"argument 1 exceeds the {QueryCommandRunner.BatchMaxArgumentChars} character limit", stderr); + Assert.DoesNotContain("batch only supports query commands", stderr); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunBatch_TooDeepJsonLine_ReturnsUsageError_Issue3022() { From b4566bfb5cba045b2bf5d0ba830f54b746fe51fa Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 6 Jun 2026 22:32:09 +0900 Subject: [PATCH 2/3] Fix find query length limit for #3100 --- USER_GUIDE.md | 4 ++-- changelog.d/unreleased/3100.fixed.md | 18 ++++++++++++++ src/CodeIndex/Cli/ConsoleUi.cs | 2 +- src/CodeIndex/Cli/QueryCommandRunner.cs | 7 ++++++ .../QueryCommandRunnerSearchTests.cs | 24 +++++++++++++++++++ 5 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 changelog.d/unreleased/3100.fixed.md diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 8871600c9b..390bf6686c 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1068,7 +1068,7 @@ cdidx find "graph table" --path src/CodeIndex/Cli/QueryCommandRunner.cs cdidx find "Graph Table" --path src/CodeIndex/Cli/QueryCommandRunner.cs --exact --before 1 --after 1 --json ``` -`find` fills the gap between repo-wide `search` and line-number-based `excerpt`: when you already know the target file, it returns matching line numbers, columns, and short surrounding context from the indexed file without falling back to raw-text tools. +`find` fills the gap between repo-wide `search` and line-number-based `excerpt`: when you already know the target file, it returns matching line numbers, columns, and short surrounding context from the indexed file without falling back to raw-text tools. The query text is capped at 1,000 characters, matching `search`. ### List files @@ -3360,7 +3360,7 @@ cdidx find "graph table" --path src/CodeIndex/Cli/QueryCommandRunner.cs cdidx find "Graph Table" --path src/CodeIndex/Cli/QueryCommandRunner.cs --exact --before 1 --after 1 --json ``` -`find` は、リポジトリ全体を対象にする `search` と、行番号が必要な `excerpt` の間を埋めるコマンドです。対象ファイルが既に分かっているときに、raw text ツールへ戻らずに、インデックス済みファイルから一致行番号・列番号・短い前後文脈を返します。 +`find` は、リポジトリ全体を対象にする `search` と、行番号が必要な `excerpt` の間を埋めるコマンドです。対象ファイルが既に分かっているときに、raw text ツールへ戻らずに、インデックス済みファイルから一致行番号・列番号・短い前後文脈を返します。query text は `search` と同じく 1,000 文字までです。 ### ファイル一覧 diff --git a/changelog.d/unreleased/3100.fixed.md b/changelog.d/unreleased/3100.fixed.md new file mode 100644 index 0000000000..dd52c7b736 --- /dev/null +++ b/changelog.d/unreleased/3100.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3100 +affected: + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs + - USER_GUIDE.md +--- + +## English + +- **`find` now enforces the shared query length limit (#3100)** — `cdidx find` and `cdidx find --count` reject query text above 1,000 characters before opening the database reader. + +## 日本語 + +- **`find` が共通の query 長上限を適用するようになりました (#3100)** — `cdidx find` と `cdidx find --count` は、query text が 1,000 文字を超える場合、database reader を開く前に拒否します。 diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index fe66e6528e..5b123da62b 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -1031,7 +1031,7 @@ private static void PrintFlagReference(Action WriteHelpLine) Console.WriteLine(" --limit , --top Max results to return (default: 20)"); Console.WriteLine(" --lang Filter by language (aliases: bat, cmd, cshtml, razor, ts, tsx, cts, mts)"); Console.WriteLine(" --path Restrict matches to glob-style path patterns (* and ?)"); - WriteHelpLine($" --query Pass a query literal, useful when the query starts with '-' (`search` max {QueryLimits.MaxQueryLength} chars)"); + WriteHelpLine($" --query Pass a query literal, useful when the query starts with '-' (`search`/`find` max {QueryLimits.MaxQueryLength} chars)"); Console.WriteLine(" --exclude-path Exclude glob-style path patterns (* and ?) (repeatable)"); Console.WriteLine(" --exclude-tests Exclude likely test files"); Console.WriteLine(" --include-generated Include generated files in query results"); diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index 2c637fd52d..906856353a 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -2727,6 +2727,13 @@ public static int RunFind(string[] cmdArgs, JsonSerializerOptions jsonOptions) Console.Error.WriteLine(FindUsage); return CommandExitCodes.UsageError; } + if (options.Query.Length > QueryLimits.MaxQueryLength) + { + Console.Error.WriteLine($"Error: {QueryLimits.FormatQueryTooLongError()}"); + Console.Error.WriteLine("Hint: Shorten the find text or split generated input into smaller queries before running `cdidx find`."); + Console.Error.WriteLine(FindUsage); + return CommandExitCodes.UsageError; + } if (options.PathPatterns.Count == 0) { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs index 4563f6fd13..71dbd4be38 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs @@ -3781,6 +3781,30 @@ public void RunFind_MissingQueryStillReportsRequiresArgument() Assert.DoesNotContain("query cannot be empty or whitespace-only", stderr); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RunFind_QueryTooLongReturnsUsageError_Issue3100(bool countOnly) + { + var args = new List + { + new('x', QueryLimits.MaxQueryLength + 1), + "--path", + "src/**/*.cs", + }; + if (countOnly) + args.Add("--count"); + + var (exitCode, _, stderr) = CaptureConsole(() => QueryCommandRunner.RunFind( + [.. args], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Contains($"Error: {QueryLimits.FormatQueryTooLongError()}", stderr); + Assert.Contains("Hint: Shorten the find text", stderr); + Assert.Contains("Usage: cdidx find", stderr); + } + [Fact] public void RunSearch_ZeroResultsHonorsStaleAfterEnvironment() { From bd95c7e55373a32899aa8ae818c9d4afc3177b46 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 6 Jun 2026 22:44:02 +0900 Subject: [PATCH 3/3] Fix literal search query bounds for #3081 --- DEVELOPER_GUIDE.md | 7 +++ USER_GUIDE.md | 6 +++ changelog.d/unreleased/3081.fixed.md | 22 +++++++++ src/CodeIndex/Cli/QueryCommandRunner.cs | 6 +++ src/CodeIndex/Database/DbSearchReader.cs | 34 +++++++++++-- .../Database/SearchQueryLimitException.cs | 9 ++++ src/CodeIndex/Mcp/McpToolHandlers.cs | 8 +++ .../DbSearchReaderIssueTests.cs | 49 +++++++++++++++++++ .../QueryCommandRunnerSearchTests.cs | 24 +++++++++ 9 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 changelog.d/unreleased/3081.fixed.md create mode 100644 src/CodeIndex/Database/SearchQueryLimitException.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 76b90e457e..4187b3268a 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -779,6 +779,11 @@ WHERE fts_chunks MATCH 'content:authenticate' ### How the search works +Literal-safe `search` queries are bounded in the reader before FTS5 +sanitization: maximum 1000 characters and 128 whitespace terms. Keep this guard +in `DbReader` so CLI, MCP, and direct reader callers share the same failure mode; +raw `--fts` queries continue to use the raw FTS complexity limits instead. + When you run: ```sql SELECT f.path, c.start_line, c.content @@ -3011,6 +3016,8 @@ exact-match flag の互換性は [USER_GUIDE.md](USER_GUIDE.md#フラグ互換 `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files` は `--path`、繰り返し指定できる `--exclude-path`、`--exclude-tests` による絞り込みを共有します。読み取り層は tests や docs より source を優先し、`search` はシンボル名やパスがクエリと正確に一致する候補をさらに上位に出して、AIクライアントが実装ファイルへ早く到達できるようにします。 +literal-safe な `search` query は reader 層で FTS5 sanitization 前に 1000 文字、128 whitespace term へ制限します。CLI、MCP、直接 reader caller の failure mode を揃えるため、この guard は `DbReader` に置きます。raw `--fts` query は別途 raw FTS complexity limit を使います。 + `search --json` と MCP の `search` は、フルチャンクを `chunk_start_line`、`chunk_end_line`、`snippet_start_line`、`snippet_end_line`、`snippet`、`match_lines`、`highlights`、`context_before`、`context_after`、`truncated_line_count`、`dropped_match_line_count`、`truncation_context` を持つ軽量スニペットへ投影します。`--snippet-lines` で抜粋長を先に制限でき（デフォルト: 8、最大: 20）、`--max-line-width`（CLI）/ `maxLineWidth`（MCP）は `find` / `references` / `excerpt` / `inspect` と同じ共有 `LineWidthFormatter.ClampLine` 契約（デフォルト: 512、最大: 4096、`0` で切り詰め解除）で各スニペット行を最初のマッチトークン周辺にクランプするため、minified / transpiled / 生成された 1 行ファイル内の 1 ヒットで数百 KB を返さなくなります。クランプされた行はスニペットに `...(+N)...` マーカーが入り、`truncation_context.char_counts`、`truncation_context.total_chars`、`highlights[].truncated`、`highlights[].original_line_length`、`highlights[].truncated_char_counts` で AI クライアントがクランプの有無と省略文字数を検出できます。`highlights[].terms` は互換性のため distinct な term list のまま残し、`highlights[].term_occurrences` は一致ごとの `term`、1-based の `line` / `column`、`length` を記録します。exact substring search では `highlights[].literal_terms` と `highlights[].literal_term_occurrences`（MCP では camelCase）も追加され、広めの診断 token list を残したまま、要求された literal phrase だけを render できます。exact ではない記号の多い code phrase 検索では、FTS tokenization が記号を失いやすい場合に exact substring semantics で再検索できるよう、CLI JSON compact result に `exact_substring_hint`、MCP `search` に `recovery_hint` を追加します。`dropped_match_line_count` は選択された snippet window 外に落ちた一致行数を示します。 マッチ行がインデックス済みシンボル範囲内にある場合、`search --json` と MCP の `search` は任意フィールドの `enclosing_symbol_name`、`enclosing_symbol_kind`、`enclosing_symbol_start_line`、`enclosing_symbol_end_line`、`enclosing_container_name` も返します。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 390bf6686c..016f0a461a 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -876,6 +876,9 @@ Search normalizes literal FTS queries to Unicode NFC before matching. If every literal token exceeds SQLite FTS5 unicode61's 1000-character token cap, zero-result JSON includes `query_degraded_reason` and `tokens_dropped`. Index validation reports long unbroken FTS tokens as `fts_token_too_long`. +Literal-safe `search` queries are capped at 1000 characters and 128 whitespace +terms. Oversized generated input is rejected before FTS5 sanitization; split it +into smaller searches or use narrower text. Guard-aware search filters primary `search` matches by nearby literal guards: `--require-before` / `--require-after` keep matches only when the guard query appears in the selected line window, while `--reject-before` / `--reject-after` @@ -3175,6 +3178,9 @@ literal FTS クエリは照合前に Unicode NFC へ正規化されます。す token が SQLite FTS5 unicode61 の 1000 文字 token 上限を超える場合、0 件 JSON には `query_degraded_reason` と `tokens_dropped` が含まれます。index validation は長い連続 FTS token を `fts_token_too_long` として報告します。 +literal-safe な `search` query は 1000 文字、128 whitespace term までです。 +生成された大きすぎる入力は FTS5 sanitization 前に拒否されるため、小さな検索へ分割するか、 +より狭い text にしてください。 guard-aware search は primary の `search` 一致を近傍の literal guard で絞り込みます: `--require-before` / `--require-after` は指定行窓内に guard query がある場合だけ残し、 `--reject-before` / `--reject-after` は guard query がある一致を落とします。JSON の検索結果には diff --git a/changelog.d/unreleased/3081.fixed.md b/changelog.d/unreleased/3081.fixed.md new file mode 100644 index 0000000000..83293d7c38 --- /dev/null +++ b/changelog.d/unreleased/3081.fixed.md @@ -0,0 +1,22 @@ +--- +category: fixed +issues: + - 3081 +affected: + - src/CodeIndex/Database/DbSearchReader.cs + - src/CodeIndex/Database/SearchQueryLimitException.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs + - USER_GUIDE.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Literal search sanitization now rejects oversized generated input (#3081)** — literal-safe `search` queries are capped at 1,000 characters and 128 whitespace terms in the database reader before FTS5 sanitization, with CLI and MCP returning usage errors instead of falling through to database failures. + +## 日本語 + +- **literal search sanitization が大きすぎる生成入力を拒否するようになりました (#3081)** — literal-safe な `search` query は FTS5 sanitization 前の database reader 層で 1,000 文字、128 whitespace term に制限され、CLI/MCP は database failure へ落とさず usage error を返します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index 906856353a..4fa5a9ee03 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -7586,6 +7586,12 @@ private static int WithDb( Console.Error.WriteLine("Hint: narrow the search with more specific query text, --lang, --path, or --exclude-tests, or reduce pagination offset before retrying guarded search."); return CommandExitCodes.UsageError; } + catch (SearchQueryLimitException ex) + { + Console.Error.WriteLine($"Error [{CommandErrorCodes.UsageError}]: {ex.Message}"); + Console.Error.WriteLine("Hint: shorten the search text or split generated input into smaller literal queries."); + return CommandExitCodes.UsageError; + } catch (Exception ex) { if (JsonOutputFailure.TryHandle(ex, out var exitCode)) diff --git a/src/CodeIndex/Database/DbSearchReader.cs b/src/CodeIndex/Database/DbSearchReader.cs index 7080a9d212..8d0c6c3e26 100644 --- a/src/CodeIndex/Database/DbSearchReader.cs +++ b/src/CodeIndex/Database/DbSearchReader.cs @@ -11,6 +11,8 @@ public partial class DbReader { internal const int FtsUnicode61MaxTokenLength = 1000; internal const string AllTokensFilteredByLengthReason = "all_tokens_filtered_by_length"; + internal const int MaxLiteralSearchQueryLength = 1000; + internal const int MaxLiteralSearchTokenCount = 128; internal const int MaxRawFtsQueryLength = 2000; internal const int MaxRawFtsBooleanOperators = 64; internal const int MaxRawFtsNearOperators = 16; @@ -44,7 +46,7 @@ internal static string SanitizeFtsQuery(string query, bool prefix) // クエリ内のダブルクォートをエスケープし、各トークンをダブルクォートで囲む。 // ユーザー入力末尾の `*` は prefix 検索の shorthand として保持し、`auth*` で // `authenticate` を raw FTS5 構文なしに検索できるようにする。 - var tokens = query.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + var tokens = SplitLiteralSearchTokens(query); if (tokens.Length == 0) return "\"\""; return string.Join(" ", tokens.Select(token => FormatFtsToken(token, prefix))); @@ -56,7 +58,7 @@ public static FtsQueryDiagnostics AnalyzeFtsQuery(string query, bool rawQuery = return FtsQueryDiagnostics.None; var normalizedQuery = NormalizeLiteralSearchQuery(query, NormalizeQueryLanguage(lang)); - var tokens = normalizedQuery.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries) + var tokens = SplitLiteralSearchTokens(normalizedQuery) .Select(token => token.Length > 1 && token.EndsWith('*') ? token[..^1] : token) .Where(token => token.Length > 0) .ToArray(); @@ -114,6 +116,8 @@ public List Search(string query, int limit = 20, string? lang = nu return []; lang = NormalizeQueryLanguage(lang); + if (!rawQuery) + ValidateLiteralSearchQueryLength(query); var normalizedQuery = rawQuery ? query : NormalizeLiteralSearchQuery(query, lang); var coverageTokens = exact ? new List() : GetSearchCoverageTokens(normalizedQuery, rawQuery); var hasGuardFilters = guardFilters is { Count: > 0 }; @@ -408,6 +412,9 @@ public QueryCountResult CountSearchResults(string query, string? lang = null, bo if (string.IsNullOrWhiteSpace(query)) return new QueryCountResult(0, 0); + if (!rawQuery) + ValidateLiteralSearchQueryLength(query); + if (guardFilters is { Count: > 0 }) { var guardedResults = Search(query, int.MaxValue, lang, rawQuery, pathPatterns, excludePathPatterns, excludeTests, deduplicate, since, exact, prefix, visibilityRank, guardFilters: guardFilters, guardWindow: guardWindow); @@ -614,7 +621,7 @@ public bool ShouldNormalizeCSharp(SearchResult result) private static string[] BuildPrimarySearchMatchTerms(string query, string normalizedQuery, bool rawQuery, bool exact) { IEnumerable rawTerms = !exact && !rawQuery - ? normalizedQuery.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries) + ? SplitLiteralSearchTokens(normalizedQuery) : [rawQuery ? query.Trim() : normalizedQuery.Trim()]; var terms = rawTerms.Select(NormalizeGuardSearchTerm).ToList(); if (!exact && rawQuery) @@ -793,6 +800,25 @@ private static string NormalizeLiteralSearchQuery(string query, string? lang) : normalized; } + private static void ValidateLiteralSearchQueryLength(string query) + { + if (query.Length <= MaxLiteralSearchQueryLength) + return; + + throw new SearchQueryLimitException( + $"literal search query is too long ({query.Length} characters); maximum is {MaxLiteralSearchQueryLength}. Split generated input into smaller queries."); + } + + private static string[] SplitLiteralSearchTokens(string query) + { + var tokens = query.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + if (tokens.Length <= MaxLiteralSearchTokenCount) + return tokens; + + throw new SearchQueryLimitException( + $"literal search query has too many terms ({tokens.Length}); maximum is {MaxLiteralSearchTokenCount}. Split generated input into smaller queries."); + } + internal static string ValidateRawFtsQuery(string query) { if (query.Length > MaxRawFtsQueryLength) @@ -1230,7 +1256,7 @@ private static string GetSearchCoverageOrderSql(int coverageTokenCount) private static List GetSearchCoverageTokens(string query, bool rawQuery) { - var tokens = rawQuery ? ExtractRawFtsCoverageTokens(query) : query.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + var tokens = rawQuery ? ExtractRawFtsCoverageTokens(query) : SplitLiteralSearchTokens(query); if (tokens.Length <= 1) return []; diff --git a/src/CodeIndex/Database/SearchQueryLimitException.cs b/src/CodeIndex/Database/SearchQueryLimitException.cs new file mode 100644 index 0000000000..a30063fdc7 --- /dev/null +++ b/src/CodeIndex/Database/SearchQueryLimitException.cs @@ -0,0 +1,9 @@ +namespace CodeIndex.Database; + +internal sealed class SearchQueryLimitException : Exception +{ + public SearchQueryLimitException(string message) + : base(message) + { + } +} diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 33878b3411..20fcac1605 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -1272,6 +1272,10 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) { countResults = reader.Search(query, MaxLimit, lang, rawQuery, pathPatterns, excludePaths, excludeTests, deduplicate, since, exact, prefix, guardFilters: guardFilters, guardWindow: guardWindow); } + catch (SearchQueryLimitException ex) + { + return CreateToolErrorResponse(id, ex.Message); + } catch (SearchGuardCandidateLimitException ex) { return CreateToolErrorResponse(id, $"guarded search is too broad: {ex.Message} Narrow the search with more specific query text, lang/path filters, or a smaller cursor offset."); @@ -1295,6 +1299,10 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) { results = reader.Search(query, FetchLimitForEnvelope(limit), lang, rawQuery, pathPatterns, excludePaths, excludeTests, deduplicate, since, exact, prefix, cursor: cursor, guardFilters: guardFilters, guardWindow: guardWindow); } + catch (SearchQueryLimitException ex) + { + return CreateToolErrorResponse(id, ex.Message); + } catch (SearchGuardCandidateLimitException ex) { return CreateToolErrorResponse(id, $"guarded search is too broad: {ex.Message} Narrow the search with more specific query text, lang/path filters, or a smaller cursor offset."); diff --git a/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs index 691885d77c..0960623d42 100644 --- a/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs +++ b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs @@ -172,6 +172,52 @@ public void Search_GuardFiltersShareSameFocusWindowAcrossFilters_Issue3084() Assert.Equal([1, 2], result.GuardEvidence!.Select(evidence => evidence.Line).ToArray()); } + [Fact] + public void Search_LiteralQueryAtLengthLimitRuns_Issue3081() + { + var query = new string('a', DbReader.MaxLiteralSearchQueryLength); + InsertIndexedFile("src/literal-length-limit.cs", "csharp", query); + + var results = _reader.Search(query, pathPatterns: ["src/literal-length-limit.cs"], limit: 1); + + var result = Assert.Single(results); + Assert.Equal("src/literal-length-limit.cs", result.Path); + } + + [Fact] + public void Search_LiteralQueryOverLengthLimitThrows_Issue3081() + { + var query = new string('a', DbReader.MaxLiteralSearchQueryLength + 1); + + var ex = Assert.Throws(() => _reader.Search(query, pathPatterns: ["src/*.cs"], limit: 1)); + + Assert.Contains("literal search query is too long", ex.Message, StringComparison.Ordinal); + Assert.Contains(DbReader.MaxLiteralSearchQueryLength.ToString(), ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Search_LiteralTokenCountAtLimitRuns_Issue3081() + { + var query = BuildLiteralTermQuery(DbReader.MaxLiteralSearchTokenCount); + InsertIndexedFile("src/literal-token-limit.cs", "csharp", query); + + var results = _reader.Search(query, pathPatterns: ["src/literal-token-limit.cs"], limit: 1); + + var result = Assert.Single(results); + Assert.Equal("src/literal-token-limit.cs", result.Path); + } + + [Fact] + public void Search_LiteralTokenCountOverLimitThrows_Issue3081() + { + var query = BuildLiteralTermQuery(DbReader.MaxLiteralSearchTokenCount + 1); + + var ex = Assert.Throws(() => _reader.Search(query, pathPatterns: ["src/*.cs"], limit: 1)); + + Assert.Contains("literal search query has too many terms", ex.Message, StringComparison.Ordinal); + Assert.Contains(DbReader.MaxLiteralSearchTokenCount.ToString(), ex.Message, StringComparison.Ordinal); + } + private void InsertIndexedFile(string path, string lang, string content, DateTime? modified = null) { var normalized = content.Replace("\r\n", "\n"); @@ -195,6 +241,9 @@ private void InsertIndexedFile(string path, string lang, string content, DateTim }]); } + private static string BuildLiteralTermQuery(int termCount) + => string.Join(' ', Enumerable.Range(0, termCount).Select(i => $"t{i:D3}")); + public void Dispose() { _reader.Dispose(); diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs index 71dbd4be38..b98d65763e 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs @@ -1077,6 +1077,30 @@ public void RunSearch_TooLongQueryReturnsUsageError_Issue1468() } } + [Fact] + public void RunSearch_LiteralTokenCountTooHighReturnsUsageError_Issue3081() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_search_literal_terms_too_many"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var query = string.Join(' ', Enumerable.Range(0, DbReader.MaxLiteralSearchTokenCount + 1).Select(i => $"t{i:D3}")); + + var (exitCode, _, stderr) = CaptureConsole(() => QueryCommandRunner.RunSearch( + [query, "--db", dbPath], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Contains("literal search query has too many terms", stderr); + Assert.Contains("smaller literal queries", stderr); + Assert.DoesNotContain("database error:", stderr); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunSearch_RawFtsTooManyNearOperatorsReturnsUsageError() {