diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index bfaf0096cd..cf52297f4d 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1018,11 +1018,13 @@ Exact-match flag compatibility is documented in [USER_GUIDE.md](USER_GUIDE.md#fl `search`, `definition`, `references`, `callers`, `callees`, `symbols`, and `files` also share path-aware narrowing via `--path`, repeatable `--exclude-path`, and `--exclude-tests`. The read layer ranks source files ahead of tests and docs, and `search` further boosts exact symbol-name and path matches so AI clients are more likely to land on implementation files first. -`search --json` and MCP `search` project full chunks into compact match-centered snippets with `chunk_start_line`, `chunk_end_line`, `snippet_start_line`, `snippet_end_line`, `snippet`, `match_lines`, `highlights`, `context_before`, `context_after`, `truncated_line_count`, `dropped_match_line_count`, and `truncation_context`. `--snippet-lines` caps the snippet length up front (default: 8, max: 20), and `--max-line-width` (CLI) / `maxLineWidth` (MCP) clamps each individual snippet line around the first match token via the shared `LineWidthFormatter.ClampLine` contract used by `find` / `references` / `excerpt` / `inspect` (default: 512, max: 4096) so a single match inside a minified / transpiled / generated single-line file no longer returns hundreds of KB per hit. Clamped lines surface `...(+N)...` markers inside the snippet and expose `truncation_context.char_counts`, `truncation_context.total_chars`, `highlights[].truncated`, `highlights[].original_line_length`, and `highlights[].truncated_char_counts` so AI clients can detect clamping and quantify omitted characters. `highlights[].terms` remains a distinct term list for compatibility; `highlights[].term_occurrences` records every matched occurrence with `term`, 1-based `line`, 1-based `column`, and `length`. Exact substring search also adds `highlights[].literal_terms` and `highlights[].literal_term_occurrences` (camelCase in MCP) so clients can render only the requested literal phrase while preserving the broader diagnostic token list. Non-exact punctuation-heavy code-phrase searches add `exact_substring_hint` to CLI JSON compact results and `recovery_hint` to MCP `search` responses so clients can retry with exact substring semantics when FTS tokenization is likely to hide punctuation. `dropped_match_line_count` reports match lines omitted because they fell outside the selected snippet window. +`search --json` and MCP `search` project full chunks into compact match-centered snippets with `chunk_start_line`, `chunk_end_line`, `snippet_start_line`, `snippet_end_line`, `snippet`, `match_lines`, `highlights`, `context_before`, `context_after`, `truncated_line_count`, `dropped_match_line_count`, and `truncation_context`. Compact CLI rows and MCP search results also echo effective output options with `snippet_lines` / `snippetLines`, `max_line_width` / `maxLineWidth`, `exact`, `raw_fts` / `rawFts`, `literal_highlights_available` / `literalHighlightsAvailable`, and optional `literal_highlight_warning` / `literalHighlightWarning`. `--snippet-lines` caps the snippet length up front (default: 8, max: 20), and `--max-line-width` (CLI) / `maxLineWidth` (MCP) clamps each individual snippet line around the first match token via the shared `LineWidthFormatter.ClampLine` contract used by `find` / `references` / `excerpt` / `inspect` (default: 512, max: 4096) so a single match inside a minified / transpiled / generated single-line file no longer returns hundreds of KB per hit. Clamped lines surface `...(+N)...` markers inside the snippet and expose `truncation_context.char_counts`, `truncation_context.total_chars`, `highlights[].truncated`, `highlights[].original_line_length`, and `highlights[].truncated_char_counts` so AI clients can detect clamping and quantify omitted characters. `highlights[].terms` remains a distinct term list for compatibility; `highlights[].term_occurrences` records every matched occurrence with `term`, 1-based `line`, 1-based `column`, `length`, plus `visible`, `visible_column`, and `visible_length` for the portion still present in the returned snippet text after line clamping. Exact substring search also adds `highlights[].literal_terms` and `highlights[].literal_term_occurrences` (camelCase in MCP) so clients can render only the requested literal phrase while preserving the broader diagnostic token list; raw FTS rows set `literal_highlight_warning` / `literalHighlightWarning` to `literal_highlights_unavailable_raw_fts` because FTS syntax can no longer be mapped to one literal phrase. Non-exact punctuation-heavy code-phrase searches add `exact_substring_hint` to CLI JSON compact results and `recovery_hint` to MCP `search` responses so clients can retry with exact substring semantics when FTS tokenization is likely to hide punctuation. `focus_mode`, `focus_line`, `focus_column`, and `focus_reason` describe the match window selected for the snippet, while `dropped_match_line_count` and optional `next_match` report match lines omitted because they fell outside that selected snippet window. When the match line falls inside an indexed symbol range, `search --json` and MCP `search` also include optional `enclosing_symbol_name`, `enclosing_symbol_kind`, `enclosing_symbol_start_line`, `enclosing_symbol_end_line`, and `enclosing_container_name`. -`excerpt --json` includes `semantic_tokens`, a lightweight range list with 1-based start/end positions, token `type`, and `modifiers`, so IDE and LLM clients can render or post-process excerpt spans without reparsing the raw `content` string. +`find --json` remains line-delimited for repeated matches and adds bounded match-span/truncation metadata to each row: `length` reports the 1-based `column` span length, `original_line_length` reports the source line length before any line-width clamp, and `snippet_truncation_context.line_count` / `char_counts` / `total_chars` / optional `reason` describe snippet clamping. `reason` is `line_width` when `--max-line-width` elides one or more snippet lines. + +`excerpt --json` includes `semantic_tokens`, a lightweight range list with 1-based start/end positions, token `type`, and `modifiers`, so IDE and LLM clients can render or post-process excerpt spans without reparsing the raw `content` string. Excerpt rows also expose `requested_start_line`, `requested_end_line`, `effective_start_line`, `effective_end_line`, `content_truncation_reasons`, and optional `content_recovery` so clients can tell when `--max-line-width` caused `line_width_cap` and replay `cdidx excerpt ... --max-line-width 0 --json` for the omitted text. Body-bearing JSON rows use matching `body_requested_*`, `body_effective_*`, `body_content_truncation_reasons`, and `body_content_recovery` fields; body reasons include `body_line_cap` for snippet/body line caps and `body_byte_cap` for definition body byte caps. `inspect` and MCP `analyze_symbol` bundle the primary definition, nearby symbols from the same file, references, callers, callees, file metadata, workspace freshness/git metadata, and graph-support metadata into one response. When those bundled graph sections actually depend on SQL-backed reads, the payload also mirrors `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason` (plus the existing camelCase aliases on MCP responses); mixed-language bundles that only return C# / JS / etc. graph rows omit the SQL trust signal entirely. This is intended for symbol-oriented AI workflows that would otherwise need several back-to-back calls. Call graph sections remain language-aware: for unsupported languages, clients can now distinguish "unsupported" from "no hits" via `graphSupported` / `graphSupportReason`, and should prefer `search` instead of assuming graph data will exist. @@ -3176,11 +3178,13 @@ exact-match flag の互換性は [USER_GUIDE.md](USER_GUIDE.md#フラグ互換 literal-safe な `search` query は reader 層で FTS5 sanitization 前に 1000 文字、128 whitespace term へ制限します。CLI、MCP、直接 reader caller の failure mode を揃えるため、この guard は `DbReader` に置きます。raw `--fts` query は別途 raw FTS complexity limit を使います。 -`search --json` と MCP の `search` は、フルチャンクを `chunk_start_line`、`chunk_end_line`、`snippet_start_line`、`snippet_end_line`、`snippet`、`match_lines`、`highlights`、`context_before`、`context_after`、`truncated_line_count`、`dropped_match_line_count`、`truncation_context` を持つ軽量スニペットへ投影します。`--snippet-lines` で抜粋長を先に制限でき（デフォルト: 8、最大: 20）、`--max-line-width`（CLI）/ `maxLineWidth`（MCP）は `find` / `references` / `excerpt` / `inspect` と同じ共有 `LineWidthFormatter.ClampLine` 契約（デフォルト: 512、最大: 4096、`0` で切り詰め解除）で各スニペット行を最初のマッチトークン周辺にクランプするため、minified / transpiled / 生成された 1 行ファイル内の 1 ヒットで数百 KB を返さなくなります。クランプされた行はスニペットに `...(+N)...` マーカーが入り、`truncation_context.char_counts`、`truncation_context.total_chars`、`highlights[].truncated`、`highlights[].original_line_length`、`highlights[].truncated_char_counts` で AI クライアントがクランプの有無と省略文字数を検出できます。`highlights[].terms` は互換性のため distinct な term list のまま残し、`highlights[].term_occurrences` は一致ごとの `term`、1-based の `line` / `column`、`length` を記録します。exact substring search では `highlights[].literal_terms` と `highlights[].literal_term_occurrences`（MCP では camelCase）も追加され、広めの診断 token list を残したまま、要求された literal phrase だけを render できます。exact ではない記号の多い code phrase 検索では、FTS tokenization が記号を失いやすい場合に exact substring semantics で再検索できるよう、CLI JSON compact result に `exact_substring_hint`、MCP `search` に `recovery_hint` を追加します。`dropped_match_line_count` は選択された snippet window 外に落ちた一致行数を示します。 +`search --json` と MCP の `search` は、フルチャンクを `chunk_start_line`、`chunk_end_line`、`snippet_start_line`、`snippet_end_line`、`snippet`、`match_lines`、`highlights`、`context_before`、`context_after`、`truncated_line_count`、`dropped_match_line_count`、`truncation_context` を持つ軽量スニペットへ投影します。compact CLI row と MCP search result は有効な出力オプションも `snippet_lines` / `snippetLines`、`max_line_width` / `maxLineWidth`、`exact`、`raw_fts` / `rawFts`、`literal_highlights_available` / `literalHighlightsAvailable`、任意の `literal_highlight_warning` / `literalHighlightWarning` として返します。`--snippet-lines` で抜粋長を先に制限でき（デフォルト: 8、最大: 20）、`--max-line-width`（CLI）/ `maxLineWidth`（MCP）は `find` / `references` / `excerpt` / `inspect` と同じ共有 `LineWidthFormatter.ClampLine` 契約（デフォルト: 512、最大: 4096、`0` で切り詰め解除）で各スニペット行を最初のマッチトークン周辺にクランプするため、minified / transpiled / 生成された 1 行ファイル内の 1 ヒットで数百 KB を返さなくなります。クランプされた行はスニペットに `...(+N)...` マーカーが入り、`truncation_context.char_counts`、`truncation_context.total_chars`、`highlights[].truncated`、`highlights[].original_line_length`、`highlights[].truncated_char_counts` で AI クライアントがクランプの有無と省略文字数を検出できます。`highlights[].terms` は互換性のため distinct な term list のまま残し、`highlights[].term_occurrences` は一致ごとの `term`、1-based の `line` / `column`、`length` に加えて、行クランプ後に返却 snippet text 内へ残っている部分を示す `visible`、`visible_column`、`visible_length` を記録します。exact substring search では `highlights[].literal_terms` と `highlights[].literal_term_occurrences`（MCP では camelCase）も追加され、広めの診断 token list を残したまま、要求された literal phrase だけを render できます。raw FTS row は FTS 構文を単一の literal phrase へ対応付けられないため、`literal_highlight_warning` / `literalHighlightWarning` に `literal_highlights_unavailable_raw_fts` を設定します。exact ではない記号の多い code phrase 検索では、FTS tokenization が記号を失いやすい場合に exact substring semantics で再検索できるよう、CLI JSON compact result に `exact_substring_hint`、MCP `search` に `recovery_hint` を追加します。`focus_mode`、`focus_line`、`focus_column`、`focus_reason` は snippet に選ばれた match window を説明し、`dropped_match_line_count` と任意の `next_match` は選択された snippet window 外に落ちた一致行を示します。 マッチ行がインデックス済みシンボル範囲内にある場合、`search --json` と MCP の `search` は任意フィールドの `enclosing_symbol_name`、`enclosing_symbol_kind`、`enclosing_symbol_start_line`、`enclosing_symbol_end_line`、`enclosing_container_name` も返します。 -`excerpt --json` は 1-based の開始/終了位置、token `type`、`modifiers` を持つ軽量 range list の `semantic_tokens` を返すため、IDE や LLM クライアントは生の `content` 文字列を再パースせずに抜粋範囲を描画・後処理できます。 +`find --json` は繰り返し一致でも line-delimited のまま維持し、各 row に bounded な match span / truncation metadata を追加します。`length` は 1-based の `column` から始まる一致長、`original_line_length` は行幅クランプ前のソース行長、`snippet_truncation_context.line_count` / `char_counts` / `total_chars` / 任意の `reason` は snippet クランプを表します。`--max-line-width` によって snippet 行が省略された場合、`reason` は `line_width` になります。 + +`excerpt --json` は 1-based の開始/終了位置、token `type`、`modifiers` を持つ軽量 range list の `semantic_tokens` を返すため、IDE や LLM クライアントは生の `content` 文字列を再パースせずに抜粋範囲を描画・後処理できます。excerpt row は `requested_start_line`、`requested_end_line`、`effective_start_line`、`effective_end_line`、`content_truncation_reasons`、任意の `content_recovery` も返すため、`--max-line-width` による `line_width_cap` を検出し、省略部分を `cdidx excerpt ... --max-line-width 0 --json` で再取得できます。body を持つ JSON row も対応する `body_requested_*`、`body_effective_*`、`body_content_truncation_reasons`、`body_content_recovery` を返します。body reason には snippet/body 行数上限の `body_line_cap` と definition body byte 上限の `body_byte_cap` があります。 `inspect` と MCP の `analyze_symbol` は、主定義、同一ファイル内の近傍シンボル、参照、caller、callee、ファイルメタデータ、さらにワークスペース鮮度/git メタデータと graph 対応メタデータを1レスポンスにまとめます。bundle 内の graph 節が実際に SQL ベースの read に依存する場合だけ、`sql_graph_contract_ready` / `sql_graph_contract_degraded_reason`（MCP では既存の camelCase alias も）も返します。mixed-language bundle で C# / JS などの graph row しか返っていない場合は SQL trust signal を出さないため、無関係なクエリが stale SQL state に引きずられません。複数の連続クエリを避けたい AI ワークフロー向けです。call graph 系の節は言語差分を考慮しており、未対応言語では `graphSupported` / `graphSupportReason` によって「未対応」と「ヒットなし」を区別できます。その場合は `search` を優先して使う前提です。 diff --git a/changelog.d/unreleased/3556.fixed.md b/changelog.d/unreleased/3556.fixed.md new file mode 100644 index 0000000000..788b2b777b --- /dev/null +++ b/changelog.d/unreleased/3556.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3556 +affected: + - DEVELOPER_GUIDE.md + - src/CodeIndex/Cli/JsonOutputContracts.cs + - src/CodeIndex/Cli/SearchSnippetFormatter.cs + - tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +--- + +## English + +- **Search snippets now explain focus selection and dropped-match navigation (#3556)** — compact search JSON includes `focus_mode`, `focus_line`, `focus_column`, `focus_reason`, and `next_match` metadata so clients can understand the selected window and jump to omitted matches. + +## 日本語 + +- **検索 snippet が focus 選択理由と dropped match の移動先を返すようになりました (#3556)** — compact search JSON に `focus_mode` / `focus_line` / `focus_column` / `focus_reason` / `next_match` を追加し、クライアントが選択された window を理解して省略された一致へ移動できるようにしました。 diff --git a/changelog.d/unreleased/3557.fixed.md b/changelog.d/unreleased/3557.fixed.md new file mode 100644 index 0000000000..00b1b6e4c3 --- /dev/null +++ b/changelog.d/unreleased/3557.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3557 +affected: + - DEVELOPER_GUIDE.md + - src/CodeIndex/Cli/SearchSnippetFormatter.cs + - src/CodeIndex/Database/LineWidthFormatter.cs + - tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +--- + +## English + +- **Search highlight occurrences now expose visible ranges after line clamping (#3557)** — search JSON keeps original source coordinates while adding `visible`, `visible_column`, and `visible_length` so clients can highlight only text present in the returned snippet. + +## 日本語 + +- **行幅クランプ後の検索 highlight occurrence が可視範囲を返すようになりました (#3557)** — search JSON は元ソース座標を維持しつつ、返却 snippet 内に存在する文字だけを強調できるよう `visible` / `visible_column` / `visible_length` を追加しました。 diff --git a/changelog.d/unreleased/3558.fixed.md b/changelog.d/unreleased/3558.fixed.md new file mode 100644 index 0000000000..8636884f4b --- /dev/null +++ b/changelog.d/unreleased/3558.fixed.md @@ -0,0 +1,20 @@ +--- +category: fixed +issues: + - 3558 +affected: + - DEVELOPER_GUIDE.md + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Cli/SearchSnippetFormatter.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - tests/CodeIndex.Tests/McpServerTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerSearchHintTests.cs +--- + +## English + +- **Search JSON now exposes effective snippet options and raw-FTS highlight availability (#3558)** — compact CLI and MCP search rows include effective snippet/mode metadata so clients can distinguish literal highlight support from raw FTS syntax gaps. + +## 日本語 + +- **検索 JSON が有効な snippet options と raw FTS の highlight 可否を返すようになりました (#3558)** — compact CLI と MCP search row に有効な snippet / mode metadata を追加し、literal highlight 対応と raw FTS 構文由来の不足を区別できるようにしました。 diff --git a/changelog.d/unreleased/3561.fixed.md b/changelog.d/unreleased/3561.fixed.md new file mode 100644 index 0000000000..dddd09a899 --- /dev/null +++ b/changelog.d/unreleased/3561.fixed.md @@ -0,0 +1,19 @@ +--- +category: fixed +issues: + - 3561 +affected: + - DEVELOPER_GUIDE.md + - src/CodeIndex/Cli/JsonOutputContracts.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - src/CodeIndex/Models/QueryResults.cs + - tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs +--- + +## English + +- **Find JSON now exposes match spans and snippet truncation context (#3561)** — `cdidx find --json` rows include `length`, `original_line_length`, and `snippet_truncation_context` so clients can highlight repeated file matches and detect line-width elisions without parsing snippet markers. + +## 日本語 + +- **find JSON が match span と snippet truncation context を返すようになりました (#3561)** — `cdidx find --json` の各 row に `length` / `original_line_length` / `snippet_truncation_context` を追加し、クライアントが同一ファイル内の複数一致を highlight し、snippet marker を解析せずに行幅省略を検出できるようにしました。 diff --git a/changelog.d/unreleased/3562.fixed.md b/changelog.d/unreleased/3562.fixed.md new file mode 100644 index 0000000000..cfa41c2a60 --- /dev/null +++ b/changelog.d/unreleased/3562.fixed.md @@ -0,0 +1,23 @@ +--- +category: fixed +issues: + - 3562 +affected: + - DEVELOPER_GUIDE.md + - src/CodeIndex/Cli/JsonOutputContracts.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - src/CodeIndex/Database/DbSymbolReader.cs + - src/CodeIndex/Models/QueryResults.cs + - tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs + - tests/CodeIndex.Tests/golden/excerpt.json +--- + +## English + +- **Excerpt and body JSON now expose range and recovery metadata (#3562)** — excerpt rows include requested/effective ranges, truncation reasons, and replayable recovery commands, while body JSON reports line and byte cap reasons with follow-up excerpt ranges. + +## 日本語 + +- **excerpt と body JSON が range と recovery metadata を返すようになりました (#3562)** — excerpt row に requested/effective range、truncation reason、再実行可能な recovery command を追加し、body JSON でも行数上限と byte 上限の reason と follow-up excerpt range を返すようにしました。 diff --git a/src/CodeIndex/Cli/ExcerptRecoveryCommandFormatter.cs b/src/CodeIndex/Cli/ExcerptRecoveryCommandFormatter.cs new file mode 100644 index 0000000000..ed00c03da9 --- /dev/null +++ b/src/CodeIndex/Cli/ExcerptRecoveryCommandFormatter.cs @@ -0,0 +1,49 @@ +using CodeIndex.Database; + +namespace CodeIndex.Cli; + +internal static class ExcerptRecoveryCommandFormatter +{ + public static void ApplyDbPath(FileExcerptResult excerpt, string dbPath) + { + ApplyDbPath(excerpt.ContentRecovery, excerpt.Path, dbPath); + } + + public static void ApplyDbPath(ExcerptRecoveryHint? recovery, string path, string dbPath) + { + if (recovery is null) + return; + + recovery.Command = BuildCommand(path, recovery.StartLine, recovery.EndLine, dbPath); + } + + private static string BuildCommand(string path, int startLine, int endLine, string dbPath) + { + var dbOption = string.IsNullOrWhiteSpace(dbPath) + ? string.Empty + : $" --db {QuoteShellArgument(NormalizeDbPath(dbPath))}"; + return $"cdidx excerpt {QuoteShellArgument(path)}{dbOption} --start {startLine} --end {endLine} --max-line-width 0 --json"; + } + + private static string NormalizeDbPath(string dbPath) + { + if (dbPath.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) + return dbPath; + + var normalized = DbPathResolver.NormalizeDbPath(dbPath); + return normalized.StartsWith("file:", StringComparison.OrdinalIgnoreCase) + ? normalized + : Path.GetFullPath(normalized); + } + + private static string QuoteShellArgument(string value) + { + if (!string.IsNullOrEmpty(value) && value.All(IsSafeShellArgumentChar)) + return value; + + return "'" + value.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; + } + + private static bool IsSafeShellArgumentChar(char c) + => char.IsLetterOrDigit(c) || c is '/' or '.' or '_' or '-' or ':'; +} diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index f3c550643a..cf13b6960b 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -436,10 +436,12 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(ExportImportCommandRunner.ExportManifest))] [JsonSerializable(typeof(ExportImportCommandRunner.ImportDryRunResult))] [JsonSerializable(typeof(ExportImportCommandRunner.ImportValidationPhaseResult))] +[JsonSerializable(typeof(ExcerptRecoveryHint))] [JsonSerializable(typeof(ExcerptSemanticToken))] [JsonSerializable(typeof(FileDependencyResult))] [JsonSerializable(typeof(FileExcerptResult))] [JsonSerializable(typeof(FileFindResult))] +[JsonSerializable(typeof(FileFindSnippetTruncationContext))] [JsonSerializable(typeof(FileIssue))] [JsonSerializable(typeof(FileResult))] [JsonSerializable(typeof(FreshnessHintResult))] @@ -517,6 +519,7 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(SearchIssueDraftExportJsonResult))] [JsonSerializable(typeof(SearchIssueDraftJsonResult))] [JsonSerializable(typeof(SearchIssueDraftSourceJsonResult))] +[JsonSerializable(typeof(SearchNextMatchHint))] [JsonSerializable(typeof(SearchResult))] [JsonSerializable(typeof(SearchTermOccurrence))] [JsonSerializable(typeof(SearchTruncationContext))] diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index ce72b6b28f..194a258638 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -957,7 +957,7 @@ private static List CollectSearchRecipeQueryR !options.NoVisibilityRank, guardFilters: options.GuardFilters, guardWindow: options.GuardWindow); - var rows = BuildSearchDisplayRows(results, options, exact, recipeQuery.Query); + var rows = BuildSearchDisplayRows(results, options, exact, recipeQuery.Query, rawFtsOverride: false); total += rows.Count; queryResults.Add(new SearchRecipeQueryResultJsonResult( recipeQuery.Name, @@ -1064,12 +1064,19 @@ private static string BuildSearchIssueDraftBody( query.FalsePositiveGuidance, query.ExactSubstring)).ToList()); - private static List BuildSearchDisplayRows(List results, QueryCommandOptions options, bool exact, string? queryOverride = null) + private static List BuildSearchDisplayRows( + List results, + QueryCommandOptions options, + bool exact, + string? queryOverride = null, + bool? rawFtsOverride = null) { var rows = new List(results.Count); var seenMatchLocations = options.NoDedup ? null : new HashSet(StringComparer.Ordinal); var displayQuery = queryOverride ?? options.Query!; - var queryContext = options.RawFts + var rawFts = rawFtsOverride ?? options.RawFts; + var effectiveRawFts = rawFts && !exact; + var queryContext = effectiveRawFts ? SearchSnippetFormatter.PrepareRawFtsQueryContext(displayQuery) : SearchSnippetFormatter.PrepareQueryContext(displayQuery); foreach (var result in results) @@ -1097,8 +1104,9 @@ private static List BuildSearchDisplayRows(List exposeLiteralHighlights: exact, preferredMatchLine: preferredOriginFilterLine.Value); } + SearchSnippetFormatter.ApplyOutputMetadata(compact, options.SnippetLines, options.MaxLineWidth, exact, rawFts); - if (!options.RawFts && compact.MatchLines.Count == 0 && compact.Highlights.Count == 0) + if (!effectiveRawFts && compact.MatchLines.Count == 0 && compact.Highlights.Count == 0) continue; if (!ApplySearchOriginFilters(compact, options)) @@ -1732,6 +1740,7 @@ public static int RunDefinition(string[] cmdArgs, JsonSerializerOptions jsonOpti return ZeroResultExitCode(options); } + ApplyBodyRecoveryCommands(results, options.DbPath); if (options.Json) { if (TryWriteFormattedLocations( @@ -1944,6 +1953,7 @@ public static int RunReferences(string[] cmdArgs, JsonSerializerOptions jsonOpti var results = reader.SearchReferences(options.Query, options.Limit, options.Lang, options.Kind, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, exact, options.MaxLineWidth); if (options.IncludeBody) AttachBodyExcerpts(reader, results, options.SnippetLines, options.MaxLineWidth); + ApplyBodyRecoveryCommands(results, options.DbPath); var sqlGraphSignal = NarrowSqlGraphContractSignalByLanguages(baseSqlGraphSignal, results.Select(result => result.Lang), options.Lang, exactGraphLanguage); var exactSignal = reader.GetReferencesExactQuerySignal(options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, includeSqlGraphContractSignal: sqlGraphSignal.Relevant); var exactZeroHint = BuildExactZeroHint( @@ -2095,6 +2105,7 @@ public static int RunCallers(string[] cmdArgs, JsonSerializerOptions jsonOptions var results = reader.GetCallers(options.Query, options.Limit, options.Lang, options.Kind, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, exact, options.RawKinds, options.RankMode); if (options.IncludeBody) AttachBodyExcerpts(reader, results, options.SnippetLines, options.MaxLineWidth); + ApplyBodyRecoveryCommands(results, options.DbPath); var sqlGraphSignal = NarrowSqlGraphContractSignalByLanguages(baseSqlGraphSignal, results.Select(result => result.Lang), options.Lang, exactGraphLanguage); var exactSignal = reader.GetCallersExactQuerySignal(options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, includeSqlGraphContractSignal: sqlGraphSignal.Relevant); var exactZeroHint = BuildExactZeroHint( @@ -2246,6 +2257,7 @@ public static int RunCallees(string[] cmdArgs, JsonSerializerOptions jsonOptions var results = reader.GetCallees(options.Query, options.Limit, options.Lang, options.Kind, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, exact, options.RawKinds, options.RankMode); if (options.IncludeBody) AttachBodyExcerpts(reader, results, options.SnippetLines, options.MaxLineWidth); + ApplyBodyRecoveryCommands(results, options.DbPath); var sqlGraphSignal = NarrowSqlGraphContractSignalByLanguages(baseSqlGraphSignal, results.Select(result => result.Lang), options.Lang, exactGraphLanguage); var exactSignal = reader.GetCalleesExactQuerySignal(options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, includeSqlGraphContractSignal: sqlGraphSignal.Relevant); var exactZeroHint = BuildExactZeroHint( @@ -2381,10 +2393,19 @@ private static void AttachBodyExcerpts(DbReader reader, IEnumerable string.Equals(existing, reason, StringComparison.Ordinal))) + excerpt.ContentTruncationReasons.Add(reason); + excerpt.ContentRecovery ??= FileExcerptResult.CreateRecoveryHint(excerpt.Path, recoveryStartLine, recoveryEndLine); + } + + private static List? CopyTruncationReasons(FileExcerptResult excerpt) + => excerpt.ContentTruncationReasons.Count > 0 ? [.. excerpt.ContentTruncationReasons] : null; + + private static void ApplyBodyRecoveryCommands(IEnumerable results, string dbPath) + { + foreach (var result in results) + ExcerptRecoveryCommandFormatter.ApplyDbPath(result.BodyContentRecovery, result.Path, dbPath); + } + + private static void ApplyBodyRecoveryCommands(IEnumerable results, string dbPath) + { + foreach (var result in results) + ExcerptRecoveryCommandFormatter.ApplyDbPath(result.BodyContentRecovery, result.Path, dbPath); + } + + private static void ApplyBodyRecoveryCommands(IEnumerable results, string dbPath) + { + foreach (var result in results) + ExcerptRecoveryCommandFormatter.ApplyDbPath(result.BodyContentRecovery, result.Path, dbPath); + } + + private static void ApplyBodyRecoveryCommands(IEnumerable results, string dbPath) + { + foreach (var result in results) + ExcerptRecoveryCommandFormatter.ApplyDbPath(result.BodyContentRecovery, result.Path, dbPath); + } + + private static void ApplyBodyRecoveryCommands(IEnumerable results, string dbPath) + { + foreach (var result in results) + ExcerptRecoveryCommandFormatter.ApplyDbPath(result.BodyContentRecovery, result.Path, dbPath); + } + + private static void ApplyBodyRecoveryCommands(SymbolAnalysisResult result, string dbPath) + { + ApplyBodyRecoveryCommands(result.Definitions, dbPath); + ApplyBodyRecoveryCommands(result.References, dbPath); + ApplyBodyRecoveryCommands(result.Callers, dbPath); + ApplyBodyRecoveryCommands(result.Callees, dbPath); } private static void WriteOptionalBodyExcerpt(int? startLine, string? content, string indent = "") @@ -2812,7 +2906,10 @@ public static int RunExcerpt(string[] cmdArgs, JsonSerializerOptions jsonOptions return ZeroResultExitCode(options); } if (options.Json) + { + ExcerptRecoveryCommandFormatter.ApplyDbPath(excerpt, options.DbPath); excerpt.SemanticTokens = BuildExcerptSemanticTokens(excerpt); + } if (options.Json) { @@ -3740,6 +3837,7 @@ public static int RunInspect(string[] cmdArgs, JsonSerializerOptions jsonOptions if (options.Json) { var compactTruncation = options.Compact ? ApplySymbolAnalysisCompactCaps(analysis, compactLimit) : null; + ApplyBodyRecoveryCommands(analysis, options.DbPath); var payload = JsonSerializer.SerializeToNode(analysis, CliJsonSerializerContextFactory.Create(jsonOptions).SymbolAnalysisResult)!.AsObject(); AddSqlGraphContractJsonFields(payload, sqlGraphSignal); if (compactTruncation != null) @@ -4542,6 +4640,7 @@ public static int RunImpact(string[] cmdArgs, JsonSerializerOptions jsonOptions) var analysis = reader.AnalyzeImpact(options.Query, maxDepth, options.Limit, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.WithPaths); if (options.IncludeBody) AttachBodyExcerpts(reader, analysis.Callers, options.SnippetLines, options.MaxLineWidth); + ApplyBodyRecoveryCommands(analysis.Callers, options.DbPath); var sqlGraphSignal = NarrowSqlGraphContractSignal( reader.GetSqlGraphContractSignal(options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests), DbReader.IsSqlLanguage(options.Lang) diff --git a/src/CodeIndex/Cli/SearchSnippetFormatter.cs b/src/CodeIndex/Cli/SearchSnippetFormatter.cs index ec96a491df..6b69a90883 100644 --- a/src/CodeIndex/Cli/SearchSnippetFormatter.cs +++ b/src/CodeIndex/Cli/SearchSnippetFormatter.cs @@ -105,6 +105,17 @@ public static CompactSearchResult ToCompactResult(SearchResult result, SearchSni ContextAfter = excerpt.ContextAfter, TruncatedLineCount = excerpt.TruncatedLineCount, DroppedMatchLineCount = excerpt.DroppedMatchLineCount, + FocusMode = excerpt.FocusMode, + FocusLine = excerpt.FocusLine, + FocusColumn = excerpt.FocusColumn, + FocusReason = excerpt.FocusReason, + NextMatch = excerpt.NextMatchLine.HasValue + ? new SearchNextMatchHint + { + Line = excerpt.NextMatchLine.Value, + RemainingMatchLineCount = excerpt.DroppedMatchLineCount, + } + : null, TruncationContext = excerpt.TruncationContext, GuardEvidence = result.GuardEvidence, GuardChecks = result.GuardChecks, @@ -209,6 +220,19 @@ public static IEnumerable ToCompactResults(IEnumerable 0 ? 1 : 0; foreach (var matchIndex in matchIndexes.Where(index => index > focusStart)) { @@ -293,10 +319,14 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, SearchSnippetQue if (normalizeCSharpVerbatimNames && isMatch && snippetLine.NormalizedText != null && snippetLine.RawIndexMap != null) { clamped = ClampNormalizedSnippetLine(originalLine, snippetLine.NormalizedText, snippetLine.RawIndexMap, maxLineWidth, normalizedQuery, tokens, caseSensitive, focusMode); + if (i == focusStart) + CaptureNormalizedFocus(snippetLine.NormalizedText, snippetLine.RawIndexMap, normalizedQuery, tokens, caseSensitive, focusMode, out focusColumn, out focusReason); } else { clamped = ClampSnippetLine(originalLine, maxLineWidth, isMatch ? normalizedQuery : null, tokens, caseSensitive, focusMode); + if (i == focusStart && isMatch) + CaptureFocus(originalLine, normalizedQuery, tokens, caseSensitive, focusMode, out focusColumn, out focusReason); } clampedLines.Add(clamped.Text); if (clamped.Truncated) @@ -319,6 +349,9 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, SearchSnippetQue ? GetMatchedTermOccurrences(snippetLine.NormalizedText, absoluteLine, normalizedQuery, [], caseSensitive, originalLine, snippetLine.RawIndexMap) : GetMatchedTermOccurrences(originalLine, absoluteLine, normalizedQuery, [], caseSensitive) : null; + ApplyVisibleRanges(termOccurrences, clamped); + if (literalTermOccurrences != null) + ApplyVisibleRanges(literalTermOccurrences, clamped); highlights.Add(new SearchHighlight { Line = absoluteLine, @@ -346,6 +379,11 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, SearchSnippetQue TruncatedAfter = end < lineCount - 1, TruncatedLineCount = truncatedLineCount, DroppedMatchLineCount = droppedMatchLineCount, + FocusMode = ToJsonFocusMode(focusMode), + FocusLine = matchIndexes.Count > 0 ? absoluteStartLine + focusStart : null, + FocusColumn = focusColumn, + FocusReason = focusReason, + NextMatchLine = matchScan.FirstDroppedMatchIndex.HasValue ? absoluteStartLine + matchScan.FirstDroppedMatchIndex.Value : null, TruncationContext = new SearchTruncationContext { LineCount = truncatedLineCount, @@ -375,25 +413,25 @@ private static int SelectFocusStart(IReadOnlyList matchIndexes, int absolut private static ClampedTextResult ClampSnippetLine(string line, int maxLineWidth, string? normalizedQuery, string[] tokens, bool caseSensitive, SearchSnippetFocusMode focusMode) { if (line.Length <= maxLineWidth) - return new ClampedTextResult(line, false); + return ClampedTextResult.Unclamped(line); if (normalizedQuery == null) return LineWidthFormatter.ClampLine(line, maxLineWidth); - var (matchColumn, matchLength) = FindBestMatchColumn(line, normalizedQuery, tokens, caseSensitive, focusMode); - if (matchColumn <= 0) + var focus = FindBestMatchFocus(line, normalizedQuery, tokens, caseSensitive, focusMode); + if (!focus.IsValid) return LineWidthFormatter.ClampLine(line, maxLineWidth); - return LineWidthFormatter.ClampLine(line, maxLineWidth, matchColumn, matchLength); + return LineWidthFormatter.ClampLine(line, maxLineWidth, focus.Column, focus.Length); } private static ClampedTextResult ClampNormalizedSnippetLine(string originalLine, string normalizedLine, int[] rawIndexMap, int maxLineWidth, string normalizedQuery, string[] tokens, bool caseSensitive, SearchSnippetFocusMode focusMode) { - var (matchColumn, matchLength) = FindBestMatchColumn(normalizedLine, normalizedQuery, tokens, caseSensitive, focusMode); - if (matchColumn <= 0) + var focus = FindBestMatchFocus(normalizedLine, normalizedQuery, tokens, caseSensitive, focusMode); + if (!focus.IsValid) return LineWidthFormatter.ClampLine(originalLine, maxLineWidth); - if (!TryReconstructRawSpan(rawIndexMap, normalizedStart: matchColumn - 1, matchLength, out var rawFocusColumn, out var rawFocusLength)) + if (!TryReconstructRawSpan(rawIndexMap, normalizedStart: focus.Column - 1, focus.Length, out var rawFocusColumn, out var rawFocusLength)) return LineWidthFormatter.ClampLine(originalLine, maxLineWidth); return LineWidthFormatter.ClampLine(originalLine, maxLineWidth, rawFocusColumn, rawFocusLength); @@ -421,11 +459,11 @@ internal static bool TryReconstructRawSpan(int[] rawIndexMap, int normalizedStar return true; } - private static (int Column, int Length) FindBestMatchColumn(string line, string normalizedQuery, string[] tokens, bool caseSensitive, SearchSnippetFocusMode focusMode) + private static SearchMatchFocus FindBestMatchFocus(string line, string normalizedQuery, string[] tokens, bool caseSensitive, SearchSnippetFocusMode focusMode) { var comparison = caseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; if (focusMode == SearchSnippetFocusMode.Leftmost) - return FindLeftmostMatchColumn(line, normalizedQuery, tokens, comparison); + return FindLeftmostMatchFocus(line, normalizedQuery, tokens, comparison); MatchCandidate best = default; var fullQueryScore = focusMode == SearchSnippetFocusMode.Proximity ? 3_000 : 4_000; @@ -435,7 +473,7 @@ private static (int Column, int Length) FindBestMatchColumn(string line, string var idx = 0; while ((idx = line.IndexOf(normalizedQuery, idx, comparison)) >= 0) { - best = ChooseBetter(best, new MatchCandidate(idx, normalizedQuery.Length, fullQueryScore + ScoreLength(normalizedQuery.Length), null)); + best = ChooseBetter(best, new MatchCandidate(idx, normalizedQuery.Length, fullQueryScore + ScoreLength(normalizedQuery.Length), null, "full_query")); idx += Math.Max(1, normalizedQuery.Length); } } @@ -449,7 +487,7 @@ private static (int Column, int Length) FindBestMatchColumn(string line, string var idx = 0; while ((idx = line.IndexOf(token, idx, comparison)) >= 0) { - var occurrence = new MatchCandidate(idx, token.Length, 1_000 + ScoreLength(token.Length), token); + var occurrence = new MatchCandidate(idx, token.Length, 1_000 + ScoreLength(token.Length), token, "token"); tokenOccurrences.Add(occurrence); best = ChooseBetter(best, occurrence); idx += Math.Max(1, token.Length); @@ -476,22 +514,23 @@ private static (int Column, int Length) FindBestMatchColumn(string line, string var clusterEnd = cluster.Max(candidate => candidate.Index + candidate.Length); var totalTokenLength = cluster.Sum(candidate => ScoreLength(candidate.Length)); - best = ChooseBetter(best, new MatchCandidate(start, clusterEnd - start, clusterScore + (tokenCount * 100) + totalTokenLength, null)); + best = ChooseBetter(best, new MatchCandidate(start, clusterEnd - start, clusterScore + (tokenCount * 100) + totalTokenLength, null, "token_cluster")); } } if (!best.IsValid) - return (0, 0); + return default; // LineWidthFormatter.ClampLine accepts a 1-based focusColumn. // LineWidthFormatter.ClampLine は 1-based の focusColumn を受け取る。 - return (best.Index + 1, Math.Max(1, best.Length)); + return new SearchMatchFocus(best.Index + 1, Math.Max(1, best.Length), best.Reason); } - private static (int Column, int Length) FindLeftmostMatchColumn(string line, string normalizedQuery, string[] tokens, StringComparison comparison) + private static SearchMatchFocus FindLeftmostMatchFocus(string line, string normalizedQuery, string[] tokens, StringComparison comparison) { int bestIndex = -1; int bestLength = 0; + var reason = "leftmost_token"; if (!string.IsNullOrEmpty(normalizedQuery)) { var idx = line.IndexOf(normalizedQuery, comparison); @@ -499,6 +538,7 @@ private static (int Column, int Length) FindLeftmostMatchColumn(string line, str { bestIndex = idx; bestLength = normalizedQuery.Length; + reason = "leftmost_full_query"; } } @@ -513,10 +553,11 @@ private static (int Column, int Length) FindLeftmostMatchColumn(string line, str { bestIndex = idx; bestLength = token.Length; + reason = "leftmost_token"; } } - return bestIndex < 0 ? (0, 0) : (bestIndex + 1, Math.Max(1, bestLength)); + return bestIndex < 0 ? default : new SearchMatchFocus(bestIndex + 1, Math.Max(1, bestLength), reason); } private static MatchCandidate ChooseBetter(MatchCandidate current, MatchCandidate candidate) @@ -534,7 +575,31 @@ private static MatchCandidate ChooseBetter(MatchCandidate current, MatchCandidat private static int ScoreLength(int length) => Math.Min(length, 99); - private readonly record struct MatchCandidate(int Index, int Length, int Score, string? Token) + private static void CaptureFocus(string line, string normalizedQuery, string[] tokens, bool caseSensitive, SearchSnippetFocusMode focusMode, out int? focusColumn, out string? focusReason) + { + var focus = FindBestMatchFocus(line, normalizedQuery, tokens, caseSensitive, focusMode); + focusColumn = focus.IsValid ? focus.Column : null; + focusReason = focus.IsValid ? focus.Reason : null; + } + + private static void CaptureNormalizedFocus(string normalizedLine, int[] rawIndexMap, string normalizedQuery, string[] tokens, bool caseSensitive, SearchSnippetFocusMode focusMode, out int? focusColumn, out string? focusReason) + { + var focus = FindBestMatchFocus(normalizedLine, normalizedQuery, tokens, caseSensitive, focusMode); + focusReason = focus.IsValid ? focus.Reason : null; + focusColumn = focus.IsValid && TryReconstructRawSpan(rawIndexMap, focus.Column - 1, focus.Length, out var rawColumn, out _) + ? rawColumn + : null; + } + + private static string ToJsonFocusMode(SearchSnippetFocusMode focusMode) => + focusMode.ToString().ToLowerInvariant(); + + private readonly record struct SearchMatchFocus(int Column, int Length, string Reason) + { + public bool IsValid => Column > 0 && Length > 0 && !string.IsNullOrEmpty(Reason); + } + + private readonly record struct MatchCandidate(int Index, int Length, int Score, string? Token, string Reason) { public bool IsValid => Index >= 0 && Length > 0; } @@ -559,6 +624,7 @@ private static SearchSnippetLineMatchScan FindMatchingLineIndexes(string content var lineCount = 0; var totalMatchCount = 0; int? focusStart = null; + int? firstDroppedMatchIndex = null; if (!string.IsNullOrWhiteSpace(query)) { @@ -567,7 +633,7 @@ private static SearchSnippetLineMatchScan FindMatchingLineIndexes(string content lineCount++; var line = normalizeCSharpVerbatimNames ? CSharpVerbatimNameNormalizer.Normalize(rawLine) : rawLine; if (line.Contains(query, comparison)) - AddTrackedMatchIndex(matches, i, maxTrackedWindowLines, ref focusStart, ref totalMatchCount); + AddTrackedMatchIndex(matches, i, maxTrackedWindowLines, ref focusStart, ref totalMatchCount, ref firstDroppedMatchIndex); } } else @@ -576,29 +642,35 @@ private static SearchSnippetLineMatchScan FindMatchingLineIndexes(string content } if (matches.Count > 0 || tokens.Length == 0) - return new SearchSnippetLineMatchScan(matches, lineCount, totalMatchCount); + return new SearchSnippetLineMatchScan(matches, lineCount, totalMatchCount, firstDroppedMatchIndex); matches.Clear(); lineCount = 0; totalMatchCount = 0; focusStart = null; + firstDroppedMatchIndex = null; foreach (var (i, rawLine) in EnumerateContentLines(content)) { lineCount++; var line = normalizeCSharpVerbatimNames ? CSharpVerbatimNameNormalizer.Normalize(rawLine) : rawLine; if (tokens.Any(token => line.Contains(token, comparison))) - AddTrackedMatchIndex(matches, i, maxTrackedWindowLines, ref focusStart, ref totalMatchCount); + AddTrackedMatchIndex(matches, i, maxTrackedWindowLines, ref focusStart, ref totalMatchCount, ref firstDroppedMatchIndex); } - return new SearchSnippetLineMatchScan(matches, lineCount, totalMatchCount); + return new SearchSnippetLineMatchScan(matches, lineCount, totalMatchCount, firstDroppedMatchIndex); } - private static void AddTrackedMatchIndex(List matches, int lineIndex, int maxTrackedWindowLines, ref int? focusStart, ref int totalMatchCount) + private static void AddTrackedMatchIndex(List matches, int lineIndex, int maxTrackedWindowLines, ref int? focusStart, ref int totalMatchCount, ref int? firstDroppedMatchIndex) { totalMatchCount++; focusStart ??= lineIndex; if ((lineIndex - focusStart.Value) + 1 <= maxTrackedWindowLines) + { matches.Add(lineIndex); + return; + } + + firstDroppedMatchIndex ??= lineIndex; } private static List ReadSnippetLines(string content, int start, int end, bool normalizeCSharpVerbatimNames) @@ -653,7 +725,7 @@ private static int CountContentLines(string content) yield return (lineIndex, content[lineStart..]); } - private sealed record SearchSnippetLineMatchScan(List MatchIndexes, int LineCount, int TotalMatchCount); + private sealed record SearchSnippetLineMatchScan(List MatchIndexes, int LineCount, int TotalMatchCount, int? FirstDroppedMatchIndex); private sealed record SearchSnippetLine(int Index, string Text, string? NormalizedText, int[]? RawIndexMap); @@ -670,6 +742,28 @@ private static List GetMatchedTermOccurrences(string line, return occurrences; } + private static void ApplyVisibleRanges(List occurrences, ClampedTextResult clamped) + { + var visibleStart = clamped.OriginalVisibleStartColumn; + var visibleEndExclusive = clamped.OriginalVisibleEndColumn + 1; + foreach (var occurrence in occurrences) + { + var occurrenceStart = occurrence.Column; + var occurrenceEndExclusive = occurrence.Column + occurrence.Length; + var intersectionStart = Math.Max(occurrenceStart, visibleStart); + var intersectionEndExclusive = Math.Min(occurrenceEndExclusive, visibleEndExclusive); + if (intersectionEndExclusive <= intersectionStart) + { + occurrence.Visible = false; + continue; + } + + occurrence.Visible = true; + occurrence.VisibleColumn = clamped.TextVisibleStartColumn + (intersectionStart - visibleStart); + occurrence.VisibleLength = intersectionEndExclusive - intersectionStart; + } + } + private static void AddTermOccurrences(List occurrences, string line, int absoluteLine, string term, StringComparison comparison, string? rawLine, int[]? rawIndexMap) { if (string.IsNullOrEmpty(term)) @@ -785,6 +879,22 @@ public sealed class CompactSearchResult public int ContextAfter { get; set; } public int TruncatedLineCount { get; set; } public int DroppedMatchLineCount { get; set; } + public int SnippetLines { get; set; } + public int MaxLineWidth { get; set; } + public bool Exact { get; set; } + public bool RawFts { get; set; } + public bool LiteralHighlightsAvailable { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? LiteralHighlightWarning { get; set; } + public string FocusMode { get; set; } = string.Empty; + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? FocusLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? FocusColumn { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FocusReason { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public SearchNextMatchHint? NextMatch { get; set; } public SearchTruncationContext TruncationContext { get; set; } = new(); [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? GuardEvidence { get; set; } @@ -805,6 +915,12 @@ public sealed class CompactSearchResult public string? EnclosingContainerName { get; set; } } +public sealed class SearchNextMatchHint +{ + public int Line { get; set; } + public int RemainingMatchLineCount { get; set; } +} + public enum SearchSnippetFocusMode { Leftmost, @@ -837,6 +953,11 @@ public sealed class SearchTermOccurrence public int Line { get; set; } public int Column { get; set; } public int Length { get; set; } + public bool Visible { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? VisibleColumn { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? VisibleLength { get; set; } } public sealed class SearchTruncationContext @@ -859,5 +980,10 @@ public sealed class SearchSnippetExcerpt public bool TruncatedAfter { get; set; } public int TruncatedLineCount { get; set; } public int DroppedMatchLineCount { get; set; } + public string FocusMode { get; set; } = string.Empty; + public int? FocusLine { get; set; } + public int? FocusColumn { get; set; } + public string? FocusReason { get; set; } + public int? NextMatchLine { get; set; } public SearchTruncationContext TruncationContext { get; set; } = new(); } diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index 9f3d5ca88a..dc8a1a0ef1 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -376,12 +376,13 @@ private static void FlushReadyFindMatches( continue; var snippetLines = snippetLineNumbers.Select(line => snippetLinesByNumber[line]).ToList(); - var clampedSnippet = LineWidthFormatter.ClampLines( + var (snippet, truncationContext) = ClampFindSnippetLines( snippetLines, maxLineWidth, focusLineIndex: snippetLineNumbers.IndexOf(pending.LineNumber), focusColumn: pending.Column + 1, focusLength: pending.Length); + var matchLine = snippetLinesByNumber[pending.LineNumber]; results.Add(new FileFindResult { @@ -389,14 +390,50 @@ private static void FlushReadyFindMatches( Lang = fileLang, Line = pending.LineNumber, Column = pending.Column + 1, + Length = pending.Length, + OriginalLineLength = matchLine.Length, StartLine = snippetLineNumbers[0], EndLine = snippetLineNumbers[^1], - Snippet = clampedSnippet.Text, - SnippetTruncated = clampedSnippet.Truncated, + Snippet = snippet, + SnippetTruncated = truncationContext.LineCount > 0, + SnippetTruncationContext = truncationContext, }); } } + private static (string Text, FileFindSnippetTruncationContext Context) ClampFindSnippetLines( + IReadOnlyList lines, + int maxLineWidth, + int focusLineIndex, + int focusColumn, + int focusLength) + { + if (lines.Count == 0) + return (string.Empty, new FileFindSnippetTruncationContext()); + + var output = new string[lines.Count]; + var truncatedCharCounts = new List(); + for (var i = 0; i < lines.Count; i++) + { + var clamped = i == focusLineIndex + ? LineWidthFormatter.ClampLine(lines[i], maxLineWidth, focusColumn, focusLength) + : LineWidthFormatter.ClampLine(lines[i], maxLineWidth); + output[i] = clamped.Text; + if (clamped.Truncated) + truncatedCharCounts.Add(clamped.TruncatedCharCount); + } + + return ( + string.Join('\n', output), + new FileFindSnippetTruncationContext + { + LineCount = truncatedCharCounts.Count, + CharCounts = truncatedCharCounts, + TotalChars = truncatedCharCounts.Sum(), + Reason = truncatedCharCounts.Count > 0 ? "line_width" : null, + }); + } + private static void PruneFindWindow( int currentLine, int before, @@ -539,8 +576,16 @@ FROM chunks c Lang = lang, StartLine = selectedLines[0], EndLine = selectedLines[^1], + RequestedStartLine = requestedStart, + RequestedEndLine = requestedEndCeiling, + EffectiveStartLine = selectedLines[0], + EffectiveEndLine = selectedLines[^1], Content = clampedContent.Text, ContentTruncated = clampedContent.Truncated, + ContentTruncationReasons = clampedContent.Truncated ? ["line_width_cap"] : [], + ContentRecovery = clampedContent.Truncated + ? FileExcerptResult.CreateRecoveryHint(path, selectedLines[0], selectedLines[^1]) + : null, }; } diff --git a/src/CodeIndex/Database/DbSymbolReader.cs b/src/CodeIndex/Database/DbSymbolReader.cs index f52b5ef915..2838ca227d 100644 --- a/src/CodeIndex/Database/DbSymbolReader.cs +++ b/src/CodeIndex/Database/DbSymbolReader.cs @@ -795,6 +795,12 @@ public List GetDefinitions(string query, int limit = 20, strin int? bodyContentEndLine = null; int? bodyContentNextStartLine = null; var bodyContentTruncated = false; + int? bodyRequestedStartLine = null; + int? bodyRequestedEndLine = null; + int? bodyEffectiveStartLine = null; + int? bodyEffectiveEndLine = null; + var bodyContentTruncationReasons = new List(); + ExcerptRecoveryHint? bodyContentRecovery = null; if (includeBody && symbol.BodyStartLine != null && symbol.BodyEndLine != null) { var requestedBodyLines = Math.Clamp( @@ -811,15 +817,31 @@ public List GetDefinitions(string query, int limit = 20, strin var bodyExcerpt = GetExcerpt(symbol.Path, effectiveBodyStartLine, cappedBodyEndLine); if (bodyExcerpt != null) { + bodyRequestedStartLine = symbol.BodyStartLine.Value; + bodyRequestedEndLine = symbol.BodyEndLine.Value; + bodyEffectiveStartLine = bodyExcerpt.StartLine; + bodyEffectiveEndLine = bodyExcerpt.EndLine; bodyContent = bodyExcerpt.Content; bodyContentStartLine = bodyExcerpt.StartLine; bodyContentEndLine = bodyExcerpt.EndLine; - bodyContentTruncated = bodyExcerpt.ContentTruncated || cappedBodyEndLine < symbol.BodyEndLine.Value; + bodyContentTruncationReasons.AddRange(bodyExcerpt.ContentTruncationReasons); + bodyContentRecovery = bodyExcerpt.ContentRecovery; + if (cappedBodyEndLine < symbol.BodyEndLine.Value) + { + bodyContentTruncated = true; + AddBodyContentTruncationReason(bodyContentTruncationReasons, "body_line_cap"); + var recoveryStartLine = cappedBodyEndLine + 1; + var recoveryEndLine = Math.Min(symbol.BodyEndLine.Value, recoveryStartLine + DefinitionBodyMaxLines - 1); + bodyContentNextStartLine = recoveryStartLine; + bodyContentRecovery ??= FileExcerptResult.CreateRecoveryHint(symbol.Path, recoveryStartLine, recoveryEndLine); + } + bodyContentTruncated |= bodyExcerpt.ContentTruncated; var byteClamp = ClampDefinitionBodyBytes(bodyContent); bodyContent = byteClamp.Content; if (byteClamp.Truncated) { bodyContentTruncated = true; + AddBodyContentTruncationReason(bodyContentTruncationReasons, "body_byte_cap"); if (byteClamp.ReturnedLineCount > 0) { bodyContentEndLine = Math.Min( @@ -836,9 +858,8 @@ public List GetDefinitions(string query, int limit = 20, strin if (nextStartLine <= symbol.BodyEndLine.Value) bodyContentNextStartLine = nextStartLine; } + bodyContentRecovery = FileExcerptResult.CreateRecoveryHint(symbol.Path, bodyExcerpt.StartLine, bodyExcerpt.EndLine); } - else if (cappedBodyEndLine < symbol.BodyEndLine.Value) - bodyContentNextStartLine = cappedBodyEndLine + 1; } } @@ -866,6 +887,12 @@ public List GetDefinitions(string query, int limit = 20, strin BodyContentEndLine = bodyContentEndLine, BodyContentNextStartLine = bodyContentNextStartLine, BodyContentTruncated = bodyContentTruncated, + BodyRequestedStartLine = bodyRequestedStartLine, + BodyRequestedEndLine = bodyRequestedEndLine, + BodyEffectiveStartLine = bodyEffectiveStartLine, + BodyEffectiveEndLine = bodyEffectiveEndLine, + BodyContentTruncationReasons = bodyContentTruncationReasons.Count > 0 ? bodyContentTruncationReasons : null, + BodyContentRecovery = bodyContentRecovery, Complexity = bodyContent != null && !bodyContentTruncated ? SymbolExtractor.EstimateComplexity(bodyContent) : null, @@ -875,6 +902,12 @@ public List GetDefinitions(string query, int limit = 20, strin return results; } + private static void AddBodyContentTruncationReason(List reasons, string reason) + { + if (!reasons.Any(existing => string.Equals(existing, reason, StringComparison.Ordinal))) + reasons.Add(reason); + } + private static (string Content, bool Truncated, int ReturnedLineCount) ClampDefinitionBodyBytes(string content) { var bytes = Encoding.UTF8.GetBytes(content); diff --git a/src/CodeIndex/Database/LineWidthFormatter.cs b/src/CodeIndex/Database/LineWidthFormatter.cs index 79e1b9d954..d84da156f9 100644 --- a/src/CodeIndex/Database/LineWidthFormatter.cs +++ b/src/CodeIndex/Database/LineWidthFormatter.cs @@ -19,9 +19,9 @@ public static ClampedTextResult ClampLine(string line, int maxLineWidth, int? fo { maxLineWidth = ClampMaxLineWidth(maxLineWidth); if (maxLineWidth <= 0) - return new ClampedTextResult(line, false); + return ClampedTextResult.Unclamped(line); if (DisplayWidth(line) <= maxLineWidth) - return new ClampedTextResult(line, false); + return ClampedTextResult.Unclamped(line); if (focusColumn is > 0) return ClampAroundFocus(line, maxLineWidth, focusColumn.Value, Math.Max(1, focusLength)); @@ -32,11 +32,11 @@ public static ClampedTextResult ClampLine(string line, int maxLineWidth, int? fo public static ClampedTextResult ClampLines(IReadOnlyList lines, int maxLineWidth, int? focusLineIndex = null, int? focusColumn = null, int focusLength = 1) { if (lines.Count == 0) - return new ClampedTextResult(string.Empty, false); + return ClampedTextResult.Unclamped(string.Empty); maxLineWidth = ClampMaxLineWidth(maxLineWidth); if (maxLineWidth <= 0) - return new ClampedTextResult(string.Join('\n', lines), false); + return ClampedTextResult.Unclamped(string.Join('\n', lines)); var output = new string[lines.Count]; var anyTruncated = false; var truncatedCharCount = 0; @@ -74,11 +74,11 @@ private static ClampedTextResult ClampFromHead(string line, int maxLineWidth) if (visibleWidth <= 0) { var safeEnd = SliceEndByDisplayWidth(line, maxLineWidth); - return new ClampedTextResult(line[..safeEnd], true, line.Length - safeEnd); + return new ClampedTextResult(line[..safeEnd], true, line.Length - safeEnd, 1, safeEnd, 1); } var safeVisibleEnd = SliceEndByDisplayWidth(line, visibleWidth); - return new ClampedTextResult(line[..safeVisibleEnd] + BuildMarker(line.Length - safeVisibleEnd), true, line.Length - safeVisibleEnd); + return new ClampedTextResult(line[..safeVisibleEnd] + BuildMarker(line.Length - safeVisibleEnd), true, line.Length - safeVisibleEnd, 1, safeVisibleEnd, 1); } private static ClampedTextResult ClampAroundFocus(string line, int maxLineWidth, int focusColumn, int focusLength) @@ -97,7 +97,7 @@ private static ClampedTextResult ClampAroundFocus(string line, int maxLineWidth, { var safeFocusStart = SafeSliceStart(line, focusStart); var safeFallbackEnd = SliceEndByDisplayWidth(line, maxLineWidth, safeFocusStart); - return new ClampedTextResult(line[safeFocusStart..safeFallbackEnd], true, line.Length - (safeFallbackEnd - safeFocusStart)); + return new ClampedTextResult(line[safeFocusStart..safeFallbackEnd], true, line.Length - (safeFallbackEnd - safeFocusStart), safeFocusStart + 1, safeFallbackEnd, 1); } var desiredCenter = focusStart + ((focusEnd - focusStart) / 2); @@ -122,7 +122,7 @@ private static ClampedTextResult ClampAroundFocus(string line, int maxLineWidth, var finalPrefix = safeStart > 0 ? BuildMarker(safeStart) : string.Empty; var finalSuffix = safeEnd < line.Length ? BuildMarker(line.Length - safeEnd) : string.Empty; var visibleChars = safeEnd - safeStart; - return new ClampedTextResult(finalPrefix + line[safeStart..safeEnd] + finalSuffix, true, line.Length - visibleChars); + return new ClampedTextResult(finalPrefix + line[safeStart..safeEnd] + finalSuffix, true, line.Length - visibleChars, safeStart + 1, safeEnd, finalPrefix.Length + 1); } // Avoid slicing in the middle of a grapheme cluster so combining marks, ZWJ emoji, @@ -487,4 +487,14 @@ private static bool IsAmbiguousRune(int value) => or >= 0x1F19B and <= 0x1F1AC; } -public readonly record struct ClampedTextResult(string Text, bool Truncated, int TruncatedCharCount = 0); +public readonly record struct ClampedTextResult( + string Text, + bool Truncated, + int TruncatedCharCount = 0, + int OriginalVisibleStartColumn = 1, + int OriginalVisibleEndColumn = 0, + int TextVisibleStartColumn = 1) +{ + public static ClampedTextResult Unclamped(string text) => + new(text, false, 0, 1, text.Length, 1); +} diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 3df8954e1a..68270b4424 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -1512,6 +1512,11 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) } var queryContext = SearchSnippetFormatter.PrepareQueryContext(query); + var compactResults = SearchSnippetFormatter + .ToCompactResults(results, queryContext, snippetLines, exact, maxLineWidth, exposeLiteralHighlights: exact) + .ToList(); + foreach (var compact in compactResults) + SearchSnippetFormatter.ApplyOutputMetadata(compact, snippetLines, maxLineWidth, exact, rawQuery); var structured = new JsonObject { ["query"] = query, @@ -1521,7 +1526,7 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) ["maxLineWidth"] = maxLineWidth, ["path"] = PathEcho(pathPatterns), ["excludeTests"] = excludeTests, - ["results"] = ToJsonArray(SearchSnippetFormatter.ToCompactResults(results, queryContext, snippetLines, exact, maxLineWidth, exposeLiteralHighlights: exact)) + ["results"] = ToJsonArray(compactResults) }; AddSearchStabilityMetadata(structured, reader, cursor, results); AddResultEnvelope(structured, results.Count, truncated ? null : results.Count, truncated); @@ -1628,6 +1633,8 @@ private JsonNode ExecuteSearchRecipe(JsonNode? id, JsonNode? args, string recipe var compactResults = SearchSnippetFormatter .ToCompactResults(results, queryContext, snippetLines, exact, maxLineWidth, exposeLiteralHighlights: exact) .ToList(); + foreach (var compact in compactResults) + SearchSnippetFormatter.ApplyOutputMetadata(compact, snippetLines, maxLineWidth, exact, rawFts: false); total += compactResults.Count; queryResults.Add(new JsonObject { @@ -1850,6 +1857,7 @@ private JsonNode ExecuteDefinition(JsonNode? id, JsonNode? args) } if (lspCompatible) QueryCommandRunner.AttachLspLocations(results); + ApplyExcerptRecoveryDbPath(results); var exactSignal = reader.GetDefinitionExactQuerySignal(lang, pathPatterns, excludePaths, excludeTests, since); var exactZeroHint = QueryCommandRunner.BuildExactZeroHint( exact, @@ -1886,6 +1894,30 @@ private JsonNode ExecuteDefinition(JsonNode? id, JsonNode? args) }); } + private void ApplyExcerptRecoveryDbPath(IEnumerable results) + { + foreach (var result in results) + ExcerptRecoveryCommandFormatter.ApplyDbPath(result.BodyContentRecovery, result.Path, _dbPath); + } + + private void ApplyExcerptRecoveryDbPath(IEnumerable results) + { + foreach (var result in results) + ExcerptRecoveryCommandFormatter.ApplyDbPath(result.BodyContentRecovery, result.Path, _dbPath); + } + + private void ApplyExcerptRecoveryDbPath(IEnumerable results) + { + foreach (var result in results) + ExcerptRecoveryCommandFormatter.ApplyDbPath(result.BodyContentRecovery, result.Path, _dbPath); + } + + private void ApplyExcerptRecoveryDbPath(IEnumerable results) + { + foreach (var result in results) + ExcerptRecoveryCommandFormatter.ApplyDbPath(result.BodyContentRecovery, result.Path, _dbPath); + } + private JsonNode ExecuteReferences(JsonNode? id, JsonNode? args) { if (!TryReadRequiredStringParameter(args, "query", out var query, out var requiredError)) @@ -2347,6 +2379,10 @@ private JsonNode ExecuteAnalyzeSymbol(JsonNode? id, JsonNode? args) analysis.SqlGraphContractReady = sqlGraphSignal.Relevant ? sqlGraphSignal.Ready : null; analysis.SqlGraphContractDegradedReason = sqlGraphSignal.Relevant ? sqlGraphSignal.DegradedReason : null; WorkspaceMetadataEnricher.Enrich(analysis, _dbPath, _dbPathExplicit); + ApplyExcerptRecoveryDbPath(analysis.Definitions); + ApplyExcerptRecoveryDbPath(analysis.References); + ApplyExcerptRecoveryDbPath(analysis.Callers); + ApplyExcerptRecoveryDbPath(analysis.Callees); var structured = ToAnalyzeSymbolJsonObject(analysis); AddSqlGraphContractSignal(structured, sqlGraphSignal); structured.Remove("exactZeroHint"); @@ -2838,6 +2874,7 @@ private JsonNode ExecuteExcerpt(JsonNode? id, JsonNode? args) return CreateToolResult(id, "No excerpt found.", emptyPayload); } + ExcerptRecoveryCommandFormatter.ApplyDbPath(excerpt, _dbPath); var payload = JsonSerializer.SerializeToNode(excerpt, _jsonOptions)!.AsObject(); ApplyExcerptOutputBudget(payload, maxOutputBytes); payload["maxOutputBytes"] = maxOutputBytes; diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index b78f3bc3c3..89ea19a203 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -232,10 +232,36 @@ public class FileExcerptResult public string? Lang { get; set; } public int StartLine { get; set; } public int EndLine { get; set; } + public int RequestedStartLine { get; set; } + public int RequestedEndLine { get; set; } + public int EffectiveStartLine { get; set; } + public int EffectiveEndLine { get; set; } public string Content { get; set; } = string.Empty; public bool ContentTruncated { get; set; } + public List ContentTruncationReasons { get; set; } = []; + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ExcerptRecoveryHint? ContentRecovery { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? SemanticTokens { get; set; } + + public static ExcerptRecoveryHint CreateRecoveryHint(string path, int startLine, int endLine) + => new() + { + StartLine = startLine, + EndLine = endLine, + Command = $"cdidx excerpt {QuoteCliArgument(path)} --start {startLine} --end {endLine} --max-line-width 0 --json", + }; + + private static string QuoteCliArgument(string value) + { + if (!string.IsNullOrEmpty(value) && value.All(IsSafeCliArgumentChar)) + return value; + + return "'" + value.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; + } + + private static bool IsSafeCliArgumentChar(char c) + => char.IsLetterOrDigit(c) || c is '/' or '.' or '_' or '-' or ':'; } public class ExcerptSemanticToken @@ -248,6 +274,13 @@ public class ExcerptSemanticToken public List Modifiers { get; set; } = []; } +public class ExcerptRecoveryHint +{ + public int StartLine { get; set; } + public int EndLine { get; set; } + public string Command { get; set; } = string.Empty; +} + public class FileFindResult { [JsonPropertyName("api_version")] @@ -256,10 +289,22 @@ public class FileFindResult public string? Lang { get; set; } public int Line { get; set; } public int Column { get; set; } + public int Length { get; set; } + public int OriginalLineLength { get; set; } public int StartLine { get; set; } public int EndLine { get; set; } public string Snippet { get; set; } = string.Empty; public bool SnippetTruncated { get; set; } + public FileFindSnippetTruncationContext SnippetTruncationContext { get; set; } = new(); +} + +public class FileFindSnippetTruncationContext +{ + public int LineCount { get; set; } + public List CharCounts { get; set; } = []; + public int TotalChars { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Reason { get; set; } } public class IndexFreshnessCheckResult @@ -309,6 +354,18 @@ public class DefinitionResult : SymbolResult [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public bool BodyContentTruncated { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyRequestedStartLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyRequestedEndLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyEffectiveStartLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyEffectiveEndLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? BodyContentTruncationReasons { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ExcerptRecoveryHint? BodyContentRecovery { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public int? Complexity { get; set; } } @@ -397,6 +454,18 @@ public class ReferenceResult public int? BodyEndLine { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public bool BodyContentTruncated { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyRequestedStartLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyRequestedEndLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyEffectiveStartLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyEffectiveEndLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? BodyContentTruncationReasons { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ExcerptRecoveryHint? BodyContentRecovery { get; set; } } public class CallerResult @@ -434,6 +503,18 @@ public class CallerResult public int? BodyEndLine { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public bool BodyContentTruncated { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyRequestedStartLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyRequestedEndLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyEffectiveStartLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyEffectiveEndLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? BodyContentTruncationReasons { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ExcerptRecoveryHint? BodyContentRecovery { get; set; } } public class CalleeResult @@ -462,6 +543,18 @@ public class CalleeResult public int? BodyEndLine { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public bool BodyContentTruncated { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyRequestedStartLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyRequestedEndLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyEffectiveStartLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyEffectiveEndLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? BodyContentTruncationReasons { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ExcerptRecoveryHint? BodyContentRecovery { get; set; } } public class ImpactResult @@ -500,6 +593,18 @@ public class ImpactResult public int? BodyEndLine { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public bool BodyContentTruncated { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyRequestedStartLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyRequestedEndLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyEffectiveStartLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? BodyEffectiveEndLine { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? BodyContentTruncationReasons { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ExcerptRecoveryHint? BodyContentRecovery { get; set; } } public static class ImpactResultKinds diff --git a/tests/CodeIndex.Tests/McpServerTests.cs b/tests/CodeIndex.Tests/McpServerTests.cs index 792d9d4b46..9f6d748b51 100644 --- a/tests/CodeIndex.Tests/McpServerTests.cs +++ b/tests/CodeIndex.Tests/McpServerTests.cs @@ -4531,6 +4531,55 @@ public void ToolsCall_Search_ListRecipesBoundsConfiguredSourceDiagnostics_Issue3 Assert.Contains(diagnostics, diagnostic => diagnostic!.GetValue().Contains("truncated after 64 entries", StringComparison.Ordinal)); } + [Fact] + public void ToolsCall_SearchRecipe_AppliesResultOutputMetadata_Issue3558() + { + InsertIndexedFile( + "src/todo.cs", + "csharp", + """ + // TODO: inspect generated SQL. + public class TodoFixture { } + """); + var recipePath = Path.Combine(_projectRoot, "search-recipes-metadata.json"); + File.WriteAllText(recipePath, """ + { + "recipes": [ + { + "name": "local-audit", + "description": "Local audit recipe", + "queries": [ + { + "name": "todo-comments", + "query": "TODO", + "description": "Find local TODO markers", + "recommendedLabels": ["audit"], + "falsePositiveGuidance": "Ignore deliberate test fixtures.", + "exactSubstring": true + } + ] + } + ] + } + """); + using var env = EnvironmentVariableScope.Capture("CDIDX_SEARCH_RECIPE_PATHS"); + env.Set("CDIDX_SEARCH_RECIPE_PATHS", recipePath); + var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"search","arguments":{"recipe":"local-audit","snippetLines":3,"maxLineWidth":96}}}""")!; + + var response = _server.HandleMessage(request)!; + + Assert.Null(response["error"]); + var structured = response["result"]!["structuredContent"]!; + var query = structured["queries"]![0]!; + var result = query["results"]![0]!; + Assert.Equal(3, result["snippetLines"]!.GetValue()); + Assert.Equal(96, result["maxLineWidth"]!.GetValue()); + Assert.True(result["exact"]!.GetValue()); + Assert.False(result["rawFts"]!.GetValue()); + Assert.True(result["literalHighlightsAvailable"]!.GetValue()); + Assert.Null(result["literalHighlightWarning"]); + } + [Fact] public void ToolsCall_Search_AcceptsScalarExcludePaths_Issue3538() { @@ -4789,7 +4838,29 @@ public void ToolsCall_Search_RawQuerySupportsFtsSyntax() Assert.Equal(1, response["result"]!["structuredContent"]!["count"]!.GetValue()); Assert.True(response["result"]!["structuredContent"]!["rawQuery"]!.GetValue()); - Assert.Equal("src/app.cs", response["result"]!["structuredContent"]!["results"]![0]!["path"]!.GetValue()); + var result = response["result"]!["structuredContent"]!["results"]![0]!; + Assert.Equal("src/app.cs", result["path"]!.GetValue()); + Assert.True(result["rawFts"]!.GetValue()); + Assert.False(result["exact"]!.GetValue()); + Assert.False(result["literalHighlightsAvailable"]!.GetValue()); + Assert.Equal("literal_highlights_unavailable_raw_fts", result["literalHighlightWarning"]!.GetValue()); + } + + [Fact] + public void ToolsCall_Search_ExactWithRawQueryReportsEffectiveLiteralHighlightMode_Issue3558() + { + var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"search","arguments":{"query":"App","rawQuery":true,"exact":true}}}""")!; + var response = _server.HandleMessage(request)!; + + Assert.Equal(1, response["result"]!["structuredContent"]!["count"]!.GetValue()); + Assert.True(response["result"]!["structuredContent"]!["rawQuery"]!.GetValue()); + var result = response["result"]!["structuredContent"]!["results"]![0]!; + Assert.True(result["exact"]!.GetValue()); + Assert.False(result["rawFts"]!.GetValue()); + Assert.True(result["literalHighlightsAvailable"]!.GetValue()); + Assert.Null(result["literalHighlightWarning"]); + var highlight = result["highlights"]![0]!; + Assert.Equal("App", highlight["literalTerms"]![0]!.GetValue()); } [Theory] @@ -7863,6 +7934,14 @@ public void ToolsCall_Excerpt_ClampsLongSingleLineContent() Assert.Contains("TARGET", structured["content"]!.GetValue()); Assert.True(structured["content"]!.GetValue().Length <= 96); Assert.Equal(96, structured["maxLineWidth"]!.GetValue()); + var recovery = structured["contentRecovery"]!; + Assert.Equal(1, recovery["startLine"]!.GetValue()); + Assert.Equal(1, recovery["endLine"]!.GetValue()); + var recoveryCommand = recovery["command"]!.GetValue(); + Assert.Contains("cdidx excerpt dist/data.txt", recoveryCommand); + Assert.Contains("--db", recoveryCommand); + Assert.Contains(_dbPath, recoveryCommand); + Assert.Contains("--start 1 --end 1 --max-line-width 0 --json", recoveryCommand); } [Fact] diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs index c53a53bfbe..a3feb90a23 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs @@ -154,6 +154,50 @@ public void Run({{parameters}}) { } } } + [Fact] + public void RunInspect_JsonBodyRecoveryCommandIncludesActiveDb_Issue3562() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_inspect_body_recovery_db_3562"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var longLiteral = new string('a', DbReader.DefinitionBodyMaxBytes + 1024); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/huge_body.py", + "python", + $"def huge_body():\n value = \"{longLiteral}\"\n return value\n"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunInspect( + ["huge_body", "--db", dbPath, "--read-only", "--json", "--body", "--lang", "python", "--exact-name"], + _jsonOptions)); + + using var document = ParseJsonOutput(stdout); + var definition = document.RootElement + .GetProperty("definitions") + .EnumerateArray() + .Single(item => item.GetProperty("name").GetString() == "huge_body"); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.True(definition.GetProperty("body_content_truncated").GetBoolean()); + var recovery = definition.GetProperty("body_content_recovery"); + var recoveryCommand = recovery.GetProperty("command").GetString(); + Assert.Contains("cdidx excerpt src/huge_body.py", recoveryCommand); + Assert.Contains("--db", recoveryCommand); + Assert.Contains("file:", recoveryCommand); + var expectedReadOnlyUri = DbContext.ToReadOnlyUri(dbPath); + Assert.Contains(expectedReadOnlyUri, recoveryCommand); + Assert.Contains("immutable=1", recoveryCommand); + Assert.Contains("mode=ro", recoveryCommand); + Assert.Contains("--start 2 --end 3 --max-line-width 0 --json", recoveryCommand); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunInspect_ParseFields_ImplyJsonAndCanonicalizeAliases_Issue3056() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSearchHintTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSearchHintTests.cs index 7b70c43655..5103decb8f 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSearchHintTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSearchHintTests.cs @@ -23,11 +23,18 @@ public void RunSearch_ExactSubstringJsonOutputsLiteralHighlightMetadata() _jsonOptions)); using var document = ParseJsonOutput(stdout); + var root = document.RootElement; var highlight = document.RootElement.GetProperty("highlights")[0]; var literalOccurrence = highlight.GetProperty("literal_term_occurrences")[0]; Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal(string.Empty, stderr); + Assert.Equal(2, root.GetProperty("snippet_lines").GetInt32()); + Assert.Equal(512, root.GetProperty("max_line_width").GetInt32()); + Assert.True(root.GetProperty("exact").GetBoolean()); + Assert.False(root.GetProperty("raw_fts").GetBoolean()); + Assert.True(root.GetProperty("literal_highlights_available").GetBoolean()); + Assert.False(root.TryGetProperty("literal_highlight_warning", out _)); Assert.Equal("CommandText = $", highlight.GetProperty("literal_terms")[0].GetString()); Assert.Equal("CommandText = $", literalOccurrence.GetProperty("term").GetString()); Assert.Equal(1, literalOccurrence.GetProperty("line").GetInt32()); @@ -40,6 +47,77 @@ public void RunSearch_ExactSubstringJsonOutputsLiteralHighlightMetadata() } } + [Fact] + public void RunSearch_ExactSubstringWithRawFtsReportsEffectiveLiteralHighlightMode_Issue3558() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_query_runner_search_exact_raw_fts_metadata_3558"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/sql.cs", + "csharp", + "var CommandText = $\"SELECT 1\";\nvar CommandText = other;"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunSearch( + ["CommandText = $", "--db", dbPath, "--json", "--fts", "--exact-substring"], + _jsonOptions)); + + using var document = ParseJsonOutput(stdout); + var root = document.RootElement; + var highlight = root.GetProperty("highlights")[0]; + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.True(root.GetProperty("exact").GetBoolean()); + Assert.False(root.GetProperty("raw_fts").GetBoolean()); + Assert.True(root.GetProperty("literal_highlights_available").GetBoolean()); + Assert.False(root.TryGetProperty("literal_highlight_warning", out _)); + Assert.Equal("CommandText = $", highlight.GetProperty("literal_terms")[0].GetString()); + Assert.Equal("CommandText = $", highlight.GetProperty("literal_term_occurrences")[0].GetProperty("term").GetString()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void RunSearch_RawFtsJsonReportsLiteralHighlightGapMetadata_Issue3558() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_query_runner_search_raw_fts_metadata_3558"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/sql.cs", + "csharp", + "var CommandText = $\"SELECT 1\";\nvar CommandText = other;"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunSearch( + ["CommandText", "--db", dbPath, "--json", "--fts", "--snippet-lines", "3", "--max-line-width", "80"], + _jsonOptions)); + + using var document = ParseJsonOutput(stdout); + var root = document.RootElement; + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal(3, root.GetProperty("snippet_lines").GetInt32()); + Assert.Equal(80, root.GetProperty("max_line_width").GetInt32()); + Assert.False(root.GetProperty("exact").GetBoolean()); + Assert.True(root.GetProperty("raw_fts").GetBoolean()); + Assert.False(root.GetProperty("literal_highlights_available").GetBoolean()); + Assert.Equal("literal_highlights_unavailable_raw_fts", root.GetProperty("literal_highlight_warning").GetString()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunSearch_PunctuationHeavyTextSuggestsExactSubstring() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs index d29dab1792..991740b1dd 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs @@ -798,6 +798,53 @@ public void Run(Exception ex, CancellationToken token) } } + [Fact] + public void RunSearch_RecipeJsonWithRawFtsReportsEffectiveSanitizedMode_Issue3558() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_search_recipe_json_raw_fts_3558"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/app.cs", + "csharp", + """ + using System.Text.Json; + + public sealed class App + { + public void Run() + { + JsonDocument.Parse("{}"); + } + } + """); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunSearch( + ["--recipe", "risky-code", "--db", dbPath, "--lang", "csharp", "--limit", "2", "--json", "--fts"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + using var document = ParseJsonOutput(stdout); + var result = document.RootElement + .GetProperty("queries") + .EnumerateArray() + .Single(item => item.GetProperty("name").GetString() == "unbounded-json-parse") + .GetProperty("results") + .EnumerateArray() + .Single(); + + Assert.False(result.GetProperty("raw_fts").GetBoolean()); + Assert.False(result.TryGetProperty("literal_highlight_warning", out _)); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Theory] [InlineData("count")] [InlineData("compact")] @@ -2291,7 +2338,24 @@ public void RunExcerpt_JsonClampsLongSingleLineContentAroundFocus() Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal(string.Empty, stderr); + Assert.Equal(1, json.GetProperty("requested_start_line").GetInt32()); + Assert.Equal(1, json.GetProperty("requested_end_line").GetInt32()); + Assert.Equal(1, json.GetProperty("effective_start_line").GetInt32()); + Assert.Equal(1, json.GetProperty("effective_end_line").GetInt32()); Assert.True(json.GetProperty("content_truncated").GetBoolean()); + var truncationReasons = json.GetProperty("content_truncation_reasons") + .EnumerateArray() + .Select(reason => reason.GetString()) + .ToArray(); + Assert.Contains("line_width_cap", truncationReasons); + var recovery = json.GetProperty("content_recovery"); + Assert.Equal(1, recovery.GetProperty("start_line").GetInt32()); + Assert.Equal(1, recovery.GetProperty("end_line").GetInt32()); + var recoveryCommand = recovery.GetProperty("command").GetString(); + Assert.Contains("cdidx excerpt dist/data.txt", recoveryCommand); + Assert.Contains("--db", recoveryCommand); + Assert.Contains(dbPath, recoveryCommand); + Assert.Contains("--start 1 --end 1 --max-line-width 0 --json", recoveryCommand); Assert.DoesNotContain(longLine, json.GetProperty("content").GetString()); Assert.Contains("TARGET", json.GetProperty("content").GetString()); Assert.True(json.GetProperty("content").GetString()!.Length <= 96); @@ -2519,9 +2583,18 @@ public void RunFind_JsonClampsLongSingleLineSnippet() Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal(string.Empty, stderr); + Assert.Equal(321, json.GetProperty("column").GetInt32()); + Assert.Equal(6, json.GetProperty("length").GetInt32()); + Assert.Equal(longLine.Length, json.GetProperty("original_line_length").GetInt32()); Assert.True(json.GetProperty("snippet_truncated").GetBoolean()); Assert.Contains("target", json.GetProperty("snippet").GetString()); Assert.True(json.GetProperty("snippet").GetString()!.Length <= 96); + var truncationContext = json.GetProperty("snippet_truncation_context"); + Assert.Equal(1, truncationContext.GetProperty("line_count").GetInt32()); + var charCount = Assert.Single(truncationContext.GetProperty("char_counts").EnumerateArray()); + Assert.True(charCount.GetInt32() > 0); + Assert.Equal(charCount.GetInt32(), truncationContext.GetProperty("total_chars").GetInt32()); + Assert.Equal("line_width", truncationContext.GetProperty("reason").GetString()); } finally { @@ -4251,10 +4324,63 @@ public void RunFind_WithJsonOutputsLineColumnAndSnippet() Assert.Equal("src/Auth.cs", json.GetProperty("path").GetString()); Assert.Equal(3, json.GetProperty("line").GetInt32()); Assert.Equal(10, json.GetProperty("column").GetInt32()); + Assert.Equal(5, json.GetProperty("length").GetInt32()); + Assert.Equal(" void Guard() {}".Length, json.GetProperty("original_line_length").GetInt32()); Assert.Equal(2, json.GetProperty("start_line").GetInt32()); Assert.Equal(4, json.GetProperty("end_line").GetInt32()); Assert.Contains("void Guard()", json.GetProperty("snippet").GetString()); Assert.Contains("void Next()", json.GetProperty("snippet").GetString()); + var truncationContext = json.GetProperty("snippet_truncation_context"); + Assert.Equal(0, truncationContext.GetProperty("line_count").GetInt32()); + Assert.Empty(truncationContext.GetProperty("char_counts").EnumerateArray()); + Assert.Equal(0, truncationContext.GetProperty("total_chars").GetInt32()); + Assert.False(truncationContext.TryGetProperty("reason", out _)); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void RunFind_WithJsonReportsSpanMetadataForMultipleMatchesInOneFile_Issue3561() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_query_runner_find_span_metadata_3561"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/search.txt", + "text", + "alpha target\nmiddle\nsecond target here\n"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunFind( + ["target", "--db", dbPath, "--path", "src/search.txt", "--json"], + _jsonOptions)); + + var rows = ParseJsonLines(stdout).Select(document => document.RootElement).ToList(); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal(2, rows.Count); + Assert.All(rows, row => + { + Assert.Equal("src/search.txt", row.GetProperty("path").GetString()); + Assert.Equal(6, row.GetProperty("length").GetInt32()); + Assert.False(row.GetProperty("snippet_truncated").GetBoolean()); + var truncationContext = row.GetProperty("snippet_truncation_context"); + Assert.Equal(0, truncationContext.GetProperty("line_count").GetInt32()); + Assert.Empty(truncationContext.GetProperty("char_counts").EnumerateArray()); + Assert.Equal(0, truncationContext.GetProperty("total_chars").GetInt32()); + Assert.False(truncationContext.TryGetProperty("reason", out _)); + }); + Assert.Equal(1, rows[0].GetProperty("line").GetInt32()); + Assert.Equal(7, rows[0].GetProperty("column").GetInt32()); + Assert.Equal("alpha target".Length, rows[0].GetProperty("original_line_length").GetInt32()); + Assert.Equal(3, rows[1].GetProperty("line").GetInt32()); + Assert.Equal(8, rows[1].GetProperty("column").GetInt32()); + Assert.Equal("second target here".Length, rows[1].GetProperty("original_line_length").GetInt32()); } finally { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs index 8194dc8b95..db2bf7ab35 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs @@ -83,6 +83,21 @@ public void RunDefinition_JsonBodyIncludesTruncationMetadata_Issue3131() Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal(string.Empty, stderr); Assert.True(json.GetProperty("body_content_truncated").GetBoolean()); + Assert.Equal(2, json.GetProperty("body_requested_start_line").GetInt32()); + Assert.True(json.GetProperty("body_requested_end_line").GetInt32() > json.GetProperty("body_effective_end_line").GetInt32()); + Assert.Equal(2, json.GetProperty("body_effective_start_line").GetInt32()); + var bodyReasons = json.GetProperty("body_content_truncation_reasons") + .EnumerateArray() + .Select(reason => reason.GetString()) + .ToArray(); + Assert.Contains("body_line_cap", bodyReasons); + var recovery = json.GetProperty("body_content_recovery"); + Assert.Equal(json.GetProperty("body_effective_end_line").GetInt32() + 1, recovery.GetProperty("start_line").GetInt32()); + var recoveryCommand = recovery.GetProperty("command").GetString(); + Assert.Contains("cdidx excerpt src/long_body.py", recoveryCommand); + Assert.Contains("--db", recoveryCommand); + Assert.Contains(dbPath, recoveryCommand); + Assert.Contains("--max-line-width 0 --json", recoveryCommand); Assert.False(json.TryGetProperty("complexity", out _)); Assert.True(CountLines(json.GetProperty("body_content").GetString()!) <= DbReader.DefinitionBodyMaxLines); Assert.DoesNotContain("value_23", json.GetProperty("body_content").GetString()); @@ -93,6 +108,52 @@ public void RunDefinition_JsonBodyIncludesTruncationMetadata_Issue3131() } } + [Fact] + public void RunDefinition_JsonBodyReportsByteCapRecovery_Issue3562() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_definition_body_byte_recovery_3562"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var longLiteral = new string('a', DbReader.DefinitionBodyMaxBytes + 1024); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/huge_body.py", + "python", + $"def huge_body():\n value = \"{longLiteral}\"\n return value\n"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunDefinition( + ["huge_body", "--db", dbPath, "--json", "--body", "--lang", "python", "--exact-name"], + _jsonOptions)); + using var document = ParseJsonOutput(stdout); + var json = document.RootElement; + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.True(json.GetProperty("body_content_truncated").GetBoolean()); + var bodyReasons = json.GetProperty("body_content_truncation_reasons") + .EnumerateArray() + .Select(reason => reason.GetString()) + .ToArray(); + Assert.Contains("body_byte_cap", bodyReasons); + Assert.Equal(2, json.GetProperty("body_effective_start_line").GetInt32()); + Assert.Equal(3, json.GetProperty("body_effective_end_line").GetInt32()); + var recovery = json.GetProperty("body_content_recovery"); + Assert.Equal(2, recovery.GetProperty("start_line").GetInt32()); + Assert.Equal(3, recovery.GetProperty("end_line").GetInt32()); + var recoveryCommand = recovery.GetProperty("command").GetString(); + Assert.Contains("cdidx excerpt src/huge_body.py", recoveryCommand); + Assert.Contains("--db", recoveryCommand); + Assert.Contains(dbPath, recoveryCommand); + Assert.Contains("--start 2 --end 3 --max-line-width 0 --json", recoveryCommand); + Assert.False(json.TryGetProperty("complexity", out _)); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunSymbols_ExactNameFindsPythonDottedImportPrefixes() { diff --git a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs index eb9268e572..a8c0b1cf36 100644 --- a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +++ b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs @@ -43,6 +43,11 @@ public void BuildExcerpt_ReportsDroppedMatchLines_WhenTailMatchesExceedWindow() Assert.Equal([20, 21, 22], excerpt.MatchLines); Assert.Equal(2, excerpt.DroppedMatchLineCount); + Assert.Equal("quality", excerpt.FocusMode); + Assert.Equal(20, excerpt.FocusLine); + Assert.Equal(1, excerpt.FocusColumn); + Assert.Equal("full_query", excerpt.FocusReason); + Assert.Equal(23, excerpt.NextMatchLine); } [Fact] @@ -64,6 +69,24 @@ public void BuildExcerpt_ReportsEveryTermOccurrenceWithPositions() }); } + [Fact] + public void BuildExcerpt_ReportsVisibleOccurrenceRangesAfterLineClamping_Issue3557() + { + var content = "Target " + new string('x', 200) + " Target"; + + var excerpt = SearchSnippetFormatter.BuildExcerpt(content, "Target", absoluteStartLine: 3, maxLines: 1, maxLineWidth: 48); + + var highlight = Assert.Single(excerpt.Highlights); + var occurrences = highlight.TermOccurrences.OrderBy(occurrence => occurrence.Column).ToArray(); + Assert.Equal(2, occurrences.Length); + Assert.True(occurrences[0].Visible); + Assert.Equal(1, occurrences[0].VisibleColumn); + Assert.Equal("Target".Length, occurrences[0].VisibleLength); + Assert.False(occurrences[1].Visible); + Assert.Null(occurrences[1].VisibleColumn); + Assert.Null(occurrences[1].VisibleLength); + } + [Fact] public void BuildExcerpt_ExactSubstringExposesLiteralOnlyHighlights() { @@ -434,6 +457,31 @@ public void ToCompactResult_ReportsTruncationMetadata_WhenLineIsClamped() Assert.Equal(compact.TruncationContext.CharCounts, compact.Highlights[0].TruncatedCharCounts); } + [Fact] + public void ToCompactResult_ReportsFocusAndNextDroppedMatchMetadata_Issue3556() + { + var content = string.Join('\n', Enumerable.Range(1, 5).Select(i => $"Target {i}")); + var result = new SearchResult + { + Path = "src/app.cs", + Lang = "csharp", + StartLine = 10, + EndLine = 14, + Content = content, + Score = -1.0, + }; + + var compact = SearchSnippetFormatter.ToCompactResult(result, "Target", maxLines: 3); + + Assert.Equal("quality", compact.FocusMode); + Assert.Equal(10, compact.FocusLine); + Assert.Equal(1, compact.FocusColumn); + Assert.Equal("full_query", compact.FocusReason); + Assert.NotNull(compact.NextMatch); + Assert.Equal(13, compact.NextMatch.Line); + Assert.Equal(2, compact.NextMatch.RemainingMatchLineCount); + } + [Fact] public void ToCompactResult_ReportsTruncationContext_WhenMultipleSnippetLinesAreClamped() { diff --git a/tests/CodeIndex.Tests/golden/excerpt.json b/tests/CodeIndex.Tests/golden/excerpt.json index bf800a6bc0..5e73b41ce6 100644 --- a/tests/CodeIndex.Tests/golden/excerpt.json +++ b/tests/CodeIndex.Tests/golden/excerpt.json @@ -4,8 +4,13 @@ "lang": "csharp", "start_line": 1, "end_line": 6, + "requested_start_line": 1, + "requested_end_line": 6, + "effective_start_line": 1, + "effective_end_line": 6, "content": "namespace Demo;\n\npublic static class Lib\n{\n public static int Add(int a, int b) =\u003E a \u002B b;\n}", "content_truncated": false, + "content_truncation_reasons": [], "semantic_tokens": [ { "start_line": 1, diff --git a/tests/CodeIndex.Tests/golden/search.json b/tests/CodeIndex.Tests/golden/search.json index 2e376913bf..e2af345175 100644 --- a/tests/CodeIndex.Tests/golden/search.json +++ b/tests/CodeIndex.Tests/golden/search.json @@ -27,7 +27,10 @@ "term": "Add", "line": 5, "column": 23, - "length": 3 + "length": 3, + "visible": true, + "visible_column": 23, + "visible_length": 3 } ], "match_origins": [ @@ -59,6 +62,15 @@ "context_after": 2, "truncated_line_count": 0, "dropped_match_line_count": 0, + "snippet_lines": 8, + "max_line_width": 512, + "exact": false, + "raw_fts": false, + "literal_highlights_available": false, + "focus_mode": "quality", + "focus_line": 5, + "focus_column": 23, + "focus_reason": "full_query", "truncation_context": { "line_count": 0, "char_counts": [],