From 0c287d4e1be7fac9e9eaafe01936b7ba6771e4a0 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 12:43:55 +0900 Subject: [PATCH 1/4] Add excerpt semantic token ranges (#1746) --- changelog.d/unreleased/1746.added.md | 17 ++++++ src/CodeIndex/Cli/JsonOutputContracts.cs | 2 + src/CodeIndex/Cli/QueryCommandRunner.cs | 53 +++++++++++++++++++ src/CodeIndex/Models/QueryResults.cs | 12 +++++ .../QueryCommandRunnerTests.cs | 5 ++ 5 files changed, 89 insertions(+) create mode 100644 changelog.d/unreleased/1746.added.md diff --git a/changelog.d/unreleased/1746.added.md b/changelog.d/unreleased/1746.added.md new file mode 100644 index 0000000000..d0d206a093 --- /dev/null +++ b/changelog.d/unreleased/1746.added.md @@ -0,0 +1,17 @@ +--- +category: added +issues: + - 1746 +affected: + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Models/QueryResults.cs + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +--- + +## English + +- **Excerpt JSON now includes semantic token ranges (#1746)** — `excerpt --json` emits `semantic_tokens` with 1-based ranges and token types so IDE and LLM clients can render excerpt spans without reparsing `content`. + +## 日本語 + +- **excerpt JSON が semantic token range を含むようになりました (#1746)** — `excerpt --json` は 1-based range と token type を持つ `semantic_tokens` を返し、IDE や LLM クライアントが `content` を再パースせずに抜粋範囲を描画できるようにします。 diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 88d6d47160..507e8650fb 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -282,6 +282,7 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(DiffSummaryOnlyJsonResult))] [JsonSerializable(typeof(DiffSummaryJsonResult))] [JsonSerializable(typeof(ExactZeroHintResult))] +[JsonSerializable(typeof(ExcerptSemanticToken))] [JsonSerializable(typeof(FileDependencyResult))] [JsonSerializable(typeof(FileExcerptResult))] [JsonSerializable(typeof(FileFindResult))] @@ -337,6 +338,7 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(ReportBundleSummary))] [JsonSerializable(typeof(SearchHighlight))] [JsonSerializable(typeof(SearchResult))] +[JsonSerializable(typeof(SearchTermOccurrence))] [JsonSerializable(typeof(SearchTruncationContext))] [JsonSerializable(typeof(StatusResult))] [JsonSerializable(typeof(StatusDbPragmaSettings))] diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index dc12eefc54..d1610d4804 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -1474,6 +1474,8 @@ public static int RunExcerpt(string[] cmdArgs, JsonSerializerOptions jsonOptions Console.Error.WriteLine("No excerpt found."); return ZeroResultExitCode(options); } + if (options.Json) + excerpt.SemanticTokens = BuildExcerptSemanticTokens(excerpt); if (options.Json) { @@ -1488,6 +1490,57 @@ public static int RunExcerpt(string[] cmdArgs, JsonSerializerOptions jsonOptions }); } + private static List BuildExcerptSemanticTokens(FileExcerptResult excerpt) + { + var tokens = new List(); + var lines = excerpt.Content.Replace("\r\n", "\n").Split('\n'); + for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) + { + var line = lines[lineIndex]; + var column = 0; + while (column < line.Length) + { + if (!IsSemanticTokenStart(line[column])) + { + column++; + continue; + } + + var start = column; + column++; + while (column < line.Length && IsSemanticTokenPart(line[column])) + column++; + + var tokenText = line[start..column]; + tokens.Add(new ExcerptSemanticToken + { + StartLine = excerpt.StartLine + lineIndex, + StartColumn = start + 1, + EndLine = excerpt.StartLine + lineIndex, + EndColumn = column + 1, + Type = ClassifySemanticToken(tokenText), + }); + } + } + + return tokens; + } + + private static bool IsSemanticTokenStart(char value) => + char.IsLetter(value) || value == '_' || char.IsDigit(value); + + private static bool IsSemanticTokenPart(char value) => + char.IsLetterOrDigit(value) || value == '_'; + + private static string ClassifySemanticToken(string token) + { + if (token.All(char.IsDigit)) + return "number"; + if (char.IsUpper(token[0])) + return "type"; + return "variable"; + } + public static int RunFind(string[] cmdArgs, JsonSerializerOptions jsonOptions) { var preparedFindArgs = PrepareFindArgs(cmdArgs, out var preparationError); diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index fbc738c58d..6df8580273 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -98,6 +98,18 @@ public class FileExcerptResult public int EndLine { get; set; } public string Content { get; set; } = string.Empty; public bool ContentTruncated { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? SemanticTokens { get; set; } +} + +public class ExcerptSemanticToken +{ + public int StartLine { get; set; } + public int StartColumn { get; set; } + public int EndLine { get; set; } + public int EndColumn { get; set; } + public string Type { get; set; } = string.Empty; + public List Modifiers { get; set; } = []; } public class FileFindResult diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index 2c853e7fd4..8570e0fabb 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -4755,6 +4755,11 @@ public void RunExcerpt_JsonClampsLongSingleLineContentAroundFocus() Assert.DoesNotContain(longLine, json.GetProperty("content").GetString()); Assert.Contains("TARGET", json.GetProperty("content").GetString()); Assert.True(json.GetProperty("content").GetString()!.Length <= 96); + var semanticTokens = json.GetProperty("semantic_tokens").EnumerateArray().ToArray(); + Assert.Contains(semanticTokens, token => + token.GetProperty("type").GetString() == "variable" && + token.GetProperty("start_line").GetInt32() == 1 && + token.GetProperty("start_column").GetInt32() > 0); } finally { From db9f9d536b9482995730b1f16d07cb18a0b2ccd0 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 12:45:01 +0900 Subject: [PATCH 2/4] Report search snippet match metadata (#1775, #1773) --- DEVELOPER_GUIDE.md | 8 ++- changelog.d/unreleased/1773.fixed.md | 16 +++++ changelog.d/unreleased/1775.fixed.md | 16 +++++ src/CodeIndex/Cli/SearchSnippetFormatter.cs | 70 ++++++++++++++++--- .../SearchSnippetFormatterTests.cs | 31 ++++++++ 5 files changed, 130 insertions(+), 11 deletions(-) create mode 100644 changelog.d/unreleased/1773.fixed.md create mode 100644 changelog.d/unreleased/1775.fixed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 9c6f21105e..fadb0cf031 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -707,7 +707,9 @@ Exact-match flag compatibility is documented in [USER_GUIDE.md](USER_GUIDE.md#fl `search`, `definition`, `references`, `callers`, `callees`, `symbols`, and `files` also share path-aware narrowing via `--path`, repeatable `--exclude-path`, and `--exclude-tests`. The read layer ranks source files ahead of tests and docs, and `search` further boosts exact symbol-name and path matches so AI clients are more likely to land on implementation files first. -`search --json` and MCP `search` project full chunks into compact match-centered snippets with `chunk_start_line`, `chunk_end_line`, `snippet_start_line`, `snippet_end_line`, `snippet`, `match_lines`, `highlights`, `context_before`, `context_after`, `truncated_line_count`, and `truncation_context`. `--snippet-lines` caps the snippet length up front (default: 8, max: 20), and `--max-line-width` (CLI) / `maxLineWidth` (MCP) clamps each individual snippet line around the first match token via the shared `LineWidthFormatter.ClampLine` contract used by `find` / `references` / `excerpt` / `inspect` (default: 512, max: 4096) so a single match inside a minified / transpiled / generated single-line file no longer returns hundreds of KB per hit. Clamped lines surface `...(+N)...` markers inside the snippet and expose `truncation_context.char_counts`, `truncation_context.total_chars`, `highlights[].truncated`, `highlights[].original_line_length`, and `highlights[].truncated_char_counts` so AI clients can detect clamping and quantify omitted characters. +`search --json` and MCP `search` project full chunks into compact match-centered snippets with `chunk_start_line`, `chunk_end_line`, `snippet_start_line`, `snippet_end_line`, `snippet`, `match_lines`, `highlights`, `context_before`, `context_after`, `truncated_line_count`, `dropped_match_line_count`, and `truncation_context`. `--snippet-lines` caps the snippet length up front (default: 8, max: 20), and `--max-line-width` (CLI) / `maxLineWidth` (MCP) clamps each individual snippet line around the first match token via the shared `LineWidthFormatter.ClampLine` contract used by `find` / `references` / `excerpt` / `inspect` (default: 512, max: 4096) so a single match inside a minified / transpiled / generated single-line file no longer returns hundreds of KB per hit. Clamped lines surface `...(+N)...` markers inside the snippet and expose `truncation_context.char_counts`, `truncation_context.total_chars`, `highlights[].truncated`, `highlights[].original_line_length`, and `highlights[].truncated_char_counts` so AI clients can detect clamping and quantify omitted characters. `highlights[].terms` remains a distinct term list for compatibility; `highlights[].term_occurrences` records every matched occurrence with `term`, 1-based `line`, 1-based `column`, and `length`. `dropped_match_line_count` reports match lines omitted because they fell outside the selected snippet window. + +`excerpt --json` includes `semantic_tokens`, a lightweight range list with 1-based start/end positions, token `type`, and `modifiers`, so IDE and LLM clients can render or post-process excerpt spans without reparsing the raw `content` string. `inspect` and MCP `analyze_symbol` bundle the primary definition, nearby symbols from the same file, references, callers, callees, file metadata, workspace freshness/git metadata, and graph-support metadata into one response. When those bundled graph sections actually depend on SQL-backed reads, the payload also mirrors `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason` (plus the existing camelCase aliases on MCP responses); mixed-language bundles that only return C# / JS / etc. graph rows omit the SQL trust signal entirely. This is intended for symbol-oriented AI workflows that would otherwise need several back-to-back calls. Call graph sections remain language-aware: for unsupported languages, clients can now distinguish "unsupported" from "no hits" via `graphSupported` / `graphSupportReason`, and should prefer `search` instead of assuming graph data will exist. @@ -2265,7 +2267,9 @@ exact-match flag の互換性は [USER_GUIDE.md](USER_GUIDE.md#フラグ互換 `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files` は `--path`、繰り返し指定できる `--exclude-path`、`--exclude-tests` による絞り込みを共有します。読み取り層は tests や docs より source を優先し、`search` はシンボル名やパスがクエリと正確に一致する候補をさらに上位に出して、AIクライアントが実装ファイルへ早く到達できるようにします。 -`search --json` と MCP の `search` は、フルチャンクを `chunk_start_line`、`chunk_end_line`、`snippet_start_line`、`snippet_end_line`、`snippet`、`match_lines`、`highlights`、`context_before`、`context_after`、`truncated_line_count`、`truncation_context` を持つ軽量スニペットへ投影します。`--snippet-lines` で抜粋長を先に制限でき（デフォルト: 8、最大: 20）、`--max-line-width`（CLI）/ `maxLineWidth`（MCP）は `find` / `references` / `excerpt` / `inspect` と同じ共有 `LineWidthFormatter.ClampLine` 契約（デフォルト: 512、最大: 4096、`0` で切り詰め解除）で各スニペット行を最初のマッチトークン周辺にクランプするため、minified / transpiled / 生成された 1 行ファイル内の 1 ヒットで数百 KB を返さなくなります。クランプされた行はスニペットに `...(+N)...` マーカーが入り、`truncation_context.char_counts`、`truncation_context.total_chars`、`highlights[].truncated`、`highlights[].original_line_length`、`highlights[].truncated_char_counts` で AI クライアントがクランプの有無と省略文字数を検出できます。 +`search --json` と MCP の `search` は、フルチャンクを `chunk_start_line`、`chunk_end_line`、`snippet_start_line`、`snippet_end_line`、`snippet`、`match_lines`、`highlights`、`context_before`、`context_after`、`truncated_line_count`、`dropped_match_line_count`、`truncation_context` を持つ軽量スニペットへ投影します。`--snippet-lines` で抜粋長を先に制限でき（デフォルト: 8、最大: 20）、`--max-line-width`（CLI）/ `maxLineWidth`（MCP）は `find` / `references` / `excerpt` / `inspect` と同じ共有 `LineWidthFormatter.ClampLine` 契約（デフォルト: 512、最大: 4096、`0` で切り詰め解除）で各スニペット行を最初のマッチトークン周辺にクランプするため、minified / transpiled / 生成された 1 行ファイル内の 1 ヒットで数百 KB を返さなくなります。クランプされた行はスニペットに `...(+N)...` マーカーが入り、`truncation_context.char_counts`、`truncation_context.total_chars`、`highlights[].truncated`、`highlights[].original_line_length`、`highlights[].truncated_char_counts` で AI クライアントがクランプの有無と省略文字数を検出できます。`highlights[].terms` は互換性のため distinct な term list のまま残し、`highlights[].term_occurrences` は一致ごとの `term`、1-based の `line` / `column`、`length` を記録します。`dropped_match_line_count` は選択された snippet window 外に落ちた一致行数を示します。 + +`excerpt --json` は 1-based の開始/終了位置、token `type`、`modifiers` を持つ軽量 range list の `semantic_tokens` を返すため、IDE や LLM クライアントは生の `content` 文字列を再パースせずに抜粋範囲を描画・後処理できます。 `inspect` と MCP の `analyze_symbol` は、主定義、同一ファイル内の近傍シンボル、参照、caller、callee、ファイルメタデータ、さらにワークスペース鮮度/git メタデータと graph 対応メタデータを1レスポンスにまとめます。bundle 内の graph 節が実際に SQL ベースの read に依存する場合だけ、`sql_graph_contract_ready` / `sql_graph_contract_degraded_reason`（MCP では既存の camelCase alias も）も返します。mixed-language bundle で C# / JS などの graph row しか返っていない場合は SQL trust signal を出さないため、無関係なクエリが stale SQL state に引きずられません。複数の連続クエリを避けたい AI ワークフロー向けです。call graph 系の節は言語差分を考慮しており、未対応言語では `graphSupported` / `graphSupportReason` によって「未対応」と「ヒットなし」を区別できます。その場合は `search` を優先して使う前提です。 diff --git a/changelog.d/unreleased/1773.fixed.md b/changelog.d/unreleased/1773.fixed.md new file mode 100644 index 0000000000..54bc351e1f --- /dev/null +++ b/changelog.d/unreleased/1773.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1773 +affected: + - src/CodeIndex/Cli/SearchSnippetFormatter.cs + - tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +--- + +## English + +- **Search highlights now include per-occurrence positions (#1773)** — `highlights[].term_occurrences` records every matched term with its line, column, and length while preserving the existing distinct `terms` list. + +## 日本語 + +- **検索 highlight が一致ごとの位置を含むようになりました (#1773)** — 既存の distinct な `terms` list は維持しつつ、`highlights[].term_occurrences` に各一致の line、column、length を記録します。 diff --git a/changelog.d/unreleased/1775.fixed.md b/changelog.d/unreleased/1775.fixed.md new file mode 100644 index 0000000000..b60a7b6365 --- /dev/null +++ b/changelog.d/unreleased/1775.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1775 +affected: + - src/CodeIndex/Cli/SearchSnippetFormatter.cs + - tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +--- + +## English + +- **Search snippets now report omitted match lines (#1775)** — `search --json` exposes `dropped_match_line_count` when additional matching lines fall outside the selected snippet window. + +## 日本語 + +- **検索スニペットが省略された一致行数を返すようになりました (#1775)** — `search --json` は、選択された snippet window 外に追加の一致行がある場合に `dropped_match_line_count` を返します。 diff --git a/src/CodeIndex/Cli/SearchSnippetFormatter.cs b/src/CodeIndex/Cli/SearchSnippetFormatter.cs index 5a58ea6904..4d89c511c0 100644 --- a/src/CodeIndex/Cli/SearchSnippetFormatter.cs +++ b/src/CodeIndex/Cli/SearchSnippetFormatter.cs @@ -49,6 +49,7 @@ public static CompactSearchResult ToCompactResult(SearchResult result, string qu ContextBefore = excerpt.ContextBefore, ContextAfter = excerpt.ContextAfter, TruncatedLineCount = excerpt.TruncatedLineCount, + DroppedMatchLineCount = excerpt.DroppedMatchLineCount, TruncationContext = excerpt.TruncationContext, Score = result.Score, }; @@ -97,13 +98,16 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, string query, in var matchIndexes = FindMatchingLineIndexes(matchLinesSource, normalizedQuery, tokens, caseSensitive); var focusStart = matchIndexes.Count > 0 ? matchIndexes[0] : 0; var focusEnd = focusStart; + var includedMatchLineCount = Math.Min(1, matchIndexes.Count); foreach (var matchIndex in matchIndexes.Skip(1)) { if ((matchIndex - focusStart) + 1 > maxLines) break; focusEnd = matchIndex; + includedMatchLineCount++; } + var droppedMatchLineCount = Math.Max(0, matchIndexes.Count - includedMatchLineCount); var focusLength = Math.Max(1, (focusEnd - focusStart) + 1); var remaining = Math.Max(0, maxLines - focusLength); @@ -160,6 +164,8 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, string query, in var absoluteLine = absoluteStartLine + i; matchLines.Add(absoluteLine); + var occurrenceLine = normalizeCSharpVerbatimNames && normalizedLines != null ? normalizedLines[i] : originalLine; + var termOccurrences = GetMatchedTermOccurrences(occurrenceLine, absoluteLine, normalizedQuery, tokens, caseSensitive); highlights.Add(new SearchHighlight { Line = absoluteLine, @@ -167,7 +173,8 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, string query, in OriginalLineLength = originalLine.Length, Truncated = clamped.Truncated, TruncatedCharCounts = clamped.Truncated ? [clamped.TruncatedCharCount] : [], - Terms = GetMatchedTerms(normalizeCSharpVerbatimNames && normalizedLines != null ? normalizedLines[i] : originalLine, normalizedQuery, tokens, caseSensitive), + Terms = GetDistinctTerms(termOccurrences), + TermOccurrences = termOccurrences, }); } @@ -183,6 +190,7 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, string query, in TruncatedBefore = start > 0, TruncatedAfter = end < lines.Length - 1, TruncatedLineCount = truncatedLineCount, + DroppedMatchLineCount = droppedMatchLineCount, TruncationContext = new SearchTruncationContext { LineCount = truncatedLineCount, @@ -390,19 +398,52 @@ private static List FindMatchingLineIndexes(string[] lines, string query, s return matches; } - private static List GetMatchedTerms(string line, string query, string[] tokens, bool caseSensitive = false) + private static List GetMatchedTermOccurrences(string line, int absoluteLine, string query, string[] tokens, bool caseSensitive = false) { var comparison = caseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; - var terms = new List(); - if (!string.IsNullOrWhiteSpace(query) && line.Contains(query, comparison)) - terms.Add(query); + var occurrences = new List(); + if (!string.IsNullOrWhiteSpace(query)) + AddTermOccurrences(occurrences, line, absoluteLine, query, comparison); foreach (var token in tokens) + AddTermOccurrences(occurrences, line, absoluteLine, token, comparison); + + return occurrences; + } + + private static void AddTermOccurrences(List occurrences, string line, int absoluteLine, string term, StringComparison comparison) + { + if (string.IsNullOrEmpty(term)) + return; + + var index = 0; + while ((index = line.IndexOf(term, index, comparison)) >= 0) { - if (terms.Contains(token, StringComparer.OrdinalIgnoreCase)) - continue; - if (line.Contains(token, comparison)) - terms.Add(token); + if (!occurrences.Any(occurrence => + occurrence.Line == absoluteLine && + occurrence.Column == index + 1 && + occurrence.Length == term.Length && + string.Equals(occurrence.Term, line.Substring(index, term.Length), StringComparison.OrdinalIgnoreCase))) + { + occurrences.Add(new SearchTermOccurrence + { + Term = line.Substring(index, term.Length), + Line = absoluteLine, + Column = index + 1, + Length = term.Length, + }); + } + index += Math.Max(1, term.Length); + } + } + + private static List GetDistinctTerms(List occurrences) + { + var terms = new List(); + foreach (var occurrence in occurrences) + { + if (!terms.Contains(occurrence.Term, StringComparer.OrdinalIgnoreCase)) + terms.Add(occurrence.Term); } return terms; @@ -443,6 +484,7 @@ public sealed class CompactSearchResult public int ContextBefore { get; set; } public int ContextAfter { get; set; } public int TruncatedLineCount { get; set; } + public int DroppedMatchLineCount { get; set; } public SearchTruncationContext TruncationContext { get; set; } = new(); public double Score { get; set; } } @@ -462,6 +504,15 @@ public sealed class SearchHighlight public bool Truncated { get; set; } public List TruncatedCharCounts { get; set; } = []; public List Terms { get; set; } = []; + public List TermOccurrences { get; set; } = []; +} + +public sealed class SearchTermOccurrence +{ + public string Term { get; set; } = string.Empty; + public int Line { get; set; } + public int Column { get; set; } + public int Length { get; set; } } public sealed class SearchTruncationContext @@ -483,5 +534,6 @@ public sealed class SearchSnippetExcerpt public bool TruncatedBefore { get; set; } public bool TruncatedAfter { get; set; } public int TruncatedLineCount { get; set; } + public int DroppedMatchLineCount { get; set; } public SearchTruncationContext TruncationContext { get; set; } = new(); } diff --git a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs index 26f3bb9193..34b8fe06cd 100644 --- a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +++ b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs @@ -30,6 +30,37 @@ public void BuildExcerpt_CapturesMultipleMatchLinesWhenTheyFit() Assert.Equal([1, 3], excerpt.MatchLines); Assert.Equal(2, excerpt.Highlights.Count); + Assert.Equal(0, excerpt.DroppedMatchLineCount); + } + + [Fact] + public void BuildExcerpt_ReportsDroppedMatchLines_WhenTailMatchesExceedWindow() + { + const string content = "Target 1\nTarget 2\nTarget 3\nTarget 4\nTarget 5"; + + var excerpt = SearchSnippetFormatter.BuildExcerpt(content, "Target", absoluteStartLine: 20, maxLines: 3); + + Assert.Equal([20, 21, 22], excerpt.MatchLines); + Assert.Equal(2, excerpt.DroppedMatchLineCount); + } + + [Fact] + public void BuildExcerpt_ReportsEveryTermOccurrenceWithPositions() + { + const string content = "Foo Foo Foo"; + + var excerpt = SearchSnippetFormatter.BuildExcerpt(content, "Foo", absoluteStartLine: 7, maxLines: 1); + + var highlight = Assert.Single(excerpt.Highlights); + Assert.Equal(["Foo"], highlight.Terms); + Assert.Equal(3, highlight.TermOccurrences.Count); + Assert.Equal([1, 5, 9], highlight.TermOccurrences.Select(occurrence => occurrence.Column).ToArray()); + Assert.All(highlight.TermOccurrences, occurrence => + { + Assert.Equal("Foo", occurrence.Term); + Assert.Equal(7, occurrence.Line); + Assert.Equal(3, occurrence.Length); + }); } [Fact] From 241444b0e6db7ba1bd2651a16b585674ad779892 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 12:55:09 +0900 Subject: [PATCH 3/4] Fix C# verbatim highlight coordinates (#1773) --- src/CodeIndex/Cli/SearchSnippetFormatter.cs | 51 ++++++++++++------- .../SearchSnippetFormatterTests.cs | 5 ++ 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/CodeIndex/Cli/SearchSnippetFormatter.cs b/src/CodeIndex/Cli/SearchSnippetFormatter.cs index 4d89c511c0..22686d899e 100644 --- a/src/CodeIndex/Cli/SearchSnippetFormatter.cs +++ b/src/CodeIndex/Cli/SearchSnippetFormatter.cs @@ -164,8 +164,9 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, string query, in var absoluteLine = absoluteStartLine + i; matchLines.Add(absoluteLine); - var occurrenceLine = normalizeCSharpVerbatimNames && normalizedLines != null ? normalizedLines[i] : originalLine; - var termOccurrences = GetMatchedTermOccurrences(occurrenceLine, absoluteLine, normalizedQuery, tokens, caseSensitive); + var termOccurrences = normalizeCSharpVerbatimNames && normalizedLines != null && rawIndexMaps != null + ? GetMatchedTermOccurrences(normalizedLines[i], absoluteLine, normalizedQuery, tokens, caseSensitive, originalLine, rawIndexMaps[i]) + : GetMatchedTermOccurrences(originalLine, absoluteLine, normalizedQuery, tokens, caseSensitive); highlights.Add(new SearchHighlight { Line = absoluteLine, @@ -173,7 +174,7 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, string query, in OriginalLineLength = originalLine.Length, Truncated = clamped.Truncated, TruncatedCharCounts = clamped.Truncated ? [clamped.TruncatedCharCount] : [], - Terms = GetDistinctTerms(termOccurrences), + Terms = GetMatchedTerms(normalizeCSharpVerbatimNames && normalizedLines != null ? normalizedLines[i] : originalLine, normalizedQuery, tokens, caseSensitive), TermOccurrences = termOccurrences, }); } @@ -398,20 +399,20 @@ private static List FindMatchingLineIndexes(string[] lines, string query, s return matches; } - private static List GetMatchedTermOccurrences(string line, int absoluteLine, string query, string[] tokens, bool caseSensitive = false) + private static List GetMatchedTermOccurrences(string line, int absoluteLine, string query, string[] tokens, bool caseSensitive = false, string? rawLine = null, int[]? rawIndexMap = null) { var comparison = caseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; var occurrences = new List(); if (!string.IsNullOrWhiteSpace(query)) - AddTermOccurrences(occurrences, line, absoluteLine, query, comparison); + AddTermOccurrences(occurrences, line, absoluteLine, query, comparison, rawLine, rawIndexMap); foreach (var token in tokens) - AddTermOccurrences(occurrences, line, absoluteLine, token, comparison); + AddTermOccurrences(occurrences, line, absoluteLine, token, comparison, rawLine, rawIndexMap); return occurrences; } - private static void AddTermOccurrences(List occurrences, string line, int absoluteLine, string term, StringComparison comparison) + private static void AddTermOccurrences(List occurrences, string line, int absoluteLine, string term, StringComparison comparison, string? rawLine, int[]? rawIndexMap) { if (string.IsNullOrEmpty(term)) return; @@ -419,31 +420,47 @@ private static void AddTermOccurrences(List occurrences, s var index = 0; while ((index = line.IndexOf(term, index, comparison)) >= 0) { + var occurrenceColumn = index + 1; + var occurrenceLength = term.Length; + var occurrenceTerm = line.Substring(index, term.Length); + if (rawLine != null && rawIndexMap != null && TryReconstructRawSpan(rawIndexMap, index, term.Length, out var rawColumn, out var rawLength)) + { + occurrenceColumn = rawColumn; + occurrenceLength = rawLength; + occurrenceTerm = rawLine.Substring(rawColumn - 1, rawLength); + } + if (!occurrences.Any(occurrence => occurrence.Line == absoluteLine && - occurrence.Column == index + 1 && - occurrence.Length == term.Length && - string.Equals(occurrence.Term, line.Substring(index, term.Length), StringComparison.OrdinalIgnoreCase))) + occurrence.Column == occurrenceColumn && + occurrence.Length == occurrenceLength && + string.Equals(occurrence.Term, occurrenceTerm, StringComparison.OrdinalIgnoreCase))) { occurrences.Add(new SearchTermOccurrence { - Term = line.Substring(index, term.Length), + Term = occurrenceTerm, Line = absoluteLine, - Column = index + 1, - Length = term.Length, + Column = occurrenceColumn, + Length = occurrenceLength, }); } index += Math.Max(1, term.Length); } } - private static List GetDistinctTerms(List occurrences) + private static List GetMatchedTerms(string line, string query, string[] tokens, bool caseSensitive = false) { + var comparison = caseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; var terms = new List(); - foreach (var occurrence in occurrences) + if (!string.IsNullOrWhiteSpace(query) && line.Contains(query, comparison)) + terms.Add(query); + + foreach (var token in tokens) { - if (!terms.Contains(occurrence.Term, StringComparer.OrdinalIgnoreCase)) - terms.Add(occurrence.Term); + if (terms.Contains(token, StringComparer.OrdinalIgnoreCase)) + continue; + if (line.Contains(token, comparison)) + terms.Add(token); } return terms; diff --git a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs index 34b8fe06cd..3b86fefea8 100644 --- a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +++ b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs @@ -126,6 +126,11 @@ public void BuildExcerpt_NormalizesCSharpVerbatimQualifiedNamesInNonExactSearch( Assert.Equal([2], excerpt.MatchLines); Assert.Single(excerpt.Highlights); Assert.Contains("Foo.Bar", excerpt.Highlights[0].Terms); + var occurrence = Assert.Single(excerpt.Highlights[0].TermOccurrences); + Assert.Equal("Foo.@Bar", occurrence.Term); + Assert.Equal(2, occurrence.Line); + Assert.Equal(8, occurrence.Column); + Assert.Equal(8, occurrence.Length); Assert.Contains("using @Foo.@Bar;", excerpt.Lines); } From 9093a28899897856d5a3b61968b498ce7097e164 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 16:55:21 +0900 Subject: [PATCH 4/4] Update JSON output snapshots (#1746, #1773, #1775) --- tests/CodeIndex.Tests/golden/excerpt.json | 132 +++++++++++++++++++++- tests/CodeIndex.Tests/golden/search.json | 9 ++ 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/tests/CodeIndex.Tests/golden/excerpt.json b/tests/CodeIndex.Tests/golden/excerpt.json index a295e1e0c5..bf800a6bc0 100644 --- a/tests/CodeIndex.Tests/golden/excerpt.json +++ b/tests/CodeIndex.Tests/golden/excerpt.json @@ -5,5 +5,135 @@ "start_line": 1, "end_line": 6, "content": "namespace Demo;\n\npublic static class Lib\n{\n public static int Add(int a, int b) =\u003E a \u002B b;\n}", - "content_truncated": false + "content_truncated": false, + "semantic_tokens": [ + { + "start_line": 1, + "start_column": 1, + "end_line": 1, + "end_column": 10, + "type": "variable", + "modifiers": [] + }, + { + "start_line": 1, + "start_column": 11, + "end_line": 1, + "end_column": 15, + "type": "type", + "modifiers": [] + }, + { + "start_line": 3, + "start_column": 1, + "end_line": 3, + "end_column": 7, + "type": "variable", + "modifiers": [] + }, + { + "start_line": 3, + "start_column": 8, + "end_line": 3, + "end_column": 14, + "type": "variable", + "modifiers": [] + }, + { + "start_line": 3, + "start_column": 15, + "end_line": 3, + "end_column": 20, + "type": "variable", + "modifiers": [] + }, + { + "start_line": 3, + "start_column": 21, + "end_line": 3, + "end_column": 24, + "type": "type", + "modifiers": [] + }, + { + "start_line": 5, + "start_column": 5, + "end_line": 5, + "end_column": 11, + "type": "variable", + "modifiers": [] + }, + { + "start_line": 5, + "start_column": 12, + "end_line": 5, + "end_column": 18, + "type": "variable", + "modifiers": [] + }, + { + "start_line": 5, + "start_column": 19, + "end_line": 5, + "end_column": 22, + "type": "variable", + "modifiers": [] + }, + { + "start_line": 5, + "start_column": 23, + "end_line": 5, + "end_column": 26, + "type": "type", + "modifiers": [] + }, + { + "start_line": 5, + "start_column": 27, + "end_line": 5, + "end_column": 30, + "type": "variable", + "modifiers": [] + }, + { + "start_line": 5, + "start_column": 31, + "end_line": 5, + "end_column": 32, + "type": "variable", + "modifiers": [] + }, + { + "start_line": 5, + "start_column": 34, + "end_line": 5, + "end_column": 37, + "type": "variable", + "modifiers": [] + }, + { + "start_line": 5, + "start_column": 38, + "end_line": 5, + "end_column": 39, + "type": "variable", + "modifiers": [] + }, + { + "start_line": 5, + "start_column": 44, + "end_line": 5, + "end_column": 45, + "type": "variable", + "modifiers": [] + }, + { + "start_line": 5, + "start_column": 48, + "end_line": 5, + "end_column": 49, + "type": "variable", + "modifiers": [] + } + ] } diff --git a/tests/CodeIndex.Tests/golden/search.json b/tests/CodeIndex.Tests/golden/search.json index 49e3f95b69..f4f34dc11b 100644 --- a/tests/CodeIndex.Tests/golden/search.json +++ b/tests/CodeIndex.Tests/golden/search.json @@ -21,12 +21,21 @@ "truncated_char_counts": [], "terms": [ "Add" + ], + "term_occurrences": [ + { + "term": "Add", + "line": 5, + "column": 23, + "length": 3 + } ] } ], "context_before": 4, "context_after": 2, "truncated_line_count": 0, + "dropped_match_line_count": 0, "truncation_context": { "line_count": 0, "char_counts": [],