diff --git a/changelog.d/unreleased/3089.internal.md b/changelog.d/unreleased/3089.internal.md new file mode 100644 index 0000000000..13732e8994 --- /dev/null +++ b/changelog.d/unreleased/3089.internal.md @@ -0,0 +1,18 @@ +--- +category: internal +issues: + - 3089 +affected: + - src/CodeIndex/Cli/SearchSnippetFormatter.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +--- + +## English + +- **Compact search formatting now reuses prepared query context (#3089)** — compact CLI and MCP search result formatting now prepares query token context once and reuses it across result snippets while preserving language-aware matching. + +## 日本語 + +- **compact search formatting が準備済み query context を再利用するようになりました (#3089)** — CLI / MCP の compact 検索結果整形では query token context を一度だけ準備し、言語別の一致処理を維持したまま各結果スニペットで再利用します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index 29ad65b862..c2e7e92498 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -1014,11 +1014,12 @@ private static List BuildSearchDisplayRows(List var rows = new List(results.Count); var seenMatchLocations = !exact || options.NoDedup ? null : new HashSet(StringComparer.Ordinal); var displayQuery = queryOverride ?? options.Query!; + var queryContext = SearchSnippetFormatter.PrepareQueryContext(displayQuery); foreach (var result in results) { var compact = SearchSnippetFormatter.ToCompactResult( result, - displayQuery, + queryContext, options.SnippetLines, exact, options.MaxLineWidth, diff --git a/src/CodeIndex/Cli/SearchSnippetFormatter.cs b/src/CodeIndex/Cli/SearchSnippetFormatter.cs index 0fca06bb5f..b6a757a7ba 100644 --- a/src/CodeIndex/Cli/SearchSnippetFormatter.cs +++ b/src/CodeIndex/Cli/SearchSnippetFormatter.cs @@ -12,9 +12,25 @@ public static class SearchSnippetFormatter public const int DefaultSnippetLines = 8; public const int MaxSnippetLines = 20; + public static SearchSnippetQueryContext PrepareQueryContext(string query) + { + var normalizedQuery = query.Trim(); + return new SearchSnippetQueryContext( + query, + new SearchSnippetPreparedQuery(normalizedQuery, BuildQueryTokens(query, normalizeCSharpVerbatimNames: false), NormalizeCSharpVerbatimNames: false), + new SearchSnippetPreparedQuery(CSharpVerbatimNameNormalizer.Normalize(normalizedQuery), BuildQueryTokens(query, normalizeCSharpVerbatimNames: true), NormalizeCSharpVerbatimNames: true)); + } + public static IReadOnlyList Format(string content, string query, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality) { - var excerpt = BuildExcerpt(content, query, absoluteStartLine: 1, maxLines, caseSensitive, maxLineWidth, lang, focusMode); + return Format(content, PrepareQueryContext(query), maxLines, caseSensitive, maxLineWidth, lang, focusMode); + } + + public static IReadOnlyList Format(string content, SearchSnippetQueryContext queryContext, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality) + { + ArgumentNullException.ThrowIfNull(queryContext); + + var excerpt = BuildExcerpt(content, queryContext, absoluteStartLine: 1, maxLines, caseSensitive, maxLineWidth, lang, focusMode); if (excerpt.Lines.Count == 0) return []; @@ -32,10 +48,17 @@ public static IReadOnlyList Format(string content, string query, int max public static CompactSearchResult ToCompactResult(SearchResult result, string query, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality, bool exposeLiteralHighlights = false) { - var excerpt = BuildExcerpt(result.Content, query, result.StartLine, maxLines, caseSensitive, maxLineWidth, lang ?? result.Lang, focusMode, exposeLiteralHighlights); + return ToCompactResult(result, PrepareQueryContext(query), maxLines, caseSensitive, maxLineWidth, lang, focusMode, exposeLiteralHighlights); + } + + public static CompactSearchResult ToCompactResult(SearchResult result, SearchSnippetQueryContext queryContext, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality, bool exposeLiteralHighlights = false) + { + ArgumentNullException.ThrowIfNull(queryContext); + + var excerpt = BuildExcerpt(result.Content, queryContext, result.StartLine, maxLines, caseSensitive, maxLineWidth, lang ?? result.Lang, focusMode, exposeLiteralHighlights); return new CompactSearchResult { - Query = query, + Query = queryContext.Query, Path = result.Path, Lang = result.Lang, Visibility = result.Visibility, @@ -63,12 +86,27 @@ public static CompactSearchResult ToCompactResult(SearchResult result, string qu public static IEnumerable ToCompactResults(IEnumerable results, string query, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality, bool exposeLiteralHighlights = false) { + var queryContext = PrepareQueryContext(query); + return ToCompactResults(results, queryContext, maxLines, caseSensitive, maxLineWidth, lang, focusMode, exposeLiteralHighlights); + } + + public static IEnumerable ToCompactResults(IEnumerable results, SearchSnippetQueryContext queryContext, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality, bool exposeLiteralHighlights = false) + { + ArgumentNullException.ThrowIfNull(queryContext); + foreach (var result in results) - yield return ToCompactResult(result, query, maxLines, caseSensitive, maxLineWidth, lang ?? result.Lang, focusMode, exposeLiteralHighlights); + yield return ToCompactResult(result, queryContext, maxLines, caseSensitive, maxLineWidth, lang ?? result.Lang, focusMode, exposeLiteralHighlights); } public static SearchSnippetExcerpt BuildExcerpt(string content, string query, int absoluteStartLine, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality, bool exposeLiteralHighlights = false) { + return BuildExcerpt(content, PrepareQueryContext(query), absoluteStartLine, maxLines, caseSensitive, maxLineWidth, lang, focusMode, exposeLiteralHighlights); + } + + public static SearchSnippetExcerpt BuildExcerpt(string content, SearchSnippetQueryContext queryContext, int absoluteStartLine, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality, bool exposeLiteralHighlights = false) + { + ArgumentNullException.ThrowIfNull(queryContext); + maxLines = ClampSnippetLines(maxLines); maxLineWidth = LineWidthFormatter.ClampMaxLineWidth(maxLineWidth); @@ -82,19 +120,10 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, string query, in }; } - var normalizedQuery = query.Trim(); - var normalizeCSharpVerbatimNames = string.Equals(lang, "csharp", StringComparison.OrdinalIgnoreCase); - if (normalizeCSharpVerbatimNames) - normalizedQuery = CSharpVerbatimNameNormalizer.Normalize(normalizedQuery); - - var tokens = query - .Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries) - .Select(NormalizeToken) - .Where(t => t.Length > 0) - .Where(t => t is not "AND" and not "OR" and not "NOT" and not "NEAR") - .Select(token => normalizeCSharpVerbatimNames ? CSharpVerbatimNameNormalizer.Normalize(token) : token) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); + var queryForLanguage = queryContext.ForLanguage(lang); + var normalizedQuery = queryForLanguage.NormalizedQuery; + var tokens = queryForLanguage.Tokens; + var normalizeCSharpVerbatimNames = queryForLanguage.NormalizeCSharpVerbatimNames; string[]? normalizedLines = null; int[][]? rawIndexMaps = null; @@ -393,6 +422,16 @@ private readonly record struct MatchCandidate(int Index, int Length, int Score, public static int ClampSnippetLines(int maxLines) => Math.Clamp(maxLines, 1, MaxSnippetLines); + private static string[] BuildQueryTokens(string query, bool normalizeCSharpVerbatimNames) => + query + .Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries) + .Select(NormalizeToken) + .Where(t => t.Length > 0) + .Where(t => t is not "AND" and not "OR" and not "NOT" and not "NEAR") + .Select(token => normalizeCSharpVerbatimNames ? CSharpVerbatimNameNormalizer.Normalize(token) : token) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + private static List FindMatchingLineIndexes(string[] lines, string query, string[] tokens, bool caseSensitive = false) { var comparison = caseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; @@ -503,6 +542,26 @@ private static string NormalizeToken(string token) } +public sealed class SearchSnippetQueryContext +{ + internal SearchSnippetQueryContext(string query, SearchSnippetPreparedQuery defaultQuery, SearchSnippetPreparedQuery csharpQuery) + { + Query = query; + DefaultQuery = defaultQuery; + CSharpQuery = csharpQuery; + } + + public string Query { get; } + + internal SearchSnippetPreparedQuery DefaultQuery { get; } + internal SearchSnippetPreparedQuery CSharpQuery { get; } + + internal SearchSnippetPreparedQuery ForLanguage(string? lang) => + string.Equals(lang, "csharp", StringComparison.OrdinalIgnoreCase) ? CSharpQuery : DefaultQuery; +} + +internal readonly record struct SearchSnippetPreparedQuery(string NormalizedQuery, string[] Tokens, bool NormalizeCSharpVerbatimNames); + public sealed class CompactSearchResult { [JsonPropertyName("api_version")] diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 5e8a185d73..dca919276a 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -1215,6 +1215,7 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) return CreateToolResult(id, "No results found.", payload); } + var queryContext = SearchSnippetFormatter.PrepareQueryContext(query); var structured = new JsonObject { ["query"] = query, @@ -1224,7 +1225,7 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) ["maxLineWidth"] = maxLineWidth, ["path"] = PathEcho(pathPatterns), ["excludeTests"] = excludeTests, - ["results"] = ToJsonArray(SearchSnippetFormatter.ToCompactResults(results, query, snippetLines, exact, maxLineWidth, exposeLiteralHighlights: exact)) + ["results"] = ToJsonArray(SearchSnippetFormatter.ToCompactResults(results, queryContext, snippetLines, exact, maxLineWidth, exposeLiteralHighlights: exact)) }; AddSearchStabilityMetadata(structured, reader, cursor, results); AddResultEnvelope(structured, results.Count, truncated ? null : results.Count, truncated); diff --git a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs index 8e122e5f19..874ff2445f 100644 --- a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +++ b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs @@ -202,6 +202,55 @@ public void BuildExcerpt_NormalizesCSharpVerbatimQualifiedNamesInNonExactSearch( Assert.Contains("using @Foo.@Bar;", excerpt.Lines); } + [Fact] + public void BuildExcerpt_PreparedQueryContextMatchesStringQuery() + { + const string content = "alpha\nbeta gamma\nalpha beta gamma\n"; + var context = SearchSnippetFormatter.PrepareQueryContext("alpha beta gamma"); + + var direct = SearchSnippetFormatter.BuildExcerpt(content, "alpha beta gamma", absoluteStartLine: 10, maxLines: 2); + var prepared = SearchSnippetFormatter.BuildExcerpt(content, context, absoluteStartLine: 10, maxLines: 2); + + Assert.Equal(direct.StartLine, prepared.StartLine); + Assert.Equal(direct.EndLine, prepared.EndLine); + Assert.Equal(direct.MatchLines, prepared.MatchLines); + Assert.Equal(direct.Lines, prepared.Lines); + Assert.Equal(direct.Highlights.Single().Terms, prepared.Highlights.Single().Terms); + } + + [Fact] + public void ToCompactResults_PreparedQueryContextRemainsLanguageAwareAcrossResults() + { + var context = SearchSnippetFormatter.PrepareQueryContext("Foo.Bar"); + var results = new[] + { + new SearchResult + { + Path = "src/a.cs", + Lang = "csharp", + StartLine = 1, + EndLine = 1, + Content = "using @Foo.@Bar;", + }, + new SearchResult + { + Path = "docs/a.md", + Lang = "markdown", + StartLine = 1, + EndLine = 1, + Content = "Foo.Bar", + }, + }; + + var compact = SearchSnippetFormatter.ToCompactResults(results, context, maxLines: 1).ToArray(); + + Assert.Equal(["src/a.cs", "docs/a.md"], compact.Select(result => result.Path).ToArray()); + var csharpOccurrence = Assert.Single(compact[0].Highlights.Single().TermOccurrences); + Assert.Equal("Foo.@Bar", csharpOccurrence.Term); + var markdownOccurrence = Assert.Single(compact[1].Highlights.Single().TermOccurrences); + Assert.Equal("Foo.Bar", markdownOccurrence.Term); + } + [Fact] public void TryReconstructRawSpan_ReturnsFalse_WhenNormalizedSpanCannotAnchorEnd() {