From 5d45f32867c78fae4b621184460bd4249d74bbfe Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 6 Jun 2026 01:00:05 +0900 Subject: [PATCH 1/6] Avoid full snippet line splitting (#3087) --- changelog.d/unreleased/3087.fixed.md | 16 ++ src/CodeIndex/Cli/SearchSnippetFormatter.cs | 140 +++++++++++++----- .../SearchSnippetFormatterTests.cs | 16 ++ 3 files changed, 132 insertions(+), 40 deletions(-) create mode 100644 changelog.d/unreleased/3087.fixed.md diff --git a/changelog.d/unreleased/3087.fixed.md b/changelog.d/unreleased/3087.fixed.md new file mode 100644 index 0000000000..a86cdd3776 --- /dev/null +++ b/changelog.d/unreleased/3087.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 3087 +affected: + - src/CodeIndex/Cli/SearchSnippetFormatter.cs + - tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +--- + +## English + +- **Search snippets avoid full-content line splitting for bounded excerpts (#3087)** — snippet formatting now scans content lines without materializing the entire split array before building the requested snippet window. + +## 日本語 + +- **検索スニペットが範囲限定excerptのために本文全体を行分割しないようになりました (#3087)** — スニペット整形は、要求されたsnippet windowを構築する前に本文全体の分割配列を作らず、行をスキャンするようになりました。 diff --git a/src/CodeIndex/Cli/SearchSnippetFormatter.cs b/src/CodeIndex/Cli/SearchSnippetFormatter.cs index b6a757a7ba..128796ecac 100644 --- a/src/CodeIndex/Cli/SearchSnippetFormatter.cs +++ b/src/CodeIndex/Cli/SearchSnippetFormatter.cs @@ -110,33 +110,23 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, SearchSnippetQue maxLines = ClampSnippetLines(maxLines); maxLineWidth = LineWidthFormatter.ClampMaxLineWidth(maxLineWidth); - var lines = content.Replace("\r\n", "\n").Split('\n'); - if (lines.Length == 0) - { - return new SearchSnippetExcerpt - { - StartLine = absoluteStartLine, - EndLine = absoluteStartLine, - }; - } - var queryForLanguage = queryContext.ForLanguage(lang); var normalizedQuery = queryForLanguage.NormalizedQuery; var tokens = queryForLanguage.Tokens; var normalizeCSharpVerbatimNames = queryForLanguage.NormalizeCSharpVerbatimNames; - string[]? normalizedLines = null; - int[][]? rawIndexMaps = null; - if (normalizeCSharpVerbatimNames) + var matchScan = FindMatchingLineIndexes(content, normalizedQuery, tokens, caseSensitive, normalizeCSharpVerbatimNames); + var lineCount = matchScan.LineCount; + if (lineCount == 0) { - normalizedLines = new string[lines.Length]; - rawIndexMaps = new int[lines.Length][]; - for (int i = 0; i < lines.Length; i++) - normalizedLines[i] = CSharpVerbatimNameNormalizer.Normalize(lines[i], out rawIndexMaps[i]); + return new SearchSnippetExcerpt + { + StartLine = absoluteStartLine, + EndLine = absoluteStartLine, + }; } - var matchLinesSource = normalizedLines ?? lines; - var matchIndexes = FindMatchingLineIndexes(matchLinesSource, normalizedQuery, tokens, caseSensitive); + var matchIndexes = matchScan.MatchIndexes; var focusStart = matchIndexes.Count > 0 ? matchIndexes[0] : 0; var focusEnd = focusStart; var includedMatchLineCount = Math.Min(1, matchIndexes.Count); @@ -156,7 +146,7 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, SearchSnippetQue var after = remaining - before; var start = Math.Max(0, focusStart - before); - var end = Math.Min(lines.Length - 1, focusEnd + after); + var end = Math.Min(lineCount - 1, focusEnd + after); while ((end - start) + 1 < maxLines) { if (start > 0) @@ -165,7 +155,7 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, SearchSnippetQue continue; } - if (end < lines.Length - 1) + if (end < lineCount - 1) { end++; continue; @@ -180,18 +170,21 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, SearchSnippetQue var clampedLines = new List((end - start) + 1); var truncatedCharCounts = new List(); var truncatedLineCount = 0; + var snippetLines = ReadSnippetLines(content, start, end, normalizeCSharpVerbatimNames); - for (int i = start; i <= end; i++) + foreach (var snippetLine in snippetLines) { - var originalLine = lines[i]; + var i = snippetLine.Index; + var originalLine = snippetLine.Text; + var isMatch = matchSet.Contains(i); ClampedTextResult clamped; - if (normalizeCSharpVerbatimNames && matchSet.Contains(i) && normalizedLines != null && rawIndexMaps != null) + if (normalizeCSharpVerbatimNames && isMatch && snippetLine.NormalizedText != null && snippetLine.RawIndexMap != null) { - clamped = ClampNormalizedSnippetLine(originalLine, normalizedLines[i], rawIndexMaps[i], maxLineWidth, normalizedQuery, tokens, caseSensitive, focusMode); + clamped = ClampNormalizedSnippetLine(originalLine, snippetLine.NormalizedText, snippetLine.RawIndexMap, maxLineWidth, normalizedQuery, tokens, caseSensitive, focusMode); } else { - clamped = ClampSnippetLine(originalLine, maxLineWidth, matchSet.Contains(i) ? normalizedQuery : null, tokens, caseSensitive, focusMode); + clamped = ClampSnippetLine(originalLine, maxLineWidth, isMatch ? normalizedQuery : null, tokens, caseSensitive, focusMode); } clampedLines.Add(clamped.Text); if (clamped.Truncated) @@ -200,18 +193,18 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, SearchSnippetQue truncatedCharCounts.Add(clamped.TruncatedCharCount); } - if (!matchSet.Contains(i)) + if (!isMatch) continue; var absoluteLine = absoluteStartLine + i; matchLines.Add(absoluteLine); - var matchLineForTerms = normalizeCSharpVerbatimNames && normalizedLines != null ? normalizedLines[i] : originalLine; - var termOccurrences = normalizeCSharpVerbatimNames && normalizedLines != null && rawIndexMaps != null - ? GetMatchedTermOccurrences(normalizedLines[i], absoluteLine, normalizedQuery, tokens, caseSensitive, originalLine, rawIndexMaps[i]) + var matchLineForTerms = normalizeCSharpVerbatimNames && snippetLine.NormalizedText != null ? snippetLine.NormalizedText : originalLine; + var termOccurrences = normalizeCSharpVerbatimNames && snippetLine.NormalizedText != null && snippetLine.RawIndexMap != null + ? GetMatchedTermOccurrences(snippetLine.NormalizedText, absoluteLine, normalizedQuery, tokens, caseSensitive, originalLine, snippetLine.RawIndexMap) : GetMatchedTermOccurrences(originalLine, absoluteLine, normalizedQuery, tokens, caseSensitive); var literalTermOccurrences = exposeLiteralHighlights - ? normalizeCSharpVerbatimNames && normalizedLines != null && rawIndexMaps != null - ? GetMatchedTermOccurrences(normalizedLines[i], absoluteLine, normalizedQuery, [], caseSensitive, originalLine, rawIndexMaps[i]) + ? normalizeCSharpVerbatimNames && snippetLine.NormalizedText != null && snippetLine.RawIndexMap != null + ? GetMatchedTermOccurrences(snippetLine.NormalizedText, absoluteLine, normalizedQuery, [], caseSensitive, originalLine, snippetLine.RawIndexMap) : GetMatchedTermOccurrences(originalLine, absoluteLine, normalizedQuery, [], caseSensitive) : null; highlights.Add(new SearchHighlight @@ -238,7 +231,7 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, SearchSnippetQue ContextBefore = focusStart - start, ContextAfter = end - focusEnd, TruncatedBefore = start > 0, - TruncatedAfter = end < lines.Length - 1, + TruncatedAfter = end < lineCount - 1, TruncatedLineCount = truncatedLineCount, DroppedMatchLineCount = droppedMatchLineCount, TruncationContext = new SearchTruncationContext @@ -432,32 +425,99 @@ private static string[] BuildQueryTokens(string query, bool normalizeCSharpVerba .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); - private static List FindMatchingLineIndexes(string[] lines, string query, string[] tokens, bool caseSensitive = false) + private static SearchSnippetLineMatchScan FindMatchingLineIndexes(string content, string query, string[] tokens, bool caseSensitive, bool normalizeCSharpVerbatimNames) { var comparison = caseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; var matches = new List(); + var lineCount = 0; if (!string.IsNullOrWhiteSpace(query)) { - for (int i = 0; i < lines.Length; i++) + foreach (var (i, rawLine) in EnumerateContentLines(content)) { - if (lines[i].Contains(query, comparison)) + lineCount++; + var line = normalizeCSharpVerbatimNames ? CSharpVerbatimNameNormalizer.Normalize(rawLine) : rawLine; + if (line.Contains(query, comparison)) matches.Add(i); } } + else + { + lineCount = CountContentLines(content); + } if (matches.Count > 0 || tokens.Length == 0) - return matches; + return new SearchSnippetLineMatchScan(matches, lineCount); - for (int i = 0; i < lines.Length; i++) + matches.Clear(); + lineCount = 0; + foreach (var (i, rawLine) in EnumerateContentLines(content)) { - if (tokens.Any(token => lines[i].Contains(token, comparison))) + lineCount++; + var line = normalizeCSharpVerbatimNames ? CSharpVerbatimNameNormalizer.Normalize(rawLine) : rawLine; + if (tokens.Any(token => line.Contains(token, comparison))) matches.Add(i); } - return matches; + return new SearchSnippetLineMatchScan(matches, lineCount); + } + + private static List ReadSnippetLines(string content, int start, int end, bool normalizeCSharpVerbatimNames) + { + var lines = new List((end - start) + 1); + foreach (var (index, rawLine) in EnumerateContentLines(content)) + { + if (index < start) + continue; + if (index > end) + break; + + if (normalizeCSharpVerbatimNames) + { + var normalized = CSharpVerbatimNameNormalizer.Normalize(rawLine, out var rawIndexMap); + lines.Add(new SearchSnippetLine(index, rawLine, normalized, rawIndexMap)); + } + else + { + lines.Add(new SearchSnippetLine(index, rawLine, null, null)); + } + } + + return lines; + } + + private static int CountContentLines(string content) + { + var count = 0; + foreach (var _ in EnumerateContentLines(content)) + count++; + return count; } + private static IEnumerable<(int Index, string Text)> EnumerateContentLines(string content) + { + var lineStart = 0; + var lineIndex = 0; + for (var i = 0; i < content.Length; i++) + { + if (content[i] != '\n') + continue; + + var lineEnd = i; + if (lineEnd > lineStart && content[lineEnd - 1] == '\r') + lineEnd--; + yield return (lineIndex, content[lineStart..lineEnd]); + lineIndex++; + lineStart = i + 1; + } + + yield return (lineIndex, content[lineStart..]); + } + + private sealed record SearchSnippetLineMatchScan(List MatchIndexes, int LineCount); + + private sealed record SearchSnippetLine(int Index, string Text, string? NormalizedText, int[]? RawIndexMap); + private static List GetMatchedTermOccurrences(string line, int absoluteLine, string query, string[] tokens, bool caseSensitive = false, string? rawLine = null, int[]? rawIndexMap = null) { var comparison = caseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; diff --git a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs index 874ff2445f..3e92bab2ff 100644 --- a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +++ b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs @@ -218,6 +218,22 @@ public void BuildExcerpt_PreparedQueryContextMatchesStringQuery() Assert.Equal(direct.Highlights.Single().Terms, prepared.Highlights.Single().Terms); } + [Fact] + public void BuildExcerpt_LargeContentMaterializesOnlyRequestedWindow_Issue3087() + { + var lines = Enumerable.Range(1, 50_000) + .Select(i => i == 25_000 ? "call Target()" : $"line {i}"); + var content = string.Join('\n', lines); + + var excerpt = SearchSnippetFormatter.BuildExcerpt(content, "Target", absoluteStartLine: 1, maxLines: 3); + + Assert.Equal(24_999, excerpt.StartLine); + Assert.Equal(25_001, excerpt.EndLine); + Assert.Equal(3, excerpt.Lines.Count); + Assert.Equal([25_000], excerpt.MatchLines); + Assert.Contains("call Target()", excerpt.Lines); + } + [Fact] public void ToCompactResults_PreparedQueryContextRemainsLanguageAwareAcrossResults() { From 4e11aae052364cda86a12fc7fa33538c51568ace Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 6 Jun 2026 01:02:18 +0900 Subject: [PATCH 2/6] Cap tracked snippet matches (#3088) --- changelog.d/unreleased/3088.fixed.md | 16 +++++++++ src/CodeIndex/Cli/SearchSnippetFormatter.cs | 30 +++++++++++----- .../SearchSnippetFormatterTests.cs | 36 +++++++++++++++++++ 3 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 changelog.d/unreleased/3088.fixed.md diff --git a/changelog.d/unreleased/3088.fixed.md b/changelog.d/unreleased/3088.fixed.md new file mode 100644 index 0000000000..5f5c272e96 --- /dev/null +++ b/changelog.d/unreleased/3088.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 3088 +affected: + - src/CodeIndex/Cli/SearchSnippetFormatter.cs + - tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +--- + +## English + +- **Search snippets cap tracked match indexes for repetitive content (#3088)** — snippet matching now keeps only match-line indexes needed for the selected snippet window while still reporting the total dropped match-line count. + +## 日本語 + +- **反復の多い本文で検索スニペットが保持する一致indexを制限しました (#3088)** — スニペット照合は選択済みsnippet windowに必要な一致行indexだけを保持しつつ、dropされた一致行数の合計は引き続き報告します。 diff --git a/src/CodeIndex/Cli/SearchSnippetFormatter.cs b/src/CodeIndex/Cli/SearchSnippetFormatter.cs index 128796ecac..d36d1189b1 100644 --- a/src/CodeIndex/Cli/SearchSnippetFormatter.cs +++ b/src/CodeIndex/Cli/SearchSnippetFormatter.cs @@ -115,7 +115,7 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, SearchSnippetQue var tokens = queryForLanguage.Tokens; var normalizeCSharpVerbatimNames = queryForLanguage.NormalizeCSharpVerbatimNames; - var matchScan = FindMatchingLineIndexes(content, normalizedQuery, tokens, caseSensitive, normalizeCSharpVerbatimNames); + var matchScan = FindMatchingLineIndexes(content, normalizedQuery, tokens, caseSensitive, normalizeCSharpVerbatimNames, maxLines); var lineCount = matchScan.LineCount; if (lineCount == 0) { @@ -129,7 +129,7 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, SearchSnippetQue var matchIndexes = matchScan.MatchIndexes; var focusStart = matchIndexes.Count > 0 ? matchIndexes[0] : 0; var focusEnd = focusStart; - var includedMatchLineCount = Math.Min(1, matchIndexes.Count); + var includedMatchLineCount = matchIndexes.Count > 0 ? 1 : 0; foreach (var matchIndex in matchIndexes.Skip(1)) { if ((matchIndex - focusStart) + 1 > maxLines) @@ -138,7 +138,7 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, SearchSnippetQue focusEnd = matchIndex; includedMatchLineCount++; } - var droppedMatchLineCount = Math.Max(0, matchIndexes.Count - includedMatchLineCount); + var droppedMatchLineCount = Math.Max(0, matchScan.TotalMatchCount - includedMatchLineCount); var focusLength = Math.Max(1, (focusEnd - focusStart) + 1); var remaining = Math.Max(0, maxLines - focusLength); @@ -425,11 +425,13 @@ private static string[] BuildQueryTokens(string query, bool normalizeCSharpVerba .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); - private static SearchSnippetLineMatchScan FindMatchingLineIndexes(string content, string query, string[] tokens, bool caseSensitive, bool normalizeCSharpVerbatimNames) + private static SearchSnippetLineMatchScan FindMatchingLineIndexes(string content, string query, string[] tokens, bool caseSensitive, bool normalizeCSharpVerbatimNames, int maxTrackedWindowLines) { var comparison = caseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; var matches = new List(); var lineCount = 0; + var totalMatchCount = 0; + int? focusStart = null; if (!string.IsNullOrWhiteSpace(query)) { @@ -438,7 +440,7 @@ private static SearchSnippetLineMatchScan FindMatchingLineIndexes(string content lineCount++; var line = normalizeCSharpVerbatimNames ? CSharpVerbatimNameNormalizer.Normalize(rawLine) : rawLine; if (line.Contains(query, comparison)) - matches.Add(i); + AddTrackedMatchIndex(matches, i, maxTrackedWindowLines, ref focusStart, ref totalMatchCount); } } else @@ -447,19 +449,29 @@ private static SearchSnippetLineMatchScan FindMatchingLineIndexes(string content } if (matches.Count > 0 || tokens.Length == 0) - return new SearchSnippetLineMatchScan(matches, lineCount); + return new SearchSnippetLineMatchScan(matches, lineCount, totalMatchCount); matches.Clear(); lineCount = 0; + totalMatchCount = 0; + focusStart = null; foreach (var (i, rawLine) in EnumerateContentLines(content)) { lineCount++; var line = normalizeCSharpVerbatimNames ? CSharpVerbatimNameNormalizer.Normalize(rawLine) : rawLine; if (tokens.Any(token => line.Contains(token, comparison))) - matches.Add(i); + AddTrackedMatchIndex(matches, i, maxTrackedWindowLines, ref focusStart, ref totalMatchCount); } - return new SearchSnippetLineMatchScan(matches, lineCount); + return new SearchSnippetLineMatchScan(matches, lineCount, totalMatchCount); + } + + private static void AddTrackedMatchIndex(List matches, int lineIndex, int maxTrackedWindowLines, ref int? focusStart, ref int totalMatchCount) + { + totalMatchCount++; + focusStart ??= lineIndex; + if ((lineIndex - focusStart.Value) + 1 <= maxTrackedWindowLines) + matches.Add(lineIndex); } private static List ReadSnippetLines(string content, int start, int end, bool normalizeCSharpVerbatimNames) @@ -514,7 +526,7 @@ private static int CountContentLines(string content) yield return (lineIndex, content[lineStart..]); } - private sealed record SearchSnippetLineMatchScan(List MatchIndexes, int LineCount); + private sealed record SearchSnippetLineMatchScan(List MatchIndexes, int LineCount, int TotalMatchCount); private sealed record SearchSnippetLine(int Index, string Text, string? NormalizedText, int[]? RawIndexMap); diff --git a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs index 3e92bab2ff..eb9268e572 100644 --- a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +++ b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs @@ -1,3 +1,4 @@ +using System.Reflection; using CodeIndex.Cli; using CodeIndex.Database; @@ -234,6 +235,21 @@ public void BuildExcerpt_LargeContentMaterializesOnlyRequestedWindow_Issue3087() Assert.Contains("call Target()", excerpt.Lines); } + [Fact] + public void BuildExcerpt_RepetitiveMatchesTrackOnlySnippetWindow_Issue3088() + { + var content = string.Join('\n', Enumerable.Repeat("Target", 50_000)); + + var excerpt = SearchSnippetFormatter.BuildExcerpt(content, "Target", absoluteStartLine: 1, maxLines: 3); + + Assert.Equal([1, 2, 3], excerpt.MatchLines); + Assert.Equal(49_997, excerpt.DroppedMatchLineCount); + Assert.Equal(3, excerpt.Highlights.Count); + var scan = InvokeFindMatchingLineIndexes(content, "Target", [], caseSensitive: false, normalizeCSharpVerbatimNames: false, maxTrackedWindowLines: 3); + Assert.Equal(3, scan.MatchIndexes.Count); + Assert.Equal(50_000, scan.TotalMatchCount); + } + [Fact] public void ToCompactResults_PreparedQueryContextRemainsLanguageAwareAcrossResults() { @@ -468,4 +484,24 @@ public void Format_DoesNotClamp_WhenMaxLineWidthIsZero() Assert.Equal(huge, joined); Assert.DoesNotContain("...(+", joined); } + + private static (IReadOnlyCollection MatchIndexes, int TotalMatchCount) InvokeFindMatchingLineIndexes( + string content, + string query, + string[] tokens, + bool caseSensitive, + bool normalizeCSharpVerbatimNames, + int maxTrackedWindowLines) + { + var method = typeof(SearchSnippetFormatter).GetMethod("FindMatchingLineIndexes", BindingFlags.Static | BindingFlags.NonPublic); + Assert.NotNull(method); + + var scan = method.Invoke(null, [content, query, tokens, caseSensitive, normalizeCSharpVerbatimNames, maxTrackedWindowLines]); + Assert.NotNull(scan); + + var type = scan!.GetType(); + var matchIndexes = Assert.IsAssignableFrom>(type.GetProperty("MatchIndexes")!.GetValue(scan)); + var totalMatchCount = Assert.IsType(type.GetProperty("TotalMatchCount")!.GetValue(scan)); + return (matchIndexes, totalMatchCount); + } } From f3520c5c9cfee42366d4509ece548050fd489ef2 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 6 Jun 2026 01:05:15 +0900 Subject: [PATCH 3/6] Precompute guarded search terms (#3083) --- changelog.d/unreleased/3083.fixed.md | 16 ++++ src/CodeIndex/Database/DbSearchReader.cs | 82 +++++++++++++------ .../DbSearchReaderIssueTests.cs | 26 ++++++ 3 files changed, 101 insertions(+), 23 deletions(-) create mode 100644 changelog.d/unreleased/3083.fixed.md diff --git a/changelog.d/unreleased/3083.fixed.md b/changelog.d/unreleased/3083.fixed.md new file mode 100644 index 0000000000..06e7efda3f --- /dev/null +++ b/changelog.d/unreleased/3083.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 3083 +affected: + - src/CodeIndex/Database/DbSearchReader.cs + - tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs +--- + +## English + +- **Guarded search precomputes primary match terms and scans candidate lines lazily (#3083)** — guard evaluation no longer rebuilds stable primary query terms or splits every candidate chunk into a full line array before choosing focus lines. + +## 日本語 + +- **guard付き検索がprimary match語を事前計算し、候補行を遅延スキャンするようになりました (#3083)** — guard評価は安定したprimary query語を候補ごとに再構築せず、focus line選択前に候補chunk全体を行配列へ分割しなくなりました。 diff --git a/src/CodeIndex/Database/DbSearchReader.cs b/src/CodeIndex/Database/DbSearchReader.cs index d8f08aad20..130a23a00d 100644 --- a/src/CodeIndex/Database/DbSearchReader.cs +++ b/src/CodeIndex/Database/DbSearchReader.cs @@ -219,7 +219,7 @@ FROM fts_chunks raw.RemoveRange(guardedCandidateLimit, raw.Count - guardedCandidateLimit); if (hasGuardFilters) - raw = FilterBySearchGuards(raw, query, normalizedQuery, rawQuery, exact, lang, guardFilters!, guardWindow); + raw = FilterBySearchGuards(raw, SearchPrimaryMatchContext.Create(query, normalizedQuery, rawQuery, exact, lang), guardFilters!, guardWindow); var results = deduplicate ? DeduplicateOverlappingResults(raw) : raw; if (guardCandidateLimitReached && results.Count < GetGuardedSearchRequestedPageEnd(limit, cursor)) @@ -484,11 +484,7 @@ FROM fts_chunks private List FilterBySearchGuards( List results, - string query, - string normalizedQuery, - bool rawQuery, - bool exact, - string? lang, + SearchPrimaryMatchContext primaryMatchContext, IReadOnlyList guardFilters, int guardWindow) { @@ -496,13 +492,13 @@ private List FilterBySearchGuards( var filtered = new List(results.Count); foreach (var result in results) { - foreach (var (focusLine, focusText) in FindPrimarySearchMatchLines(result, query, normalizedQuery, rawQuery, exact, lang)) + foreach (var (focusLine, focusText) in FindPrimarySearchMatchLines(result, primaryMatchContext)) { var guardEvidence = new List(); var keep = true; foreach (var filter in guardFilters) { - var match = FindGuardEvidence(result.Path, focusLine, filter, guardWindow, lang ?? result.Lang); + var match = FindGuardEvidence(result.Path, focusLine, filter, guardWindow, primaryMatchContext.GetEffectiveLang(result)); var matched = match != null; if (filter.Role == SearchGuardRole.Require && !matched) { @@ -540,30 +536,50 @@ private List FilterBySearchGuards( return filtered; } - private static List<(int LineNumber, string Text)> FindPrimarySearchMatchLines(SearchResult result, string query, string normalizedQuery, bool rawQuery, bool exact, string? lang) + private static List<(int LineNumber, string Text)> FindPrimarySearchMatchLines(SearchResult result, SearchPrimaryMatchContext context) { - var terms = BuildPrimarySearchMatchTerms(query, normalizedQuery, rawQuery, exact); - var lines = SplitContentLines(result.Content); - if (terms.Length == 0) - return [(result.StartLine, lines.FirstOrDefault() ?? string.Empty)]; - - var normalizeCSharp = string.Equals(lang ?? result.Lang, "csharp", StringComparison.OrdinalIgnoreCase); - var comparison = exact ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; - var requireAllTermsOnLine = !rawQuery && !exact && terms.Length > 1; + if (context.Terms.Length == 0) + { + foreach (var (lineIndex, text) in EnumerateContentLines(result.Content)) + return [(result.StartLine + lineIndex, text)]; + + return [(result.StartLine, string.Empty)]; + } + + var normalizeCSharp = context.ShouldNormalizeCSharp(result); var matches = new List<(int LineNumber, string Text)>(); - for (var i = 0; i < lines.Length; i++) + foreach (var (lineIndex, text) in EnumerateContentLines(result.Content)) { - var line = normalizeCSharp ? CSharpVerbatimNameNormalizer.Normalize(lines[i]) : lines[i]; - var lineMatches = requireAllTermsOnLine - ? terms.All(term => line.Contains(term, comparison)) - : terms.Any(term => line.Contains(term, comparison)); + var line = normalizeCSharp ? CSharpVerbatimNameNormalizer.Normalize(text) : text; + var lineMatches = context.RequireAllTermsOnLine + ? context.Terms.All(term => line.Contains(term, context.Comparison)) + : context.Terms.Any(term => line.Contains(term, context.Comparison)); if (lineMatches) - matches.Add((result.StartLine + i, lines[i])); + matches.Add((result.StartLine + lineIndex, text)); } return matches; } + private sealed record SearchPrimaryMatchContext( + string[] Terms, + bool RawQuery, + bool Exact, + string? QueryLang) + { + public StringComparison Comparison => Exact ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; + + public bool RequireAllTermsOnLine => !RawQuery && !Exact && Terms.Length > 1; + + public static SearchPrimaryMatchContext Create(string query, string normalizedQuery, bool rawQuery, bool exact, string? queryLang) + => new(BuildPrimarySearchMatchTerms(query, normalizedQuery, rawQuery, exact), rawQuery, exact, queryLang); + + public string? GetEffectiveLang(SearchResult result) => QueryLang ?? result.Lang; + + public bool ShouldNormalizeCSharp(SearchResult result) + => string.Equals(GetEffectiveLang(result), "csharp", StringComparison.OrdinalIgnoreCase); + } + private static string[] BuildPrimarySearchMatchTerms(string query, string normalizedQuery, bool rawQuery, bool exact) { IEnumerable rawTerms = !exact && !rawQuery @@ -677,6 +693,26 @@ private static string NormalizeGuardSearchTerm(string value) private static string[] SplitContentLines(string content) => content.Replace("\r\n", "\n").Split('\n'); + private static IEnumerable<(int Index, string Text)> EnumerateContentLines(string content) + { + var lineStart = 0; + var lineIndex = 0; + for (var i = 0; i < content.Length; i++) + { + if (content[i] != '\n') + continue; + + var lineEnd = i; + if (lineEnd > lineStart && content[lineEnd - 1] == '\r') + lineEnd--; + yield return (lineIndex, content[lineStart..lineEnd]); + lineIndex++; + lineStart = i + 1; + } + + yield return (lineIndex, content[lineStart..]); + } + private static string FormatSearchGuardRole(SearchGuardRole role) => role == SearchGuardRole.Require ? "require" : "reject"; diff --git a/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs index 36f35c12e0..bcfc707306 100644 --- a/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs +++ b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs @@ -91,6 +91,32 @@ public void Search_GuardFiltersDoNotRejectWhenCandidateCountExactlyMatchesBudget Assert.Empty(results); } + [Fact] + public void Search_GuardFiltersFocusLargeCandidateWithPreparedPrimaryTerms_Issue3083() + { + var lines = Enumerable.Range(1, 40_000) + .Select(i => i switch + { + 24_999 => "public void Setup() { GuardMarker(); }", + 25_000 => "public void Run() { Primary Needle(); }", + _ => $"// filler {i}", + }); + InsertIndexedFile("src/guard-primary-large.cs", "csharp", string.Join('\n', lines)); + + var results = _reader.Search( + "Primary Needle", + pathPatterns: ["src/guard-primary-large.cs"], + limit: 1, + guardFilters: [new SearchGuardFilter(SearchGuardRole.Require, SearchGuardDirection.Before, "GuardMarker")], + guardWindow: 1); + + var result = Assert.Single(results); + Assert.Equal(25_000, result.StartLine); + Assert.Equal("public void Run() { Primary Needle(); }", result.Content); + var evidence = Assert.Single(result.GuardEvidence!); + Assert.Equal(24_999, evidence.Line); + } + private void InsertIndexedFile(string path, string lang, string content, DateTime? modified = null) { var normalized = content.Replace("\r\n", "\n"); From a23427885a633bcca2c49e7a145d82189cdc94a8 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 6 Jun 2026 01:08:22 +0900 Subject: [PATCH 4/6] Bound guard line-window reads (#3085) --- changelog.d/unreleased/3085.fixed.md | 16 +++++++++++ src/CodeIndex/Database/DbSearchReader.cs | 25 ++++++++++++----- .../DbSearchReaderIssueTests.cs | 27 +++++++++++++++++++ 3 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 changelog.d/unreleased/3085.fixed.md diff --git a/changelog.d/unreleased/3085.fixed.md b/changelog.d/unreleased/3085.fixed.md new file mode 100644 index 0000000000..f5f9f76aad --- /dev/null +++ b/changelog.d/unreleased/3085.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 3085 +affected: + - src/CodeIndex/Database/DbSearchReader.cs + - tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs +--- + +## English + +- **Guarded search reads only requested line windows from matching chunks (#3085)** — guard context extraction now materializes only the focused line range instead of splitting every overlapping chunk into all lines. + +## 日本語 + +- **guard付き検索が一致chunkから要求された行windowだけを読み取るようになりました (#3085)** — guard文脈抽出は、重なったchunk全体を全行分割せず、focusされた行範囲だけをmaterializeします。 diff --git a/src/CodeIndex/Database/DbSearchReader.cs b/src/CodeIndex/Database/DbSearchReader.cs index 130a23a00d..5762367df4 100644 --- a/src/CodeIndex/Database/DbSearchReader.cs +++ b/src/CodeIndex/Database/DbSearchReader.cs @@ -657,13 +657,14 @@ FROM chunks c while (reader.TrackedRead()) { var chunkStartLine = reader.GetInt32(0); - var chunkLines = SplitContentLines(reader.GetString(1)); - for (var i = 0; i < chunkLines.Length; i++) + var relativeStart = Math.Max(0, startLine - chunkStartLine); + var relativeEnd = Math.Max(relativeStart, endLine - chunkStartLine); + foreach (var (lineOffset, text) in EnumerateContentLines(reader.GetString(1), relativeStart, relativeEnd)) { - var lineNumber = chunkStartLine + i; + var lineNumber = chunkStartLine + lineOffset; if (lineNumber < startLine || lineNumber > endLine) continue; - linesByNumber.TryAdd(lineNumber, chunkLines[i]); + linesByNumber.TryAdd(lineNumber, text); } } @@ -693,8 +694,14 @@ private static string NormalizeGuardSearchTerm(string value) private static string[] SplitContentLines(string content) => content.Replace("\r\n", "\n").Split('\n'); - private static IEnumerable<(int Index, string Text)> EnumerateContentLines(string content) + private static IEnumerable<(int Index, string Text)> EnumerateContentLines(string content) => + EnumerateContentLines(content, startIndex: 0, endIndex: int.MaxValue); + + private static IEnumerable<(int Index, string Text)> EnumerateContentLines(string content, int startIndex, int endIndex) { + if (endIndex < startIndex) + yield break; + var lineStart = 0; var lineIndex = 0; for (var i = 0; i < content.Length; i++) @@ -705,12 +712,16 @@ private static string[] SplitContentLines(string content) var lineEnd = i; if (lineEnd > lineStart && content[lineEnd - 1] == '\r') lineEnd--; - yield return (lineIndex, content[lineStart..lineEnd]); + if (lineIndex > endIndex) + yield break; + if (lineIndex >= startIndex) + yield return (lineIndex, content[lineStart..lineEnd]); lineIndex++; lineStart = i + 1; } - yield return (lineIndex, content[lineStart..]); + if (lineIndex >= startIndex && lineIndex <= endIndex) + yield return (lineIndex, content[lineStart..]); } private static string FormatSearchGuardRole(SearchGuardRole role) diff --git a/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs index bcfc707306..fcc3652c02 100644 --- a/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs +++ b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs @@ -117,6 +117,33 @@ public void Search_GuardFiltersFocusLargeCandidateWithPreparedPrimaryTerms_Issue Assert.Equal(24_999, evidence.Line); } + [Fact] + public void Search_GuardFiltersReadTinyWindowFromLargeChunk_Issue3085() + { + var lines = Enumerable.Range(1, 40_000) + .Select(i => i switch + { + 1 => "public void Setup() { TinyGuardMarker(); }", + 2 => "public void Run() { TinyWindowNeedle(); }", + _ => $"// filler {i}", + }); + InsertIndexedFile("src/guard-window-large.cs", "csharp", string.Join('\n', lines)); + + var results = _reader.Search( + "TinyWindowNeedle", + exact: true, + pathPatterns: ["src/guard-window-large.cs"], + limit: 1, + guardFilters: [new SearchGuardFilter(SearchGuardRole.Require, SearchGuardDirection.Before, "TinyGuardMarker")], + guardWindow: 1); + + var result = Assert.Single(results); + Assert.Equal(2, result.StartLine); + var evidence = Assert.Single(result.GuardEvidence!); + Assert.Equal(1, evidence.Line); + Assert.Equal("public void Setup() { TinyGuardMarker(); }", evidence.Text); + } + private void InsertIndexedFile(string path, string lang, string content, DateTime? modified = null) { var normalized = content.Replace("\r\n", "\n"); From 1cbce44531705ed58bcab1af88b1589e44034ef4 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 6 Jun 2026 01:10:45 +0900 Subject: [PATCH 5/6] Cache guarded search line windows (#3084) --- changelog.d/unreleased/3084.fixed.md | 16 ++++++++++ src/CodeIndex/Database/DbSearchReader.cs | 32 +++++++++++++++++-- .../DbSearchReaderIssueTests.cs | 28 ++++++++++++++++ 3 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 changelog.d/unreleased/3084.fixed.md diff --git a/changelog.d/unreleased/3084.fixed.md b/changelog.d/unreleased/3084.fixed.md new file mode 100644 index 0000000000..61dc81c12d --- /dev/null +++ b/changelog.d/unreleased/3084.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 3084 +affected: + - src/CodeIndex/Database/DbSearchReader.cs + - tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs +--- + +## English + +- **Guarded search caches line-window reads within each request (#3084)** — guard evaluation now reuses bounded path/window reads for repeated filters or nearby candidates while keeping the cache scoped and capped. + +## 日本語 + +- **guard付き検索がリクエスト内でline-window読み取りをcacheするようになりました (#3084)** — guard評価は、繰り返しfilterや近接候補で同じpath/windowの読み取りを再利用し、cacheはリクエスト内かつ上限付きに保ちます。 diff --git a/src/CodeIndex/Database/DbSearchReader.cs b/src/CodeIndex/Database/DbSearchReader.cs index 5762367df4..9009de1438 100644 --- a/src/CodeIndex/Database/DbSearchReader.cs +++ b/src/CodeIndex/Database/DbSearchReader.cs @@ -22,6 +22,7 @@ public partial class DbReader internal const int MaxGuardedSearchCandidates = 1000; private const int MinGuardedSearchCandidates = 200; private const int GuardedSearchOverFetchFactor = 50; + private const int MaxSearchGuardLineWindowCacheEntries = 256; /// /// Sanitize user input for FTS5 MATCH by quoting each token as a phrase. @@ -490,6 +491,7 @@ private List FilterBySearchGuards( { guardWindow = Math.Clamp(guardWindow, 0, MaxSearchGuardWindow); var filtered = new List(results.Count); + var lineWindowCache = new Dictionary>(); foreach (var result in results) { foreach (var (focusLine, focusText) in FindPrimarySearchMatchLines(result, primaryMatchContext)) @@ -498,7 +500,7 @@ private List FilterBySearchGuards( var keep = true; foreach (var filter in guardFilters) { - var match = FindGuardEvidence(result.Path, focusLine, filter, guardWindow, primaryMatchContext.GetEffectiveLang(result)); + var match = FindGuardEvidence(result.Path, focusLine, filter, guardWindow, primaryMatchContext.GetEffectiveLang(result), lineWindowCache); var matched = match != null; if (filter.Role == SearchGuardRole.Require && !matched) { @@ -596,7 +598,13 @@ private static string[] BuildPrimarySearchMatchTerms(string query, string normal .ToArray(); } - private SearchGuardEvidence? FindGuardEvidence(string path, int focusLine, SearchGuardFilter filter, int guardWindow, string? lang) + private SearchGuardEvidence? FindGuardEvidence( + string path, + int focusLine, + SearchGuardFilter filter, + int guardWindow, + string? lang, + Dictionary> lineWindowCache) { var windowStart = filter.Direction == SearchGuardDirection.Before ? Math.Max(1, focusLine - guardWindow) @@ -608,7 +616,7 @@ private static string[] BuildPrimarySearchMatchTerms(string query, string normal if (windowEnd < windowStart) return null; - var lineWindow = ReadLineWindow(path, windowStart, windowEnd); + var lineWindow = ReadLineWindow(path, windowStart, windowEnd, lineWindowCache); if (lineWindow.Count == 0) return null; @@ -637,6 +645,22 @@ private static string[] BuildPrimarySearchMatchTerms(string query, string normal return null; } + private SortedDictionary ReadLineWindow( + string path, + int startLine, + int endLine, + Dictionary> lineWindowCache) + { + var key = new SearchGuardLineWindowKey(path, startLine, endLine); + if (lineWindowCache.TryGetValue(key, out var cached)) + return cached; + + var lineWindow = ReadLineWindow(path, startLine, endLine); + if (lineWindowCache.Count < MaxSearchGuardLineWindowCacheEntries) + lineWindowCache[key] = lineWindow; + return lineWindow; + } + private SortedDictionary ReadLineWindow(string path, int startLine, int endLine) { var linesByNumber = new SortedDictionary(); @@ -671,6 +695,8 @@ FROM chunks c return linesByNumber; } + private readonly record struct SearchGuardLineWindowKey(string Path, int StartLine, int EndLine); + private static List PageGuardedSearchResults(List results, int limit, SearchCursor? cursor) { var offset = Math.Max(0, cursor?.Offset ?? 0); diff --git a/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs index fcc3652c02..691885d77c 100644 --- a/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs +++ b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs @@ -144,6 +144,34 @@ public void Search_GuardFiltersReadTinyWindowFromLargeChunk_Issue3085() Assert.Equal("public void Setup() { TinyGuardMarker(); }", evidence.Text); } + [Fact] + public void Search_GuardFiltersShareSameFocusWindowAcrossFilters_Issue3084() + { + InsertIndexedFile( + "src/guard-cache.cs", + "csharp", + """ + public void First() { FirstGuardMarker(); } + public void Second() { SecondGuardMarker(); } + public void Run() { CachedWindowNeedle(); } + """); + + var results = _reader.Search( + "CachedWindowNeedle", + exact: true, + pathPatterns: ["src/guard-cache.cs"], + limit: 1, + guardFilters: + [ + new SearchGuardFilter(SearchGuardRole.Require, SearchGuardDirection.Before, "FirstGuardMarker"), + new SearchGuardFilter(SearchGuardRole.Require, SearchGuardDirection.Before, "SecondGuardMarker"), + ], + guardWindow: 2); + + var result = Assert.Single(results); + Assert.Equal([1, 2], result.GuardEvidence!.Select(evidence => evidence.Line).ToArray()); + } + private void InsertIndexedFile(string path, string lang, string content, DateTime? modified = null) { var normalized = content.Replace("\r\n", "\n"); From 011770199c42db6485d663c25426d2f96f3ca9e7 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 6 Jun 2026 01:14:23 +0900 Subject: [PATCH 6/6] Reuse search match context for enclosing symbols (#3086) --- changelog.d/unreleased/3086.fixed.md | 16 ++++ src/CodeIndex/Database/DbSearchReader.cs | 101 +++++++++++++++-------- tests/CodeIndex.Tests/DbReaderTests.cs | 30 +++++++ 3 files changed, 111 insertions(+), 36 deletions(-) create mode 100644 changelog.d/unreleased/3086.fixed.md diff --git a/changelog.d/unreleased/3086.fixed.md b/changelog.d/unreleased/3086.fixed.md new file mode 100644 index 0000000000..492896ecd5 --- /dev/null +++ b/changelog.d/unreleased/3086.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 3086 +affected: + - src/CodeIndex/Database/DbSearchReader.cs + - tests/CodeIndex.Tests/DbReaderTests.cs +--- + +## English + +- **Search enclosing-symbol lookup reuses prepared match-line context (#3086)** — search enrichment now caches normalized query terms per language and streams candidate content to the first matching line instead of rebuilding normalized line arrays per result. + +## 日本語 + +- **検索のenclosing symbol lookupが準備済みmatch-line contextを再利用するようになりました (#3086)** — 検索結果の補強は、言語ごとの正規化query語をcacheし、候補本文を最初の一致行までstreamして、結果ごとに正規化済み行配列を再構築しなくなりました。 diff --git a/src/CodeIndex/Database/DbSearchReader.cs b/src/CodeIndex/Database/DbSearchReader.cs index 9009de1438..7080a9d212 100644 --- a/src/CodeIndex/Database/DbSearchReader.cs +++ b/src/CodeIndex/Database/DbSearchReader.cs @@ -117,6 +117,7 @@ public List Search(string query, int limit = 20, string? lang = nu var normalizedQuery = rawQuery ? query : NormalizeLiteralSearchQuery(query, lang); var coverageTokens = exact ? new List() : GetSearchCoverageTokens(normalizedQuery, rawQuery); var hasGuardFilters = guardFilters is { Count: > 0 }; + var searchMatchLineContext = SearchMatchLineContext.Create(query, lang, exact); var exactSubstringBoost = !exact && !rawQuery && IsPunctuationHeavyLiteralQuery(query); var guardedCandidateLimit = hasGuardFilters ? GetGuardedSearchCandidateLimit(limit, cursor) : 0; using var cmd = _conn.CreateCommand(); @@ -226,7 +227,7 @@ FROM fts_chunks if (guardCandidateLimitReached && results.Count < GetGuardedSearchRequestedPageEnd(limit, cursor)) throw new SearchGuardCandidateLimitException(guardedCandidateLimit, limit, cursor?.Offset ?? 0); - AttachSearchEnclosingSymbols(results, query, exact); + AttachSearchEnclosingSymbols(results, searchMatchLineContext); return hasGuardFilters ? PageGuardedSearchResults(results, limit, cursor) : results; } @@ -250,11 +251,11 @@ private static long GetGuardedSearchRequestedPageEnd(int limit, SearchCursor? cu return requestedOffset + requestedLimit; } - private void AttachSearchEnclosingSymbols(IReadOnlyList results, string query, bool caseSensitive) + private void AttachSearchEnclosingSymbols(IReadOnlyList results, SearchMatchLineContext matchLineContext) { foreach (var result in results) { - var matchLine = GetFirstSearchMatchLine(result, query, caseSensitive); + var matchLine = GetFirstSearchMatchLine(result, matchLineContext); if (!matchLine.HasValue) continue; @@ -270,54 +271,82 @@ private void AttachSearchEnclosingSymbols(IReadOnlyList results, s } } - private static int? GetFirstSearchMatchLine(SearchResult result, string query, bool caseSensitive) + private static int? GetFirstSearchMatchLine(SearchResult result, SearchMatchLineContext context) { - var lines = result.Content.Replace("\r\n", "\n").Split('\n'); - if (lines.Length == 0) - return null; + var prepared = context.ForResult(result); + int? firstTokenMatchLine = null; - var normalizedQuery = ExactSourceSearchNormalizer.Normalize(query.Trim(), result.Lang); - var normalizedLines = new string[lines.Length]; - for (int i = 0; i < lines.Length; i++) - normalizedLines[i] = ExactSourceSearchNormalizer.Normalize(lines[i], result.Lang); + foreach (var (lineIndex, text) in EnumerateContentLines(result.Content)) + { + var line = prepared.NormalizeLine(text); + if (!string.IsNullOrWhiteSpace(prepared.NormalizedQuery) && + line.Contains(prepared.NormalizedQuery, prepared.Comparison)) + { + return result.StartLine + lineIndex; + } - var tokens = query - .Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries) - .Select(NormalizeSearchSnippetToken) - .Where(token => token.Length > 0) - .Where(token => token is not "AND" and not "OR" and not "NOT" and not "NEAR") - .Select(token => ExactSourceSearchNormalizer.Normalize(token, result.Lang)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); + if (firstTokenMatchLine.HasValue || prepared.Tokens.Length == 0) + continue; - var matchIndexes = FindSearchMatchingLineIndexes(normalizedLines, normalizedQuery, tokens, caseSensitive); - return matchIndexes.Count > 0 ? result.StartLine + matchIndexes[0] : null; + if (prepared.Tokens.Any(token => line.Contains(token, prepared.Comparison))) + firstTokenMatchLine = result.StartLine + lineIndex; + } + + return firstTokenMatchLine; } - private static List FindSearchMatchingLineIndexes(string[] lines, string query, string[] tokens, bool caseSensitive) + private sealed class SearchMatchLineContext { - var comparison = caseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; - var matches = new List(); + private readonly string _query; + private readonly string? _queryLang; + private readonly bool _caseSensitive; + private readonly Dictionary _termsByLang = new(StringComparer.OrdinalIgnoreCase); - if (!string.IsNullOrWhiteSpace(query)) + private SearchMatchLineContext(string query, string? queryLang, bool caseSensitive) { - for (int i = 0; i < lines.Length; i++) - { - if (lines[i].Contains(query, comparison)) - matches.Add(i); - } + _query = query; + _queryLang = queryLang; + _caseSensitive = caseSensitive; } - if (matches.Count > 0 || tokens.Length == 0) - return matches; + public static SearchMatchLineContext Create(string query, string? queryLang, bool caseSensitive) + => new(query, queryLang, caseSensitive); - for (int i = 0; i < lines.Length; i++) + public SearchMatchLineTerms ForResult(SearchResult result) { - if (tokens.Any(token => lines[i].Contains(token, comparison))) - matches.Add(i); + var lang = _queryLang ?? result.Lang; + var key = lang ?? string.Empty; + if (_termsByLang.TryGetValue(key, out var prepared)) + return prepared; + + prepared = SearchMatchLineTerms.Create(_query, lang, _caseSensitive); + _termsByLang[key] = prepared; + return prepared; } + } - return matches; + private sealed record SearchMatchLineTerms( + string NormalizedQuery, + string[] Tokens, + StringComparison Comparison, + string? Lang) + { + public static SearchMatchLineTerms Create(string query, string? lang, bool caseSensitive) + { + var normalizedQuery = ExactSourceSearchNormalizer.Normalize(query.Trim(), lang); + var tokens = query + .Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries) + .Select(NormalizeSearchSnippetToken) + .Where(token => token.Length > 0) + .Where(token => token is not "AND" and not "OR" and not "NOT" and not "NEAR") + .Select(token => ExactSourceSearchNormalizer.Normalize(token, lang)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + var comparison = caseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; + return new SearchMatchLineTerms(normalizedQuery, tokens, comparison, lang); + } + + public string NormalizeLine(string line) => ExactSourceSearchNormalizer.Normalize(line, Lang); } private static string NormalizeSearchSnippetToken(string token) diff --git a/tests/CodeIndex.Tests/DbReaderTests.cs b/tests/CodeIndex.Tests/DbReaderTests.cs index b4bae49362..e4531ffd94 100644 --- a/tests/CodeIndex.Tests/DbReaderTests.cs +++ b/tests/CodeIndex.Tests/DbReaderTests.cs @@ -213,6 +213,36 @@ public void Search_GuardFiltersReadAcrossChunkBoundaries_Issue2852() Assert.Single(cursorResults); } + [Fact] + public void Search_EnclosingSymbolUsesPreparedMatchLineContext_Issue3086() + { + var filler = string.Join('\n', Enumerable.Range(1, 2_000).Select(i => $" // filler {i}")); + InsertIndexedFile( + "src/enclosing-large.cs", + "csharp", + $$""" + namespace Demo; + public class Worker + { + public void Run() + { + {{filler}} + EnclosingNeedle(); + } + } + """); + + var results = _reader.Search( + "EnclosingNeedle", + exact: true, + pathPatterns: ["src/enclosing-large.cs"], + limit: 1); + + var result = Assert.Single(results); + Assert.Equal("Run", result.EnclosingSymbolName); + Assert.Equal("function", result.EnclosingSymbolKind); + } + [Theory] [InlineData("rowid:authenticate", "rowid:")] [InlineData("title:authenticate", "title:")]