From 4463f155d954f123702904f9deabc4f99f2eb7b2 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 5 Jun 2026 02:00:36 +0900 Subject: [PATCH 1/3] Fix punctuation-heavy search ranking (#2998) --- changelog.d/unreleased/2998.fixed.md | 16 ++++ src/CodeIndex/Database/DbSearchReader.cs | 45 +++++++++- .../DbSearchReaderIssueTests.cs | 82 +++++++++++++++++++ 3 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 changelog.d/unreleased/2998.fixed.md create mode 100644 tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs diff --git a/changelog.d/unreleased/2998.fixed.md b/changelog.d/unreleased/2998.fixed.md new file mode 100644 index 0000000000..110a1f3a9f --- /dev/null +++ b/changelog.d/unreleased/2998.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 2998 +affected: + - src/CodeIndex/Database/DbSearchReader.cs + - tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs +--- + +## English + +- **Punctuation-heavy search phrases now rank exact substring hits first (#2998)** - ordinary `search` results now boost literal substring matches for code-like punctuation queries such as `catch {`, reducing punctuation-only noise before users need to fall back to `--exact-substring`. + +## 日本語 + +- **句読点を多く含む search phrase で exact substring hit を優先するようになりました (#2998)** - `catch {` のような code-like punctuation query では通常の `search` 結果でも literal substring match を上位に寄せ、`--exact-substring` に切り替える前の punctuation-only noise を減らします。 diff --git a/src/CodeIndex/Database/DbSearchReader.cs b/src/CodeIndex/Database/DbSearchReader.cs index a2469fdc75..838bd6efb9 100644 --- a/src/CodeIndex/Database/DbSearchReader.cs +++ b/src/CodeIndex/Database/DbSearchReader.cs @@ -113,6 +113,7 @@ public List Search(string query, int limit = 20, string? lang = nu var normalizedQuery = rawQuery ? query : NormalizeLiteralSearchQuery(query, lang); var coverageTokens = exact ? new List() : GetSearchCoverageTokens(normalizedQuery, rawQuery); var hasGuardFilters = guardFilters is { Count: > 0 }; + var exactSubstringBoost = !exact && !rawQuery && IsPunctuationHeavyLiteralQuery(query); using var cmd = _conn.CreateCommand(); string sql; @@ -153,7 +154,7 @@ FROM fts_chunks if (since != null && _fileColumns.Contains("modified")) sql += " AND f.modified >= @since"; AppendPathFilters(ref sql, pathPatterns, excludePathPatterns, excludeTests); - sql += $" ORDER BY {GetSearchOrderSql(coverageTokens.Count)}"; + sql += $" ORDER BY {GetSearchOrderSql(coverageTokens.Count, exactSubstringBoost)}"; if (!hasGuardFilters) sql += " LIMIT @limit"; if (cursor is { } && !hasGuardFilters) @@ -386,7 +387,7 @@ FROM fts_chunks sql += " AND f.modified >= @since"; AppendPathFilters(ref sql, pathPatterns, excludePathPatterns, excludeTests); - sql += $" ORDER BY {GetSearchOrderSql(coverageTokens.Count)}"; + sql += $" ORDER BY {GetSearchOrderSql(coverageTokens.Count, exactSubstringBoost: false)}"; cmd.CommandText = sql; if (exact) @@ -1038,10 +1039,46 @@ private static bool OverlapsOrTouches((int Start, int End) interval, int start, } } - private static string GetSearchOrderSql(int coverageTokenCount) + private static string GetSearchOrderSql(int coverageTokenCount, bool exactSubstringBoost) { var coverageOrder = GetSearchCoverageOrderSql(coverageTokenCount); - return $"{PathBucketOrder}, {ExactSymbolMatchOrder}, {PrefixSymbolMatchOrder}, {SearchVisibilityOrder}, {PathTextMatchOrder}, {ChunkTextMatchOrder}, {ChunkStructuredFieldOrder}, {ChunkSymbolKindOrder}, {ChunkSymbolDepthOrder}, {coverageOrder}rank, f.modified DESC, f.path, c.id ASC"; + var exactSubstringOrder = exactSubstringBoost + ? $"CASE WHEN instr({GetExactSearchTextSql("c.content", "f.lang")}, {GetExactSearchTextSql("@rankingQuery", "f.lang")}) > 0 THEN 0 ELSE 1 END, " + : string.Empty; + return $"{PathBucketOrder}, {exactSubstringOrder}{ExactSymbolMatchOrder}, {PrefixSymbolMatchOrder}, {SearchVisibilityOrder}, {PathTextMatchOrder}, {ChunkTextMatchOrder}, {ChunkStructuredFieldOrder}, {ChunkSymbolKindOrder}, {ChunkSymbolDepthOrder}, {coverageOrder}rank, f.modified DESC, f.path, c.id ASC"; + } + + private static bool IsPunctuationHeavyLiteralQuery(string query) + { + var trimmed = query.Trim(); + if (trimmed.Length == 0 || !trimmed.Any(char.IsLetterOrDigit)) + return false; + + var tokens = trimmed.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + var punctuationCount = trimmed.Count(IsCodePunctuation); + return punctuationCount >= 2 || tokens.Any(IsStandaloneCodeOperatorToken); + } + + private static bool IsStandaloneCodeOperatorToken(string token) + => token.Length > 0 + && token.All(ch => !char.IsLetterOrDigit(ch) && !char.IsWhiteSpace(ch) && ch != '_') + && token.Any(IsCodePunctuation); + + private static bool IsCodePunctuation(char ch) + { + if (char.IsLetterOrDigit(ch) || char.IsWhiteSpace(ch) || ch == '_') + return false; + + return ch is '.' + or ':' or ';' or ',' + or '=' or '$' or '@' or '#' + or '%' or '^' or '&' or '|' + or '!' or '?' or '+' or '-' + or '*' or '/' or '\\' + or '<' or '>' + or '(' or ')' or '[' or ']' + or '{' or '}' + or '"' or '\'' or '`' or '~'; } private static string GetSearchCoverageOrderSql(int coverageTokenCount) diff --git a/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs new file mode 100644 index 0000000000..0a593dc2cd --- /dev/null +++ b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs @@ -0,0 +1,82 @@ +using CodeIndex.Database; +using CodeIndex.Models; + +namespace CodeIndex.Tests; + +[Collection("SQLite pool sensitive")] +public sealed class DbSearchReaderIssueTests : IDisposable +{ + private readonly string _dbPath; + private readonly DbContext _db; + private readonly DbWriter _writer; + private readonly DbReader _reader; + + public DbSearchReaderIssueTests() + { + _dbPath = Path.Combine(Path.GetTempPath(), $"codeindex_search_issue_test_{Guid.NewGuid():N}.db"); + _db = new DbContext(_dbPath); + _db.InitializeSchema(); + _writer = new DbWriter(_db.Connection); + _writer.MarkGraphReady(); + _writer.MarkIssuesReady(); + _writer.MarkFoldReady(); + _reader = new DbReader(_db.Connection); + } + + [Fact] + public void Search_PunctuationHeavyPhraseRanksExactSubstringFirst_Issue2998() + { + InsertIndexedFile( + "src/search-boost-newer.cs", + "csharp", + """ + try + { + } + catch + { + } + """, + modified: new DateTime(2025, 6, 2, 0, 0, 0, DateTimeKind.Utc)); + InsertIndexedFile( + "src/search-boost-exact.cs", + "csharp", + "public void Run() { try { Work(); } catch { } }", + modified: new DateTime(2025, 6, 1, 0, 0, 0, DateTimeKind.Utc)); + + var results = _reader.Search("catch {", pathPatterns: ["src/search-boost-*.cs"], limit: 2); + + Assert.NotEmpty(results); + Assert.Equal("src/search-boost-exact.cs", results[0].Path); + } + + private void InsertIndexedFile(string path, string lang, string content, DateTime? modified = null) + { + var normalized = content.Replace("\r\n", "\n"); + var lines = normalized.Split('\n'); + var fileId = _writer.UpsertFile(new FileRecord + { + Path = path, + Lang = lang, + Size = normalized.Length, + Lines = lines.Length, + Modified = modified ?? new DateTime(2025, 6, 1, 0, 0, 0, DateTimeKind.Utc), + }); + + _writer.InsertChunks([new ChunkRecord + { + FileId = fileId, + ChunkIndex = 0, + StartLine = 1, + EndLine = lines.Length, + Content = normalized, + }]); + } + + public void Dispose() + { + _reader.Dispose(); + _db.Dispose(); + TestProjectHelper.DeleteFile(_dbPath); + } +} From 7342539dc8497a809332b46be1852bc0b58ebde5 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 5 Jun 2026 02:14:34 +0900 Subject: [PATCH 2/3] Bound guarded search candidate collection (#3082) --- changelog.d/unreleased/3082.fixed.md | 19 +++++++++ src/CodeIndex/Cli/QueryCommandRunner.cs | 6 +++ src/CodeIndex/Database/DbSearchReader.cs | 37 ++++++++++++++++- .../SearchGuardCandidateLimitException.cs | 16 ++++++++ src/CodeIndex/Mcp/McpToolHandlers.cs | 20 ++++++++- .../DbSearchReaderIssueTests.cs | 41 +++++++++++++++++++ 6 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 changelog.d/unreleased/3082.fixed.md create mode 100644 src/CodeIndex/Database/SearchGuardCandidateLimitException.cs diff --git a/changelog.d/unreleased/3082.fixed.md b/changelog.d/unreleased/3082.fixed.md new file mode 100644 index 0000000000..3fc6e6c32f --- /dev/null +++ b/changelog.d/unreleased/3082.fixed.md @@ -0,0 +1,19 @@ +--- +category: fixed +issues: + - 3082 +affected: + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Database/DbSearchReader.cs + - src/CodeIndex/Database/SearchGuardCandidateLimitException.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs +--- + +## English + +- **Guarded search now bounds candidate collection before pagination (#3082)** - searches using guard filters now apply a capped over-fetch budget in SQL instead of collecting every matching chunk in memory, and return a validation error when a guarded query is too broad to satisfy the requested page within that budget. + +## 日本語 + +- **guard filter 付き search が pagination 前の候補収集を制限するようになりました (#3082)** - guard filter を使う検索は、全一致 chunk を memory に集める代わりに SQL 側で capped over-fetch budget を適用し、その budget 内で要求ページを満たせないほど広い guarded query には validation error を返します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index 4fd7dc0883..b0dde703e0 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -6510,6 +6510,12 @@ private static int WithDb(QueryCommandOptions options, JsonSerializerOptions jso } return CommandExitCodes.UsageError; } + catch (SearchGuardCandidateLimitException ex) + { + Console.Error.WriteLine($"Error [{CommandErrorCodes.UsageError}]: guarded search is too broad: {ex.Message}"); + Console.Error.WriteLine("Hint: narrow the search with more specific query text, --lang, --path, or --exclude-tests, or reduce pagination offset before retrying guarded search."); + return CommandExitCodes.UsageError; + } catch (Exception ex) { if (JsonOutputFailure.TryHandle(ex, out var exitCode)) diff --git a/src/CodeIndex/Database/DbSearchReader.cs b/src/CodeIndex/Database/DbSearchReader.cs index 838bd6efb9..d8f08aad20 100644 --- a/src/CodeIndex/Database/DbSearchReader.cs +++ b/src/CodeIndex/Database/DbSearchReader.cs @@ -19,6 +19,9 @@ public partial class DbReader internal const int DefaultSearchGuardWindow = 8; internal const int MaxSearchGuardWindow = 200; internal const int MaxSearchGuardFilters = 8; + internal const int MaxGuardedSearchCandidates = 1000; + private const int MinGuardedSearchCandidates = 200; + private const int GuardedSearchOverFetchFactor = 50; /// /// Sanitize user input for FTS5 MATCH by quoting each token as a phrase. @@ -114,6 +117,7 @@ public List Search(string query, int limit = 20, string? lang = nu var coverageTokens = exact ? new List() : GetSearchCoverageTokens(normalizedQuery, rawQuery); var hasGuardFilters = guardFilters is { Count: > 0 }; var exactSubstringBoost = !exact && !rawQuery && IsPunctuationHeavyLiteralQuery(query); + var guardedCandidateLimit = hasGuardFilters ? GetGuardedSearchCandidateLimit(limit, cursor) : 0; using var cmd = _conn.CreateCommand(); string sql; @@ -155,7 +159,9 @@ FROM fts_chunks sql += " AND f.modified >= @since"; AppendPathFilters(ref sql, pathPatterns, excludePathPatterns, excludeTests); sql += $" ORDER BY {GetSearchOrderSql(coverageTokens.Count, exactSubstringBoost)}"; - if (!hasGuardFilters) + if (hasGuardFilters) + sql += " LIMIT @candidateFetchLimit"; + else sql += " LIMIT @limit"; if (cursor is { } && !hasGuardFilters) sql += " OFFSET @cursorOffset"; @@ -169,6 +175,8 @@ FROM fts_chunks AddSearchCoverageParameters(cmd, coverageTokens); if (!hasGuardFilters) cmd.Parameters.AddWithValue("@limit", limit); + else + cmd.Parameters.AddWithValue("@candidateFetchLimit", guardedCandidateLimit + 1); if (lang != null) cmd.Parameters.AddWithValue("@lang", lang); if (since != null && _fileColumns.Contains("modified")) @@ -206,14 +214,41 @@ FROM fts_chunks throw new FtsQuerySyntaxException(ex.Message, ex); } + var guardCandidateLimitReached = hasGuardFilters && raw.Count > guardedCandidateLimit; + if (guardCandidateLimitReached) + raw.RemoveRange(guardedCandidateLimit, raw.Count - guardedCandidateLimit); + if (hasGuardFilters) raw = FilterBySearchGuards(raw, query, normalizedQuery, rawQuery, exact, lang, guardFilters!, guardWindow); var results = deduplicate ? DeduplicateOverlappingResults(raw) : raw; + if (guardCandidateLimitReached && results.Count < GetGuardedSearchRequestedPageEnd(limit, cursor)) + throw new SearchGuardCandidateLimitException(guardedCandidateLimit, limit, cursor?.Offset ?? 0); + AttachSearchEnclosingSymbols(results, query, exact); return hasGuardFilters ? PageGuardedSearchResults(results, limit, cursor) : results; } + private static int GetGuardedSearchCandidateLimit(int limit, SearchCursor? cursor) + { + var requestedLimit = Math.Max(0L, limit); + var requestedOffset = Math.Max(0L, cursor?.Offset ?? 0); + var requestedPageEnd = requestedOffset + requestedLimit; + if (requestedPageEnd <= 0) + return 0; + + var overFetched = requestedPageEnd * GuardedSearchOverFetchFactor; + var candidateLimit = Math.Max(MinGuardedSearchCandidates, overFetched); + return (int)Math.Min(MaxGuardedSearchCandidates, candidateLimit); + } + + private static long GetGuardedSearchRequestedPageEnd(int limit, SearchCursor? cursor) + { + var requestedLimit = Math.Max(0L, limit); + var requestedOffset = Math.Max(0L, cursor?.Offset ?? 0); + return requestedOffset + requestedLimit; + } + private void AttachSearchEnclosingSymbols(IReadOnlyList results, string query, bool caseSensitive) { foreach (var result in results) diff --git a/src/CodeIndex/Database/SearchGuardCandidateLimitException.cs b/src/CodeIndex/Database/SearchGuardCandidateLimitException.cs new file mode 100644 index 0000000000..32c689cebe --- /dev/null +++ b/src/CodeIndex/Database/SearchGuardCandidateLimitException.cs @@ -0,0 +1,16 @@ +namespace CodeIndex.Database; + +internal sealed class SearchGuardCandidateLimitException : Exception +{ + public SearchGuardCandidateLimitException(int candidateLimit, int requestedLimit, int requestedOffset) + : base($"guarded search inspected the maximum {candidateLimit} candidate chunks before satisfying the requested page (limit {requestedLimit}, offset {requestedOffset}).") + { + CandidateLimit = candidateLimit; + RequestedLimit = requestedLimit; + RequestedOffset = requestedOffset; + } + + public int CandidateLimit { get; } + public int RequestedLimit { get; } + public int RequestedOffset { get; } +} diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 828d8c1a87..53dfa895fb 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -1130,7 +1130,15 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) { if (countOnly) { - var countResults = reader.Search(query, MaxLimit, lang, rawQuery, pathPatterns, excludePaths, excludeTests, deduplicate, since, exact, prefix, guardFilters: guardFilters, guardWindow: guardWindow); + List countResults; + try + { + countResults = reader.Search(query, MaxLimit, lang, rawQuery, pathPatterns, excludePaths, excludeTests, deduplicate, since, exact, prefix, guardFilters: guardFilters, guardWindow: guardWindow); + } + catch (SearchGuardCandidateLimitException ex) + { + return CreateToolErrorResponse(id, $"guarded search is too broad: {ex.Message} Narrow the search with more specific query text, lang/path filters, or a smaller cursor offset."); + } var truncatedCount = countResults.Count >= MaxLimit; var payload = BuildCountOnlyPayload(countResults.Count, truncatedCount ? null : countResults.Count, truncatedCount, countResults, result => result.Path); payload["query"] = query; @@ -1145,7 +1153,15 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) return CreateToolResult(id, $"Counted {countResults.Count} search result(s).", payload); } - var results = reader.Search(query, FetchLimitForEnvelope(limit), lang, rawQuery, pathPatterns, excludePaths, excludeTests, deduplicate, since, exact, prefix, cursor: cursor, guardFilters: guardFilters, guardWindow: guardWindow); + List results; + try + { + results = reader.Search(query, FetchLimitForEnvelope(limit), lang, rawQuery, pathPatterns, excludePaths, excludeTests, deduplicate, since, exact, prefix, cursor: cursor, guardFilters: guardFilters, guardWindow: guardWindow); + } + catch (SearchGuardCandidateLimitException ex) + { + return CreateToolErrorResponse(id, $"guarded search is too broad: {ex.Message} Narrow the search with more specific query text, lang/path filters, or a smaller cursor offset."); + } var ftsDiagnostics = DbReader.AnalyzeFtsQuery(query, rawQuery, prefix, lang); var truncated = TrimToRequestedLimit(results, limit); if (results.Count == 0) diff --git a/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs index 0a593dc2cd..36f35c12e0 100644 --- a/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs +++ b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs @@ -50,6 +50,47 @@ public void Search_PunctuationHeavyPhraseRanksExactSubstringFirst_Issue2998() Assert.Equal("src/search-boost-exact.cs", results[0].Path); } + [Fact] + public void Search_GuardFiltersRejectTooBroadCandidateCollectionBeforePagination_Issue3082() + { + for (var i = 0; i < 200; i++) + InsertIndexedFile($"src/guard-budget-{i:0000}.cs", "csharp", "public void Run() { BudgetNeedle(); }"); + + InsertIndexedFile( + "src/guard-budget-9999.cs", + "csharp", + """ + public void Setup() { GuardMarker(); } + public void Run() { BudgetNeedle(); } + """); + + var ex = Assert.Throws(() => _reader.Search( + "BudgetNeedle", + pathPatterns: ["src/guard-budget-*.cs"], + limit: 1, + guardFilters: [new SearchGuardFilter(SearchGuardRole.Require, SearchGuardDirection.Before, "GuardMarker")], + guardWindow: 1)); + + Assert.Equal(200, ex.CandidateLimit); + Assert.Contains("guarded search inspected the maximum", ex.Message); + } + + [Fact] + public void Search_GuardFiltersDoNotRejectWhenCandidateCountExactlyMatchesBudget_Issue3082() + { + for (var i = 0; i < 200; i++) + InsertIndexedFile($"src/guard-budget-exact-{i:0000}.cs", "csharp", "public void Run() { ExactBudgetNeedle(); }"); + + var results = _reader.Search( + "ExactBudgetNeedle", + pathPatterns: ["src/guard-budget-exact-*.cs"], + limit: 1, + guardFilters: [new SearchGuardFilter(SearchGuardRole.Require, SearchGuardDirection.Before, "MissingGuardMarker")], + guardWindow: 1); + + Assert.Empty(results); + } + private void InsertIndexedFile(string path, string lang, string content, DateTime? modified = null) { var normalized = content.Replace("\r\n", "\n"); From e078876c9c9ef74f62ecc6d37c83404efc296d80 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 5 Jun 2026 02:36:04 +0900 Subject: [PATCH 3/3] Document guarded search broad-query errors (#3082) --- USER_GUIDE.md | 7 +++++++ changelog.d/unreleased/3082.fixed.md | 1 + 2 files changed, 8 insertions(+) diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 59a7135545..39bf04338b 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -854,6 +854,10 @@ Guard-aware search filters primary `search` matches by nearby literal guards: appears in the selected line window, while `--reject-before` / `--reject-after` drop matches when the guard query appears. JSON search results include `guard_evidence` for required guards that matched. +Guarded searches inspect a bounded candidate set before pagination; if a guarded +query is too broad to satisfy the requested page within that budget, CLI and MCP +return a validation error. Narrow with more specific query text, `--lang`, +`--path`, `--exclude-tests`, or a smaller MCP cursor offset. The MCP `search` tool exposes the same mode as camelCase arguments: `requireBefore`, `requireAfter`, `rejectBefore`, `rejectAfter`, and `guardWindow`. @@ -3058,6 +3062,9 @@ guard-aware search は primary の `search` 一致を近傍の literal guard で `--require-before` / `--require-after` は指定行窓内に guard query がある場合だけ残し、 `--reject-before` / `--reject-after` は guard query がある一致を落とします。JSON の検索結果には 一致した required guard の `guard_evidence` が含まれます。 +guard filter を使う検索は pagination 前に上限付きの候補集合だけを調べます。その budget 内で +要求ページを満たせないほど query が広い場合、CLI/MCP は validation error を返します。 +query text、`--lang`、`--path`、`--exclude-tests` で絞り込むか、MCP cursor の offset を小さくしてください。 MCP `search` tool では同じ mode を camelCase 引数 `requireBefore`, `requireAfter`, `rejectBefore`, `rejectAfter`, `guardWindow` で指定できます。 diff --git a/changelog.d/unreleased/3082.fixed.md b/changelog.d/unreleased/3082.fixed.md index 3fc6e6c32f..cf328d0c9d 100644 --- a/changelog.d/unreleased/3082.fixed.md +++ b/changelog.d/unreleased/3082.fixed.md @@ -8,6 +8,7 @@ affected: - src/CodeIndex/Database/SearchGuardCandidateLimitException.cs - src/CodeIndex/Mcp/McpToolHandlers.cs - tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs + - USER_GUIDE.md --- ## English