From 2b82bb73c8fce648d1a98ca8c1aece2228d6157d Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 19:49:24 +0900 Subject: [PATCH 1/3] Fix FTS zero-result diagnostics (#1458, #1776) --- USER_GUIDE.md | 12 +++++++ changelog.d/unreleased/1458.fixed.md | 20 +++++++++++ src/CodeIndex/Cli/JsonOutputContracts.cs | 1 + src/CodeIndex/Cli/QueryCommandRunner.cs | 14 ++++++-- src/CodeIndex/Database/DbSearchReader.cs | 46 +++++++++++++++++++++--- src/CodeIndex/Mcp/McpToolHandlers.cs | 16 +++++++++ src/CodeIndex/Models/QueryResults.cs | 8 +++++ tests/CodeIndex.Tests/DbReaderTests.cs | 30 ++++++++++++++++ 8 files changed, 140 insertions(+), 7 deletions(-) create mode 100644 changelog.d/unreleased/1458.fixed.md diff --git a/USER_GUIDE.md b/USER_GUIDE.md index e486388045..4608aa9b33 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -739,6 +739,12 @@ cdidx search "--open-reports" --path README.md --count # quoted literal that st cdidx search --query "--path" --path README.md # search for an option-looking literal ``` +Search normalizes literal FTS queries to Unicode NFC before matching. If every +literal token is a common stop word or exceeds SQLite FTS5 unicode61's +1000-character token cap, zero-result JSON includes `query_degraded_reason` and +`tokens_dropped`. Index validation reports long unbroken FTS tokens as +`fts_token_too_long`. + ### Debugging queries Add `--verbose` to any query command (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `deps`, `impact`, `unused`, or `hotspots`) to print query diagnostics to stderr without changing normal stdout: @@ -2735,6 +2741,12 @@ cdidx search "--open-reports" --path README.md --count # `--` で始まる引 cdidx search --query "--path" --path README.md # オプションに見えるリテラルを検索 ``` +literal FTS クエリは照合前に Unicode NFC へ正規化されます。すべての literal +token が一般的な stop word、または SQLite FTS5 unicode61 の 1000 文字 token +上限を超える場合、0 件 JSON には `query_degraded_reason` と `tokens_dropped` +が含まれます。index validation は長い連続 FTS token を +`fts_token_too_long` として報告します。 + ### クエリのデバッグ 任意の query command（`search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`find`、`excerpt`、`map`、`inspect`、`outline`、`status`、`validate`、`deps`、`impact`、`unused`、`hotspots`）に `--verbose` を付けると、通常の stdout を変えずに query 診断を stderr へ出力します: diff --git a/changelog.d/unreleased/1458.fixed.md b/changelog.d/unreleased/1458.fixed.md new file mode 100644 index 0000000000..1c4dcfc42b --- /dev/null +++ b/changelog.d/unreleased/1458.fixed.md @@ -0,0 +1,20 @@ +--- +category: fixed +issues: + - 1458 + - 1606 + - 1776 +affected: + - src/CodeIndex/Database/DbSearchReader.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - USER_GUIDE.md +--- + +## English + +- **Search now reports degraded all-filtered FTS queries (#1458, #1606, #1776)** — zero-result JSON now distinguishes stop-word-only and overlong-token-only literal FTS queries, literal search normalizes Unicode input to NFC, and validation reports FTS5 unicode61 tokens longer than 1000 characters as `fts_token_too_long`. + +## 日本語 + +- **FTS クエリが全て除外された場合の検索診断を追加しました (#1458, #1606, #1776)** — 0 件 JSON は stop word のみ、または長すぎる token のみの literal FTS クエリを区別し、literal 検索は Unicode 入力を NFC 正規化し、validation は FTS5 unicode61 の 1000 文字上限を超える token を `fts_token_too_long` として報告します。 diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 87e05fd567..44d6c0d92e 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -344,6 +344,7 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(FileIssue))] [JsonSerializable(typeof(FileResult))] [JsonSerializable(typeof(FreshnessHintResult))] +[JsonSerializable(typeof(FtsQueryDiagnostics))] [JsonSerializable(typeof(GroupedHotspotResult))] [JsonSerializable(typeof(GroupedSymbolHotspotJsonResult))] [JsonSerializable(typeof(ImpactAnalysisResult))] diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index b939afffa8..baee27b354 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -409,10 +409,11 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions) if (options.CountOnly) { var counts = reader.CountSearchResults(options.Query, options.Lang, options.RawFts, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, !options.NoDedup, options.Since, exact, options.Prefix, !options.NoVisibilityRank); + var queryDiagnostics = DbReader.AnalyzeFtsQuery(options.Query, options.RawFts, options.Prefix, options.Lang); if (counts.Count == 0) { Console.WriteLine(options.Json - ? BuildJsonZeroResultPayload(reader, jsonOptions, includeFiles: true, query: options.Query, queryOptions: options).ToJsonString(jsonOptions) + ? BuildJsonZeroResultPayload(reader, jsonOptions, includeFiles: true, query: options.Query, ftsQueryDiagnostics: queryDiagnostics, queryOptions: options).ToJsonString(jsonOptions) : "0"); return CommandExitCodes.Success; } @@ -424,6 +425,7 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions) } var results = reader.Search(options.Query, options.Limit, options.Lang, options.RawFts, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, !options.NoDedup, options.Since, exact, options.Prefix, !options.NoVisibilityRank); + var ftsQueryDiagnostics = DbReader.AnalyzeFtsQuery(options.Query, options.RawFts, options.Prefix, options.Lang); if (results.Count == 0) { if (options.Json && TryWriteEmptyFormattedResult(options, jsonOptions)) @@ -440,7 +442,7 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions) } else { - Console.WriteLine(BuildJsonZeroResultPayload(reader, jsonOptions, resultsKey: "results", query: options.Query, queryOptions: options).ToJsonString(jsonOptions)); + Console.WriteLine(BuildJsonZeroResultPayload(reader, jsonOptions, resultsKey: "results", query: options.Query, ftsQueryDiagnostics: ftsQueryDiagnostics, queryOptions: options).ToJsonString(jsonOptions)); jsonDoneCount = 0; } } @@ -4365,7 +4367,7 @@ private static JsonObject BuildUnusedResultsByBucketJson(IEnumerable public partial class DbReader { + internal const int FtsUnicode61MaxTokenLength = 1000; + internal const string AllTokensFilteredAsStopwordsReason = "all_tokens_filtered_as_stopwords"; + internal const string AllTokensFilteredByLengthReason = "all_tokens_filtered_by_length"; internal const int MaxRawFtsQueryLength = 2000; internal const int MaxRawFtsBooleanOperators = 64; internal const int MaxRawFtsNearOperators = 16; @@ -40,6 +44,30 @@ internal static string SanitizeFtsQuery(string query, bool prefix) return string.Join(" ", tokens.Select(token => FormatFtsToken(token, prefix))); } + public static FtsQueryDiagnostics AnalyzeFtsQuery(string query, bool rawQuery = false, bool prefix = false, string? lang = null) + { + if (rawQuery || string.IsNullOrWhiteSpace(query)) + return FtsQueryDiagnostics.None; + + var normalizedQuery = NormalizeLiteralSearchQuery(query, NormalizeQueryLanguage(lang)); + var tokens = normalizedQuery.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries) + .Select(token => token.Length > 1 && token.EndsWith('*') ? token[..^1] : token) + .Where(token => token.Length > 0) + .ToArray(); + if (tokens.Length == 0) + return FtsQueryDiagnostics.None; + + var tooLong = tokens.Where(token => token.EnumerateRunes().Count() > FtsUnicode61MaxTokenLength).Distinct(StringComparer.Ordinal).ToArray(); + if (tooLong.Length == tokens.Length) + return new FtsQueryDiagnostics(AllTokensFilteredByLengthReason, tooLong); + + var stopwords = tokens.Where(token => CommonFtsStopwords.Contains(token.Normalize(NormalizationForm.FormC), StringComparer.OrdinalIgnoreCase)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); + if (stopwords.Length == tokens.Length) + return new FtsQueryDiagnostics(AllTokensFilteredAsStopwordsReason, stopwords); + + return FtsQueryDiagnostics.None; + } + /// /// Build a single FTS5 phrase token. A trailing user-supplied `*` is preserved as a prefix /// shorthand; otherwise the token is quoted as a literal phrase. When @@ -269,10 +297,20 @@ FROM fts_chunks return new QueryCountResult(count, fileCount); } - private static string NormalizeLiteralSearchQuery(string query, string? lang) => - string.Equals(lang, "csharp", StringComparison.OrdinalIgnoreCase) - ? CSharpVerbatimNameNormalizer.Normalize(query) - : query; + private static string NormalizeLiteralSearchQuery(string query, string? lang) + { + var normalized = query.Normalize(NormalizationForm.FormC); + return string.Equals(lang, "csharp", StringComparison.OrdinalIgnoreCase) + ? CSharpVerbatimNameNormalizer.Normalize(normalized) + : normalized; + } + + private static readonly string[] CommonFtsStopwords = + [ + "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "from", "if", "in", "into", + "is", "it", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", + "these", "they", "this", "to", "was", "will", "with" + ]; internal static string ValidateRawFtsQuery(string query) { diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 50de8ef1c4..db4f84aead 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -146,6 +146,18 @@ private static void AddFreshnessHint(JsonObject payload, DbReader reader) payload["freshness_degraded_reason"] = freshness.FreshnessDegradedReason; } + private static void AddFtsQueryDiagnostics(JsonObject payload, FtsQueryDiagnostics diagnostics) + { + if (!diagnostics.HasDegradation) + return; + + payload["query_degraded_reason"] = diagnostics.QueryDegradedReason; + var dropped = new JsonArray(); + foreach (var token in diagnostics.TokensDropped) + dropped.Add(token); + payload["tokens_dropped"] = dropped; + } + private static void AddExactZeroHint(JsonObject payload, ExactZeroHintResult? exactZeroHint) { if (exactZeroHint == null) @@ -855,10 +867,13 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) payload["rawQuery"] = rawQuery; payload["path"] = PathEcho(pathPatterns); payload["excludeTests"] = excludeTests; + if (countResults.Count == 0) + AddFtsQueryDiagnostics(payload, DbReader.AnalyzeFtsQuery(query, rawQuery, prefix, lang)); return CreateToolResult(id, $"Counted {countResults.Count} search result(s).", payload); } var results = reader.Search(query, FetchLimitForEnvelope(limit), lang, rawQuery, pathPatterns, excludePaths, excludeTests, deduplicate, since, exact, prefix); + var ftsDiagnostics = DbReader.AnalyzeFtsQuery(query, rawQuery, prefix, lang); var truncated = TrimToRequestedLimit(results, limit); if (results.Count == 0) { @@ -872,6 +887,7 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) ["excludeTests"] = excludeTests, ["results"] = new JsonArray() }; + AddFtsQueryDiagnostics(payload, ftsDiagnostics); AddResultEnvelope(payload, 0, 0, truncated: false); AddRecoveryHint( payload, diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index de7b56144a..d2cd4e3bb6 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -22,6 +22,14 @@ public class SearchResult public readonly record struct QueryCountResult(int Count, int FileCount, bool IncludesSql = false); +public sealed record FtsQueryDiagnostics( + [property: JsonPropertyName("query_degraded_reason")] string? QueryDegradedReason, + [property: JsonPropertyName("tokens_dropped")] IReadOnlyList TokensDropped) +{ + public static FtsQueryDiagnostics None { get; } = new(null, []); + public bool HasDegradation => QueryDegradedReason != null; +} + public enum ReferenceRankMode { Weighted, diff --git a/tests/CodeIndex.Tests/DbReaderTests.cs b/tests/CodeIndex.Tests/DbReaderTests.cs index 31e3a9d96f..f911266e23 100644 --- a/tests/CodeIndex.Tests/DbReaderTests.cs +++ b/tests/CodeIndex.Tests/DbReaderTests.cs @@ -128,6 +128,36 @@ public void CountSearchResults_NormalizesJavascriptLangSpelling() Assert.Equal(1, counts.FileCount); } + [Fact] + public void AnalyzeFtsQuery_AllStopwords_ReturnsDegradedReason() + { + var diagnostics = DbReader.AnalyzeFtsQuery("and or the"); + + Assert.Equal(DbReader.AllTokensFilteredAsStopwordsReason, diagnostics.QueryDegradedReason); + Assert.Equal(["and", "or", "the"], diagnostics.TokensDropped); + } + + [Fact] + public void AnalyzeFtsQuery_AllTokensTooLong_ReturnsDegradedReason() + { + var query = new string('x', DbReader.FtsUnicode61MaxTokenLength + 1); + + var diagnostics = DbReader.AnalyzeFtsQuery(query); + + Assert.Equal(DbReader.AllTokensFilteredByLengthReason, diagnostics.QueryDegradedReason); + Assert.Equal([query], diagnostics.TokensDropped); + } + + [Fact] + public void Search_ExplicitPrefixMatchesLatinDiacriticToken() + { + InsertIndexedFile("src/cafe.md", "markdown", "menu café_au_lait\n"); + + var results = _reader.Search("café*", lang: "markdown"); + + Assert.Contains(results, r => r.Path == "src/cafe.md"); + } + [Theory] [InlineData("rowid:authenticate", "rowid:")] [InlineData("title:authenticate", "title:")] From 22bb6c898c74cae9977896801f456965f3a3bed4 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 19:49:37 +0900 Subject: [PATCH 2/3] Report overlong FTS tokens during validation (#1606) --- src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 49 +++++++++++++++++++ tests/CodeIndex.Tests/FileIndexerTests.cs | 14 ++++++ 2 files changed, 63 insertions(+) diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index 3473929b2a..40f70165d7 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -3430,6 +3430,18 @@ public static List ValidateContent(string relativePath, byte[] rawByt }); } + var longFtsTokenLine = FindOversizeFtsTokenLine(content, CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength); + if (longFtsTokenLine > 0) + { + issues.Add(new FileIssue + { + Path = relativePath, + Kind = "fts_token_too_long", + Line = longFtsTokenLine, + Message = $"Line {longFtsTokenLine} contains an FTS5 unicode61 token longer than {CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength} characters; that token is not searchable through FTS", + }); + } + return issues; } @@ -3598,6 +3610,43 @@ private static int FindOversizeLine(string content, int maxLineLength) return 0; } + private static int FindOversizeFtsTokenLine(string content, int maxTokenLength) + { + if (string.IsNullOrEmpty(content)) + return 0; + + var lineNumber = 1; + var tokenLength = 0; + foreach (var rune in content.EnumerateRunes()) + { + if (rune.Value == '\n') + { + lineNumber++; + tokenLength = 0; + continue; + } + + if (IsLikelyUnicode61TokenRune(rune)) + { + tokenLength++; + if (tokenLength > maxTokenLength) + return lineNumber; + } + else + { + tokenLength = 0; + } + } + + return 0; + } + + private static bool IsLikelyUnicode61TokenRune(Rune rune) + => rune.Value == '_' + || Rune.IsLetter(rune) + || Rune.IsDigit(rune) + || Rune.GetUnicodeCategory(rune) == UnicodeCategory.NonSpacingMark; + /// /// Count U+FFFD replacement characters in decoded content. /// デコード済みcontent内のU+FFFD置換文字数を計上する。 diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 058274e251..01f27896a9 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -4646,6 +4646,20 @@ public void ValidateContent_NoOversizeLine_DoesNotEmitLineTooLongIssue() Assert.DoesNotContain(issues, i => i.Kind == "line_too_long"); } + [Fact] + public void ValidateContent_OversizeFtsToken_EmitsFtsTokenTooLongIssue() + { + var token = new string('x', CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength + 1); + var content = "ok\nconst value = " + token + ";\n"; + var raw = System.Text.Encoding.UTF8.GetBytes(content); + + var issues = FileIndexer.ValidateContent("generated.js", raw, content); + + var issue = Assert.Single(issues, i => i.Kind == "fts_token_too_long"); + Assert.Equal(2, issue.Line); + Assert.Contains("not searchable through FTS", issue.Message); + } + [Fact] public void SymbolExtractor_Extract_OversizeLine_ReturnsEmpty() { From f1259f676690887d5bc0d1354fd4125e255a2377 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 20:07:26 +0900 Subject: [PATCH 3/3] Remove invalid FTS stopword diagnostics (#1458) --- USER_GUIDE.md | 14 ++++++-------- changelog.d/unreleased/1458.fixed.md | 4 ++-- src/CodeIndex/Database/DbSearchReader.cs | 12 ------------ tests/CodeIndex.Tests/DbReaderTests.cs | 9 --------- 4 files changed, 8 insertions(+), 31 deletions(-) diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 4608aa9b33..ceccee2201 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -740,10 +740,9 @@ cdidx search --query "--path" --path README.md # search for an option-l ``` Search normalizes literal FTS queries to Unicode NFC before matching. If every -literal token is a common stop word or exceeds SQLite FTS5 unicode61's -1000-character token cap, zero-result JSON includes `query_degraded_reason` and -`tokens_dropped`. Index validation reports long unbroken FTS tokens as -`fts_token_too_long`. +literal token exceeds SQLite FTS5 unicode61's 1000-character token cap, +zero-result JSON includes `query_degraded_reason` and `tokens_dropped`. Index +validation reports long unbroken FTS tokens as `fts_token_too_long`. ### Debugging queries @@ -2742,10 +2741,9 @@ cdidx search --query "--path" --path README.md # オプションに見 ``` literal FTS クエリは照合前に Unicode NFC へ正規化されます。すべての literal -token が一般的な stop word、または SQLite FTS5 unicode61 の 1000 文字 token -上限を超える場合、0 件 JSON には `query_degraded_reason` と `tokens_dropped` -が含まれます。index validation は長い連続 FTS token を -`fts_token_too_long` として報告します。 +token が SQLite FTS5 unicode61 の 1000 文字 token 上限を超える場合、0 件 +JSON には `query_degraded_reason` と `tokens_dropped` が含まれます。index +validation は長い連続 FTS token を `fts_token_too_long` として報告します。 ### クエリのデバッグ diff --git a/changelog.d/unreleased/1458.fixed.md b/changelog.d/unreleased/1458.fixed.md index 1c4dcfc42b..1642f6d8d8 100644 --- a/changelog.d/unreleased/1458.fixed.md +++ b/changelog.d/unreleased/1458.fixed.md @@ -13,8 +13,8 @@ affected: ## English -- **Search now reports degraded all-filtered FTS queries (#1458, #1606, #1776)** — zero-result JSON now distinguishes stop-word-only and overlong-token-only literal FTS queries, literal search normalizes Unicode input to NFC, and validation reports FTS5 unicode61 tokens longer than 1000 characters as `fts_token_too_long`. +- **Search now reports degraded overlong-token FTS queries (#1458, #1606, #1776)** — zero-result JSON now distinguishes overlong-token-only literal FTS queries, literal search normalizes Unicode input to NFC, and validation reports FTS5 unicode61 tokens longer than 1000 characters as `fts_token_too_long`. ## 日本語 -- **FTS クエリが全て除外された場合の検索診断を追加しました (#1458, #1606, #1776)** — 0 件 JSON は stop word のみ、または長すぎる token のみの literal FTS クエリを区別し、literal 検索は Unicode 入力を NFC 正規化し、validation は FTS5 unicode61 の 1000 文字上限を超える token を `fts_token_too_long` として報告します。 +- **長すぎる token だけの FTS クエリに検索診断を追加しました (#1458, #1606, #1776)** — 0 件 JSON は長すぎる token のみの literal FTS クエリを区別し、literal 検索は Unicode 入力を NFC 正規化し、validation は FTS5 unicode61 の 1000 文字上限を超える token を `fts_token_too_long` として報告します。 diff --git a/src/CodeIndex/Database/DbSearchReader.cs b/src/CodeIndex/Database/DbSearchReader.cs index 63393d2c47..5d2c54692f 100644 --- a/src/CodeIndex/Database/DbSearchReader.cs +++ b/src/CodeIndex/Database/DbSearchReader.cs @@ -10,7 +10,6 @@ namespace CodeIndex.Database; public partial class DbReader { internal const int FtsUnicode61MaxTokenLength = 1000; - internal const string AllTokensFilteredAsStopwordsReason = "all_tokens_filtered_as_stopwords"; internal const string AllTokensFilteredByLengthReason = "all_tokens_filtered_by_length"; internal const int MaxRawFtsQueryLength = 2000; internal const int MaxRawFtsBooleanOperators = 64; @@ -61,10 +60,6 @@ public static FtsQueryDiagnostics AnalyzeFtsQuery(string query, bool rawQuery = if (tooLong.Length == tokens.Length) return new FtsQueryDiagnostics(AllTokensFilteredByLengthReason, tooLong); - var stopwords = tokens.Where(token => CommonFtsStopwords.Contains(token.Normalize(NormalizationForm.FormC), StringComparer.OrdinalIgnoreCase)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); - if (stopwords.Length == tokens.Length) - return new FtsQueryDiagnostics(AllTokensFilteredAsStopwordsReason, stopwords); - return FtsQueryDiagnostics.None; } @@ -305,13 +300,6 @@ private static string NormalizeLiteralSearchQuery(string query, string? lang) : normalized; } - private static readonly string[] CommonFtsStopwords = - [ - "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "from", "if", "in", "into", - "is", "it", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", - "these", "they", "this", "to", "was", "will", "with" - ]; - internal static string ValidateRawFtsQuery(string query) { if (query.Length > MaxRawFtsQueryLength) diff --git a/tests/CodeIndex.Tests/DbReaderTests.cs b/tests/CodeIndex.Tests/DbReaderTests.cs index f911266e23..8f493282a0 100644 --- a/tests/CodeIndex.Tests/DbReaderTests.cs +++ b/tests/CodeIndex.Tests/DbReaderTests.cs @@ -128,15 +128,6 @@ public void CountSearchResults_NormalizesJavascriptLangSpelling() Assert.Equal(1, counts.FileCount); } - [Fact] - public void AnalyzeFtsQuery_AllStopwords_ReturnsDegradedReason() - { - var diagnostics = DbReader.AnalyzeFtsQuery("and or the"); - - Assert.Equal(DbReader.AllTokensFilteredAsStopwordsReason, diagnostics.QueryDegradedReason); - Assert.Equal(["and", "or", "the"], diagnostics.TokensDropped); - } - [Fact] public void AnalyzeFtsQuery_AllTokensTooLong_ReturnsDegradedReason() {