diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 7d072b16b1..5ac711f802 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -746,6 +746,11 @@ cdidx search "--open-reports" --path README.md --count # quoted literal that st cdidx search --query "--path" --path README.md # search for an option-looking literal ``` +Search normalizes literal FTS queries to Unicode NFC before matching. If every +literal token exceeds SQLite FTS5 unicode61's 1000-character token cap, +zero-result JSON includes `query_degraded_reason` and `tokens_dropped`. Index +validation reports long unbroken FTS tokens as `fts_token_too_long`. + ### Debugging queries Add `--verbose` to any query command (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `deps`, `impact`, `unused`, or `hotspots`) to print query diagnostics to stderr without changing normal stdout: @@ -2775,6 +2780,11 @@ cdidx search "--open-reports" --path README.md --count # `--` で始まる引 cdidx search --query "--path" --path README.md # オプションに見えるリテラルを検索 ``` +literal FTS クエリは照合前に Unicode NFC へ正規化されます。すべての literal +token が SQLite FTS5 unicode61 の 1000 文字 token 上限を超える場合、0 件 +JSON には `query_degraded_reason` と `tokens_dropped` が含まれます。index +validation は長い連続 FTS token を `fts_token_too_long` として報告します。 + ### クエリのデバッグ 任意の query command（`search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`find`、`excerpt`、`map`、`inspect`、`outline`、`status`、`validate`、`deps`、`impact`、`unused`、`hotspots`）に `--verbose` を付けると、通常の stdout を変えずに query 診断を stderr へ出力します: diff --git a/changelog.d/unreleased/1458.fixed.md b/changelog.d/unreleased/1458.fixed.md new file mode 100644 index 0000000000..1642f6d8d8 --- /dev/null +++ b/changelog.d/unreleased/1458.fixed.md @@ -0,0 +1,20 @@ +--- +category: fixed +issues: + - 1458 + - 1606 + - 1776 +affected: + - src/CodeIndex/Database/DbSearchReader.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - USER_GUIDE.md +--- + +## English + +- **Search now reports degraded overlong-token FTS queries (#1458, #1606, #1776)** — zero-result JSON now distinguishes overlong-token-only literal FTS queries, literal search normalizes Unicode input to NFC, and validation reports FTS5 unicode61 tokens longer than 1000 characters as `fts_token_too_long`. + +## 日本語 + +- **長すぎる token だけの FTS クエリに検索診断を追加しました (#1458, #1606, #1776)** — 0 件 JSON は長すぎる token のみの literal FTS クエリを区別し、literal 検索は Unicode 入力を NFC 正規化し、validation は FTS5 unicode61 の 1000 文字上限を超える token を `fts_token_too_long` として報告します。 diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 87e05fd567..44d6c0d92e 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -344,6 +344,7 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(FileIssue))] [JsonSerializable(typeof(FileResult))] [JsonSerializable(typeof(FreshnessHintResult))] +[JsonSerializable(typeof(FtsQueryDiagnostics))] [JsonSerializable(typeof(GroupedHotspotResult))] [JsonSerializable(typeof(GroupedSymbolHotspotJsonResult))] [JsonSerializable(typeof(ImpactAnalysisResult))] diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index b939afffa8..baee27b354 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -409,10 +409,11 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions) if (options.CountOnly) { var counts = reader.CountSearchResults(options.Query, options.Lang, options.RawFts, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, !options.NoDedup, options.Since, exact, options.Prefix, !options.NoVisibilityRank); + var queryDiagnostics = DbReader.AnalyzeFtsQuery(options.Query, options.RawFts, options.Prefix, options.Lang); if (counts.Count == 0) { Console.WriteLine(options.Json - ? BuildJsonZeroResultPayload(reader, jsonOptions, includeFiles: true, query: options.Query, queryOptions: options).ToJsonString(jsonOptions) + ? BuildJsonZeroResultPayload(reader, jsonOptions, includeFiles: true, query: options.Query, ftsQueryDiagnostics: queryDiagnostics, queryOptions: options).ToJsonString(jsonOptions) : "0"); return CommandExitCodes.Success; } @@ -424,6 +425,7 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions) } var results = reader.Search(options.Query, options.Limit, options.Lang, options.RawFts, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, !options.NoDedup, options.Since, exact, options.Prefix, !options.NoVisibilityRank); + var ftsQueryDiagnostics = DbReader.AnalyzeFtsQuery(options.Query, options.RawFts, options.Prefix, options.Lang); if (results.Count == 0) { if (options.Json && TryWriteEmptyFormattedResult(options, jsonOptions)) @@ -440,7 +442,7 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions) } else { - Console.WriteLine(BuildJsonZeroResultPayload(reader, jsonOptions, resultsKey: "results", query: options.Query, queryOptions: options).ToJsonString(jsonOptions)); + Console.WriteLine(BuildJsonZeroResultPayload(reader, jsonOptions, resultsKey: "results", query: options.Query, ftsQueryDiagnostics: ftsQueryDiagnostics, queryOptions: options).ToJsonString(jsonOptions)); jsonDoneCount = 0; } } @@ -4365,7 +4367,7 @@ private static JsonObject BuildUnusedResultsByBucketJson(IEnumerable public partial class DbReader { + internal const int FtsUnicode61MaxTokenLength = 1000; + internal const string AllTokensFilteredByLengthReason = "all_tokens_filtered_by_length"; internal const int MaxRawFtsQueryLength = 2000; internal const int MaxRawFtsBooleanOperators = 64; internal const int MaxRawFtsNearOperators = 16; @@ -40,6 +43,26 @@ internal static string SanitizeFtsQuery(string query, bool prefix) return string.Join(" ", tokens.Select(token => FormatFtsToken(token, prefix))); } + public static FtsQueryDiagnostics AnalyzeFtsQuery(string query, bool rawQuery = false, bool prefix = false, string? lang = null) + { + if (rawQuery || string.IsNullOrWhiteSpace(query)) + return FtsQueryDiagnostics.None; + + var normalizedQuery = NormalizeLiteralSearchQuery(query, NormalizeQueryLanguage(lang)); + var tokens = normalizedQuery.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries) + .Select(token => token.Length > 1 && token.EndsWith('*') ? token[..^1] : token) + .Where(token => token.Length > 0) + .ToArray(); + if (tokens.Length == 0) + return FtsQueryDiagnostics.None; + + var tooLong = tokens.Where(token => token.EnumerateRunes().Count() > FtsUnicode61MaxTokenLength).Distinct(StringComparer.Ordinal).ToArray(); + if (tooLong.Length == tokens.Length) + return new FtsQueryDiagnostics(AllTokensFilteredByLengthReason, tooLong); + + return FtsQueryDiagnostics.None; + } + /// /// Build a single FTS5 phrase token. A trailing user-supplied `*` is preserved as a prefix /// shorthand; otherwise the token is quoted as a literal phrase. When @@ -269,10 +292,13 @@ FROM fts_chunks return new QueryCountResult(count, fileCount); } - private static string NormalizeLiteralSearchQuery(string query, string? lang) => - string.Equals(lang, "csharp", StringComparison.OrdinalIgnoreCase) - ? CSharpVerbatimNameNormalizer.Normalize(query) - : query; + private static string NormalizeLiteralSearchQuery(string query, string? lang) + { + var normalized = query.Normalize(NormalizationForm.FormC); + return string.Equals(lang, "csharp", StringComparison.OrdinalIgnoreCase) + ? CSharpVerbatimNameNormalizer.Normalize(normalized) + : normalized; + } internal static string ValidateRawFtsQuery(string query) { diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index 3473929b2a..40f70165d7 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -3430,6 +3430,18 @@ public static List ValidateContent(string relativePath, byte[] rawByt }); } + var longFtsTokenLine = FindOversizeFtsTokenLine(content, CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength); + if (longFtsTokenLine > 0) + { + issues.Add(new FileIssue + { + Path = relativePath, + Kind = "fts_token_too_long", + Line = longFtsTokenLine, + Message = $"Line {longFtsTokenLine} contains an FTS5 unicode61 token longer than {CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength} characters; that token is not searchable through FTS", + }); + } + return issues; } @@ -3598,6 +3610,43 @@ private static int FindOversizeLine(string content, int maxLineLength) return 0; } + private static int FindOversizeFtsTokenLine(string content, int maxTokenLength) + { + if (string.IsNullOrEmpty(content)) + return 0; + + var lineNumber = 1; + var tokenLength = 0; + foreach (var rune in content.EnumerateRunes()) + { + if (rune.Value == '\n') + { + lineNumber++; + tokenLength = 0; + continue; + } + + if (IsLikelyUnicode61TokenRune(rune)) + { + tokenLength++; + if (tokenLength > maxTokenLength) + return lineNumber; + } + else + { + tokenLength = 0; + } + } + + return 0; + } + + private static bool IsLikelyUnicode61TokenRune(Rune rune) + => rune.Value == '_' + || Rune.IsLetter(rune) + || Rune.IsDigit(rune) + || Rune.GetUnicodeCategory(rune) == UnicodeCategory.NonSpacingMark; + /// /// Count U+FFFD replacement characters in decoded content. /// デコード済みcontent内のU+FFFD置換文字数を計上する。 diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 50de8ef1c4..db4f84aead 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -146,6 +146,18 @@ private static void AddFreshnessHint(JsonObject payload, DbReader reader) payload["freshness_degraded_reason"] = freshness.FreshnessDegradedReason; } + private static void AddFtsQueryDiagnostics(JsonObject payload, FtsQueryDiagnostics diagnostics) + { + if (!diagnostics.HasDegradation) + return; + + payload["query_degraded_reason"] = diagnostics.QueryDegradedReason; + var dropped = new JsonArray(); + foreach (var token in diagnostics.TokensDropped) + dropped.Add(token); + payload["tokens_dropped"] = dropped; + } + private static void AddExactZeroHint(JsonObject payload, ExactZeroHintResult? exactZeroHint) { if (exactZeroHint == null) @@ -855,10 +867,13 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) payload["rawQuery"] = rawQuery; payload["path"] = PathEcho(pathPatterns); payload["excludeTests"] = excludeTests; + if (countResults.Count == 0) + AddFtsQueryDiagnostics(payload, DbReader.AnalyzeFtsQuery(query, rawQuery, prefix, lang)); return CreateToolResult(id, $"Counted {countResults.Count} search result(s).", payload); } var results = reader.Search(query, FetchLimitForEnvelope(limit), lang, rawQuery, pathPatterns, excludePaths, excludeTests, deduplicate, since, exact, prefix); + var ftsDiagnostics = DbReader.AnalyzeFtsQuery(query, rawQuery, prefix, lang); var truncated = TrimToRequestedLimit(results, limit); if (results.Count == 0) { @@ -872,6 +887,7 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) ["excludeTests"] = excludeTests, ["results"] = new JsonArray() }; + AddFtsQueryDiagnostics(payload, ftsDiagnostics); AddResultEnvelope(payload, 0, 0, truncated: false); AddRecoveryHint( payload, diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index de7b56144a..d2cd4e3bb6 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -22,6 +22,14 @@ public class SearchResult public readonly record struct QueryCountResult(int Count, int FileCount, bool IncludesSql = false); +public sealed record FtsQueryDiagnostics( + [property: JsonPropertyName("query_degraded_reason")] string? QueryDegradedReason, + [property: JsonPropertyName("tokens_dropped")] IReadOnlyList TokensDropped) +{ + public static FtsQueryDiagnostics None { get; } = new(null, []); + public bool HasDegradation => QueryDegradedReason != null; +} + public enum ReferenceRankMode { Weighted, diff --git a/tests/CodeIndex.Tests/DbReaderTests.cs b/tests/CodeIndex.Tests/DbReaderTests.cs index 31e3a9d96f..8f493282a0 100644 --- a/tests/CodeIndex.Tests/DbReaderTests.cs +++ b/tests/CodeIndex.Tests/DbReaderTests.cs @@ -128,6 +128,27 @@ public void CountSearchResults_NormalizesJavascriptLangSpelling() Assert.Equal(1, counts.FileCount); } + [Fact] + public void AnalyzeFtsQuery_AllTokensTooLong_ReturnsDegradedReason() + { + var query = new string('x', DbReader.FtsUnicode61MaxTokenLength + 1); + + var diagnostics = DbReader.AnalyzeFtsQuery(query); + + Assert.Equal(DbReader.AllTokensFilteredByLengthReason, diagnostics.QueryDegradedReason); + Assert.Equal([query], diagnostics.TokensDropped); + } + + [Fact] + public void Search_ExplicitPrefixMatchesLatinDiacriticToken() + { + InsertIndexedFile("src/cafe.md", "markdown", "menu café_au_lait\n"); + + var results = _reader.Search("café*", lang: "markdown"); + + Assert.Contains(results, r => r.Path == "src/cafe.md"); + } + [Theory] [InlineData("rowid:authenticate", "rowid:")] [InlineData("title:authenticate", "title:")] diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 058274e251..01f27896a9 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -4646,6 +4646,20 @@ public void ValidateContent_NoOversizeLine_DoesNotEmitLineTooLongIssue() Assert.DoesNotContain(issues, i => i.Kind == "line_too_long"); } + [Fact] + public void ValidateContent_OversizeFtsToken_EmitsFtsTokenTooLongIssue() + { + var token = new string('x', CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength + 1); + var content = "ok\nconst value = " + token + ";\n"; + var raw = System.Text.Encoding.UTF8.GetBytes(content); + + var issues = FileIndexer.ValidateContent("generated.js", raw, content); + + var issue = Assert.Single(issues, i => i.Kind == "fts_token_too_long"); + Assert.Equal(2, issue.Line); + Assert.Contains("not searchable through FTS", issue.Message); + } + [Fact] public void SymbolExtractor_Extract_OversizeLine_ReturnsEmpty() {