Skip to content

Commit a07cddd

Browse files
authored
Merge pull request #2773 from Widthdom/fix-issue1458-1606-1776
Fix FTS search degradation diagnostics
2 parents 3b92439 + 6908b8e commit a07cddd

10 files changed

Lines changed: 180 additions & 7 deletions

File tree

USER_GUIDE.md

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -749,6 +749,11 @@ cdidx search "--open-reports" --path README.md --count # quoted literal that st
749749
cdidx search --query "--path" --path README.md # search for an option-looking literal
750750
```
751751

752+
Search normalizes literal FTS queries to Unicode NFC before matching. If every
753+
literal token exceeds SQLite FTS5 unicode61's 1000-character token cap,
754+
zero-result JSON includes `query_degraded_reason` and `tokens_dropped`. Index
755+
validation reports long unbroken FTS tokens as `fts_token_too_long`.
756+
752757
### Debugging queries
753758

754759
Add `--verbose` to any query command (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `deps`, `impact`, `unused`, or `hotspots`) to print query diagnostics to stderr without changing normal stdout:
@@ -2783,6 +2788,11 @@ cdidx search "--open-reports" --path README.md --count # `--` で始まる引
27832788
cdidx search --query "--path" --path README.md # オプションに見えるリテラルを検索
27842789
```
27852790

2791+
literal FTS クエリは照合前に Unicode NFC へ正規化されます。すべての literal
2792+
token が SQLite FTS5 unicode61 の 1000 文字 token 上限を超える場合、0 件
2793+
JSON には `query_degraded_reason``tokens_dropped` が含まれます。index
2794+
validation は長い連続 FTS token を `fts_token_too_long` として報告します。
2795+
27862796
### クエリのデバッグ
27872797

27882798
任意の query command(`search``definition``references``callers``callees``symbols``files``find``excerpt``map``inspect``outline``status``validate``deps``impact``unused``hotspots`)に `--verbose` を付けると、通常の stdout を変えずに query 診断を stderr へ出力します:
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
---
2+
category: fixed
3+
issues:
4+
- 1458
5+
- 1606
6+
- 1776
7+
affected:
8+
- src/CodeIndex/Database/DbSearchReader.cs
9+
- src/CodeIndex/Cli/QueryCommandRunner.cs
10+
- src/CodeIndex/Indexer/Scanning/FileIndexer.cs
11+
- USER_GUIDE.md
12+
---
13+
14+
## English
15+
16+
- **Search now reports degraded overlong-token FTS queries (#1458, #1606, #1776)** — zero-result JSON now distinguishes overlong-token-only literal FTS queries, literal search normalizes Unicode input to NFC, and validation reports FTS5 unicode61 tokens longer than 1000 characters as `fts_token_too_long`.
17+
18+
## 日本語
19+
20+
- **長すぎる token だけの FTS クエリに検索診断を追加しました (#1458, #1606, #1776)** — 0 件 JSON は長すぎる token のみの literal FTS クエリを区別し、literal 検索は Unicode 入力を NFC 正規化し、validation は FTS5 unicode61 の 1000 文字上限を超える token を `fts_token_too_long` として報告します。

src/CodeIndex/Cli/JsonOutputContracts.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,7 @@ internal sealed record VersionInfoJsonResult(
372372
[JsonSerializable(typeof(FileIssue))]
373373
[JsonSerializable(typeof(FileResult))]
374374
[JsonSerializable(typeof(FreshnessHintResult))]
375+
[JsonSerializable(typeof(FtsQueryDiagnostics))]
375376
[JsonSerializable(typeof(GroupedHotspotResult))]
376377
[JsonSerializable(typeof(GroupedSymbolHotspotJsonResult))]
377378
[JsonSerializable(typeof(ImpactAnalysisResult))]

src/CodeIndex/Cli/QueryCommandRunner.cs

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -418,10 +418,11 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions)
418418
if (options.CountOnly)
419419
{
420420
var counts = reader.CountSearchResults(options.Query, options.Lang, options.RawFts, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, !options.NoDedup, options.Since, exact, options.Prefix, !options.NoVisibilityRank);
421+
var queryDiagnostics = DbReader.AnalyzeFtsQuery(options.Query, options.RawFts, options.Prefix, options.Lang);
421422
if (counts.Count == 0)
422423
{
423424
Console.WriteLine(options.Json
424-
? BuildJsonZeroResultPayload(reader, jsonOptions, includeFiles: true, query: options.Query, queryOptions: options).ToJsonString(jsonOptions)
425+
? BuildJsonZeroResultPayload(reader, jsonOptions, includeFiles: true, query: options.Query, ftsQueryDiagnostics: queryDiagnostics, queryOptions: options).ToJsonString(jsonOptions)
425426
: "0");
426427
return CommandExitCodes.Success;
427428
}
@@ -433,6 +434,7 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions)
433434
}
434435

435436
var results = reader.Search(options.Query, options.Limit, options.Lang, options.RawFts, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, !options.NoDedup, options.Since, exact, options.Prefix, !options.NoVisibilityRank);
437+
var ftsQueryDiagnostics = DbReader.AnalyzeFtsQuery(options.Query, options.RawFts, options.Prefix, options.Lang);
436438
if (results.Count == 0)
437439
{
438440
if (options.Json && TryWriteEmptyFormattedResult(options, jsonOptions))
@@ -449,7 +451,7 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions)
449451
}
450452
else
451453
{
452-
Console.WriteLine(BuildJsonZeroResultPayload(reader, jsonOptions, resultsKey: "results", query: options.Query, queryOptions: options).ToJsonString(jsonOptions));
454+
Console.WriteLine(BuildJsonZeroResultPayload(reader, jsonOptions, resultsKey: "results", query: options.Query, ftsQueryDiagnostics: ftsQueryDiagnostics, queryOptions: options).ToJsonString(jsonOptions));
453455
jsonDoneCount = 0;
454456
}
455457
}
@@ -4665,7 +4667,7 @@ private static JsonObject BuildUnusedResultsByBucketJson(IEnumerable<UnusedSymbo
46654667
// `--kind replacement_chra` のようなタイプミスを did-you-mean で救うため、
46664668
// FileIndexer.cs 内の `Kind = "..."` 代入と同期させる (#1582)。
46674669
private static readonly string[] AllValidValidateKinds =
4668-
["bom", "cr_only_line_endings", "file_too_large", "line_too_long", "mixed_line_endings", "mixed_line_endings_three_way", "non_utf8_likely", "null_byte", "replacement_char", "utf16_bom"];
4670+
["bom", "cr_only_line_endings", "file_too_large", "fts_token_too_long", "line_too_long", "mixed_line_endings", "mixed_line_endings_three_way", "non_utf8_likely", "null_byte", "replacement_char", "utf16_bom"];
46694671

46704672
public static int RunValidate(string[] cmdArgs, JsonSerializerOptions jsonOptions)
46714673
{
@@ -6838,6 +6840,7 @@ private static JsonObject BuildJsonZeroResultPayload(
68386840
string? resultsKey = null,
68396841
string? query = null,
68406842
ExactZeroHintResult? exactZeroHint = null,
6843+
FtsQueryDiagnostics? ftsQueryDiagnostics = null,
68416844
bool includeFiles = false,
68426845
bool? graphTableAvailable = null,
68436846
bool? degraded = null,
@@ -6868,6 +6871,11 @@ private static JsonObject BuildJsonZeroResultPayload(
68686871
}
68696872
if (exactZeroHint != null)
68706873
payload["exact_zero_hint"] = JsonSerializer.SerializeToNode(exactZeroHint, CliJsonSerializerContextFactory.Create(jsonOptions).ExactZeroHintResult);
6874+
if (ftsQueryDiagnostics is { HasDegradation: true })
6875+
{
6876+
payload["query_degraded_reason"] = ftsQueryDiagnostics.QueryDegradedReason;
6877+
payload["tokens_dropped"] = JsonSerializer.SerializeToNode(ftsQueryDiagnostics.TokensDropped.ToList(), CliJsonSerializerContextFactory.Create(jsonOptions).ListString);
6878+
}
68716879
if (queryOptions != null)
68726880
payload["query_context"] = BuildQueryContextJson(queryOptions, jsonOptions);
68736881
extraFields?.Invoke(payload);

src/CodeIndex/Database/DbSearchReader.cs

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using System.Text;
12
using Microsoft.Data.Sqlite;
23

34
namespace CodeIndex.Database;
@@ -8,6 +9,8 @@ namespace CodeIndex.Database;
89
/// </summary>
910
public partial class DbReader
1011
{
12+
internal const int FtsUnicode61MaxTokenLength = 1000;
13+
internal const string AllTokensFilteredByLengthReason = "all_tokens_filtered_by_length";
1114
internal const int MaxRawFtsQueryLength = 2000;
1215
internal const int MaxRawFtsBooleanOperators = 64;
1316
internal const int MaxRawFtsNearOperators = 16;
@@ -40,6 +43,26 @@ internal static string SanitizeFtsQuery(string query, bool prefix)
4043
return string.Join(" ", tokens.Select(token => FormatFtsToken(token, prefix)));
4144
}
4245

46+
public static FtsQueryDiagnostics AnalyzeFtsQuery(string query, bool rawQuery = false, bool prefix = false, string? lang = null)
47+
{
48+
if (rawQuery || string.IsNullOrWhiteSpace(query))
49+
return FtsQueryDiagnostics.None;
50+
51+
var normalizedQuery = NormalizeLiteralSearchQuery(query, NormalizeQueryLanguage(lang));
52+
var tokens = normalizedQuery.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)
53+
.Select(token => token.Length > 1 && token.EndsWith('*') ? token[..^1] : token)
54+
.Where(token => token.Length > 0)
55+
.ToArray();
56+
if (tokens.Length == 0)
57+
return FtsQueryDiagnostics.None;
58+
59+
var tooLong = tokens.Where(token => token.EnumerateRunes().Count() > FtsUnicode61MaxTokenLength).Distinct(StringComparer.Ordinal).ToArray();
60+
if (tooLong.Length == tokens.Length)
61+
return new FtsQueryDiagnostics(AllTokensFilteredByLengthReason, tooLong);
62+
63+
return FtsQueryDiagnostics.None;
64+
}
65+
4366
/// <summary>
4467
/// Build a single FTS5 phrase token. A trailing user-supplied `*` is preserved as a prefix
4568
/// shorthand; otherwise the token is quoted as a literal phrase. When <paramref name="prefix"/>
@@ -269,10 +292,13 @@ FROM fts_chunks
269292
return new QueryCountResult(count, fileCount);
270293
}
271294

272-
private static string NormalizeLiteralSearchQuery(string query, string? lang) =>
273-
string.Equals(lang, "csharp", StringComparison.OrdinalIgnoreCase)
274-
? CSharpVerbatimNameNormalizer.Normalize(query)
275-
: query;
295+
private static string NormalizeLiteralSearchQuery(string query, string? lang)
296+
{
297+
var normalized = query.Normalize(NormalizationForm.FormC);
298+
return string.Equals(lang, "csharp", StringComparison.OrdinalIgnoreCase)
299+
? CSharpVerbatimNameNormalizer.Normalize(normalized)
300+
: normalized;
301+
}
276302

277303
internal static string ValidateRawFtsQuery(string query)
278304
{

src/CodeIndex/Indexer/Scanning/FileIndexer.cs

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3430,6 +3430,18 @@ public static List<FileIssue> ValidateContent(string relativePath, byte[] rawByt
34303430
});
34313431
}
34323432

3433+
var longFtsTokenLine = FindOversizeFtsTokenLine(content, CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength);
3434+
if (longFtsTokenLine > 0)
3435+
{
3436+
issues.Add(new FileIssue
3437+
{
3438+
Path = relativePath,
3439+
Kind = "fts_token_too_long",
3440+
Line = longFtsTokenLine,
3441+
Message = $"Line {longFtsTokenLine} contains an FTS5 unicode61 token longer than {CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength} characters; that token is not searchable through FTS",
3442+
});
3443+
}
3444+
34333445
return issues;
34343446
}
34353447

@@ -3598,6 +3610,43 @@ private static int FindOversizeLine(string content, int maxLineLength)
35983610
return 0;
35993611
}
36003612

3613+
private static int FindOversizeFtsTokenLine(string content, int maxTokenLength)
3614+
{
3615+
if (string.IsNullOrEmpty(content))
3616+
return 0;
3617+
3618+
var lineNumber = 1;
3619+
var tokenLength = 0;
3620+
foreach (var rune in content.EnumerateRunes())
3621+
{
3622+
if (rune.Value == '\n')
3623+
{
3624+
lineNumber++;
3625+
tokenLength = 0;
3626+
continue;
3627+
}
3628+
3629+
if (IsLikelyUnicode61TokenRune(rune))
3630+
{
3631+
tokenLength++;
3632+
if (tokenLength > maxTokenLength)
3633+
return lineNumber;
3634+
}
3635+
else
3636+
{
3637+
tokenLength = 0;
3638+
}
3639+
}
3640+
3641+
return 0;
3642+
}
3643+
3644+
private static bool IsLikelyUnicode61TokenRune(Rune rune)
3645+
=> rune.Value == '_'
3646+
|| Rune.IsLetter(rune)
3647+
|| Rune.IsDigit(rune)
3648+
|| Rune.GetUnicodeCategory(rune) == UnicodeCategory.NonSpacingMark;
3649+
36013650
/// <summary>
36023651
/// Count U+FFFD replacement characters in decoded content.
36033652
/// デコード済みcontent内のU+FFFD置換文字数を計上する。

src/CodeIndex/Mcp/McpToolHandlers.cs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,18 @@ private static void AddFreshnessHint(JsonObject payload, DbReader reader)
147147
payload["freshness_degraded_reason"] = freshness.FreshnessDegradedReason;
148148
}
149149

150+
private static void AddFtsQueryDiagnostics(JsonObject payload, FtsQueryDiagnostics diagnostics)
151+
{
152+
if (!diagnostics.HasDegradation)
153+
return;
154+
155+
payload["query_degraded_reason"] = diagnostics.QueryDegradedReason;
156+
var dropped = new JsonArray();
157+
foreach (var token in diagnostics.TokensDropped)
158+
dropped.Add(token);
159+
payload["tokens_dropped"] = dropped;
160+
}
161+
150162
private static void AddExactZeroHint(JsonObject payload, ExactZeroHintResult? exactZeroHint)
151163
{
152164
if (exactZeroHint == null)
@@ -936,10 +948,13 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args)
936948
payload["rawQuery"] = rawQuery;
937949
payload["path"] = PathEcho(pathPatterns);
938950
payload["excludeTests"] = excludeTests;
951+
if (countResults.Count == 0)
952+
AddFtsQueryDiagnostics(payload, DbReader.AnalyzeFtsQuery(query, rawQuery, prefix, lang));
939953
return CreateToolResult(id, $"Counted {countResults.Count} search result(s).", payload);
940954
}
941955

942956
var results = reader.Search(query, FetchLimitForEnvelope(limit), lang, rawQuery, pathPatterns, excludePaths, excludeTests, deduplicate, since, exact, prefix);
957+
var ftsDiagnostics = DbReader.AnalyzeFtsQuery(query, rawQuery, prefix, lang);
943958
var truncated = TrimToRequestedLimit(results, limit);
944959
if (results.Count == 0)
945960
{
@@ -953,6 +968,7 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args)
953968
["excludeTests"] = excludeTests,
954969
["results"] = new JsonArray()
955970
};
971+
AddFtsQueryDiagnostics(payload, ftsDiagnostics);
956972
AddResultEnvelope(payload, 0, 0, truncated: false);
957973
AddRecoveryHint(
958974
payload,

src/CodeIndex/Models/QueryResults.cs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,14 @@ public class SearchResult
2222

2323
public readonly record struct QueryCountResult(int Count, int FileCount, bool IncludesSql = false);
2424

25+
public sealed record FtsQueryDiagnostics(
26+
[property: JsonPropertyName("query_degraded_reason")] string? QueryDegradedReason,
27+
[property: JsonPropertyName("tokens_dropped")] IReadOnlyList<string> TokensDropped)
28+
{
29+
public static FtsQueryDiagnostics None { get; } = new(null, []);
30+
public bool HasDegradation => QueryDegradedReason != null;
31+
}
32+
2533
public enum ReferenceRankMode
2634
{
2735
Weighted,

tests/CodeIndex.Tests/DbReaderTests.cs

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,27 @@ public void CountSearchResults_NormalizesJavascriptLangSpelling()
128128
Assert.Equal(1, counts.FileCount);
129129
}
130130

131+
[Fact]
132+
public void AnalyzeFtsQuery_AllTokensTooLong_ReturnsDegradedReason()
133+
{
134+
var query = new string('x', DbReader.FtsUnicode61MaxTokenLength + 1);
135+
136+
var diagnostics = DbReader.AnalyzeFtsQuery(query);
137+
138+
Assert.Equal(DbReader.AllTokensFilteredByLengthReason, diagnostics.QueryDegradedReason);
139+
Assert.Equal([query], diagnostics.TokensDropped);
140+
}
141+
142+
[Fact]
143+
public void Search_ExplicitPrefixMatchesLatinDiacriticToken()
144+
{
145+
InsertIndexedFile("src/cafe.md", "markdown", "menu café_au_lait\n");
146+
147+
var results = _reader.Search("café*", lang: "markdown");
148+
149+
Assert.Contains(results, r => r.Path == "src/cafe.md");
150+
}
151+
131152
[Theory]
132153
[InlineData("rowid:authenticate", "rowid:")]
133154
[InlineData("title:authenticate", "title:")]

tests/CodeIndex.Tests/FileIndexerTests.cs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4646,6 +4646,20 @@ public void ValidateContent_NoOversizeLine_DoesNotEmitLineTooLongIssue()
46464646
Assert.DoesNotContain(issues, i => i.Kind == "line_too_long");
46474647
}
46484648

4649+
[Fact]
4650+
public void ValidateContent_OversizeFtsToken_EmitsFtsTokenTooLongIssue()
4651+
{
4652+
var token = new string('x', CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength + 1);
4653+
var content = "ok\nconst value = " + token + ";\n";
4654+
var raw = System.Text.Encoding.UTF8.GetBytes(content);
4655+
4656+
var issues = FileIndexer.ValidateContent("generated.js", raw, content);
4657+
4658+
var issue = Assert.Single(issues, i => i.Kind == "fts_token_too_long");
4659+
Assert.Equal(2, issue.Line);
4660+
Assert.Contains("not searchable through FTS", issue.Message);
4661+
}
4662+
46494663
[Fact]
46504664
public void SymbolExtractor_Extract_OversizeLine_ReturnsEmpty()
46514665
{

0 commit comments

Comments
 (0)