Widthdom · Widthdom · May 31, 2026 · May 31, 2026 · May 31, 2026 · May 31, 2026
diff --git a/USER_GUIDE.md b/USER_GUIDE.md
@@ -746,6 +746,11 @@ cdidx search "--open-reports" --path README.md --count  # quoted literal that st
 cdidx search --query "--path" --path README.md          # search for an option-looking literal
 ```
 
+Search normalizes literal FTS queries to Unicode NFC before matching. If every
+literal token exceeds SQLite FTS5 unicode61's 1000-character token cap,
+zero-result JSON includes `query_degraded_reason` and `tokens_dropped`. Index
+validation reports long unbroken FTS tokens as `fts_token_too_long`.
+
 ### Debugging queries
 
 Add `--verbose` to any query command (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `deps`, `impact`, `unused`, or `hotspots`) to print query diagnostics to stderr without changing normal stdout:
@@ -2775,6 +2780,11 @@ cdidx search "--open-reports" --path README.md --count  # `--` で始まる引
 cdidx search --query "--path" --path README.md          # オプションに見えるリテラルを検索
 ```
 
+literal FTS クエリは照合前に Unicode NFC へ正規化されます。すべての literal
+token が SQLite FTS5 unicode61 の 1000 文字 token 上限を超える場合、0 件
+JSON には `query_degraded_reason` と `tokens_dropped` が含まれます。index
+validation は長い連続 FTS token を `fts_token_too_long` として報告します。
+
 ### クエリのデバッグ
 
 任意の query command（`search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`find`、`excerpt`、`map`、`inspect`、`outline`、`status`、`validate`、`deps`、`impact`、`unused`、`hotspots`）に `--verbose` を付けると、通常の stdout を変えずに query 診断を stderr へ出力します:

diff --git a/changelog.d/unreleased/1458.fixed.md b/changelog.d/unreleased/1458.fixed.md
@@ -0,0 +1,20 @@
+---
+category: fixed
+issues:
+  - 1458
+  - 1606
+  - 1776
+affected:
+  - src/CodeIndex/Database/DbSearchReader.cs
+  - src/CodeIndex/Cli/QueryCommandRunner.cs
+  - src/CodeIndex/Indexer/Scanning/FileIndexer.cs
+  - USER_GUIDE.md
+---
+
+## English
+
+- **Search now reports degraded overlong-token FTS queries (#1458, #1606, #1776)** — zero-result JSON now distinguishes overlong-token-only literal FTS queries, literal search normalizes Unicode input to NFC, and validation reports FTS5 unicode61 tokens longer than 1000 characters as `fts_token_too_long`.
+
+## 日本語
+
+- **長すぎる token だけの FTS クエリに検索診断を追加しました (#1458, #1606, #1776)** — 0 件 JSON は長すぎる token のみの literal FTS クエリを区別し、literal 検索は Unicode 入力を NFC 正規化し、validation は FTS5 unicode61 の 1000 文字上限を超える token を `fts_token_too_long` として報告します。
diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs
@@ -344,6 +344,7 @@ internal sealed record VersionInfoJsonResult(
 [JsonSerializable(typeof(FileIssue))]
 [JsonSerializable(typeof(FileResult))]
 [JsonSerializable(typeof(FreshnessHintResult))]
+[JsonSerializable(typeof(FtsQueryDiagnostics))]
 [JsonSerializable(typeof(GroupedHotspotResult))]
 [JsonSerializable(typeof(GroupedSymbolHotspotJsonResult))]
 [JsonSerializable(typeof(ImpactAnalysisResult))]

diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs
@@ -409,10 +409,11 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions)
             if (options.CountOnly)
             {
                 var counts = reader.CountSearchResults(options.Query, options.Lang, options.RawFts, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, !options.NoDedup, options.Since, exact, options.Prefix, !options.NoVisibilityRank);
+                var queryDiagnostics = DbReader.AnalyzeFtsQuery(options.Query, options.RawFts, options.Prefix, options.Lang);
                 if (counts.Count == 0)
                 {
                     Console.WriteLine(options.Json
-                        ? BuildJsonZeroResultPayload(reader, jsonOptions, includeFiles: true, query: options.Query, queryOptions: options).ToJsonString(jsonOptions)
+                        ? BuildJsonZeroResultPayload(reader, jsonOptions, includeFiles: true, query: options.Query, ftsQueryDiagnostics: queryDiagnostics, queryOptions: options).ToJsonString(jsonOptions)
                         : "0");
                     return CommandExitCodes.Success;
                 }
@@ -424,6 +425,7 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions)
             }
 
             var results = reader.Search(options.Query, options.Limit, options.Lang, options.RawFts, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, !options.NoDedup, options.Since, exact, options.Prefix, !options.NoVisibilityRank);
+            var ftsQueryDiagnostics = DbReader.AnalyzeFtsQuery(options.Query, options.RawFts, options.Prefix, options.Lang);
             if (results.Count == 0)
             {
                 if (options.Json && TryWriteEmptyFormattedResult(options, jsonOptions))
@@ -440,7 +442,7 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions)
                     }
                     else
                     {
-                        Console.WriteLine(BuildJsonZeroResultPayload(reader, jsonOptions, resultsKey: "results", query: options.Query, queryOptions: options).ToJsonString(jsonOptions));
+                        Console.WriteLine(BuildJsonZeroResultPayload(reader, jsonOptions, resultsKey: "results", query: options.Query, ftsQueryDiagnostics: ftsQueryDiagnostics, queryOptions: options).ToJsonString(jsonOptions));
                         jsonDoneCount = 0;
                     }
                 }
@@ -4365,7 +4367,7 @@ private static JsonObject BuildUnusedResultsByBucketJson(IEnumerable<UnusedSymbo
     // `--kind replacement_chra` のようなタイプミスを did-you-mean で救うため、
     // FileIndexer.cs 内の `Kind = "..."` 代入と同期させる (#1582)。
     private static readonly string[] AllValidValidateKinds =
-        ["bom", "cr_only_line_endings", "file_too_large", "line_too_long", "mixed_line_endings", "mixed_line_endings_three_way", "non_utf8_likely", "null_byte", "replacement_char", "utf16_bom"];
+        ["bom", "cr_only_line_endings", "file_too_large", "fts_token_too_long", "line_too_long", "mixed_line_endings", "mixed_line_endings_three_way", "non_utf8_likely", "null_byte", "replacement_char", "utf16_bom"];
 
     public static int RunValidate(string[] cmdArgs, JsonSerializerOptions jsonOptions)
     {
@@ -6492,6 +6494,7 @@ private static JsonObject BuildJsonZeroResultPayload(
         string? resultsKey = null,
         string? query = null,
         ExactZeroHintResult? exactZeroHint = null,
+        FtsQueryDiagnostics? ftsQueryDiagnostics = null,
         bool includeFiles = false,
         bool? graphTableAvailable = null,
         bool? degraded = null,
@@ -6522,6 +6525,11 @@ private static JsonObject BuildJsonZeroResultPayload(
         }
         if (exactZeroHint != null)
             payload["exact_zero_hint"] = JsonSerializer.SerializeToNode(exactZeroHint, CliJsonSerializerContextFactory.Create(jsonOptions).ExactZeroHintResult);
+        if (ftsQueryDiagnostics is { HasDegradation: true })
+        {
+            payload["query_degraded_reason"] = ftsQueryDiagnostics.QueryDegradedReason;
+            payload["tokens_dropped"] = JsonSerializer.SerializeToNode(ftsQueryDiagnostics.TokensDropped.ToList(), CliJsonSerializerContextFactory.Create(jsonOptions).ListString);
+        }
         if (queryOptions != null)
             payload["query_context"] = BuildQueryContextJson(queryOptions, jsonOptions);
         extraFields?.Invoke(payload);

diff --git a/src/CodeIndex/Database/DbSearchReader.cs b/src/CodeIndex/Database/DbSearchReader.cs
@@ -1,3 +1,4 @@
+using System.Text;
 using Microsoft.Data.Sqlite;
 
 namespace CodeIndex.Database;
@@ -8,6 +9,8 @@ namespace CodeIndex.Database;
 /// </summary>
 public partial class DbReader
 {
+    internal const int FtsUnicode61MaxTokenLength = 1000;
+    internal const string AllTokensFilteredByLengthReason = "all_tokens_filtered_by_length";
     internal const int MaxRawFtsQueryLength = 2000;
     internal const int MaxRawFtsBooleanOperators = 64;
     internal const int MaxRawFtsNearOperators = 16;
@@ -40,6 +43,26 @@ internal static string SanitizeFtsQuery(string query, bool prefix)
         return string.Join(" ", tokens.Select(token => FormatFtsToken(token, prefix)));
     }
 
+    public static FtsQueryDiagnostics AnalyzeFtsQuery(string query, bool rawQuery = false, bool prefix = false, string? lang = null)
+    {
+        if (rawQuery || string.IsNullOrWhiteSpace(query))
+            return FtsQueryDiagnostics.None;
+
+        var normalizedQuery = NormalizeLiteralSearchQuery(query, NormalizeQueryLanguage(lang));
+        var tokens = normalizedQuery.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)
+            .Select(token => token.Length > 1 && token.EndsWith('*') ? token[..^1] : token)
+            .Where(token => token.Length > 0)
+            .ToArray();
+        if (tokens.Length == 0)
+            return FtsQueryDiagnostics.None;
+
+        var tooLong = tokens.Where(token => token.EnumerateRunes().Count() > FtsUnicode61MaxTokenLength).Distinct(StringComparer.Ordinal).ToArray();
+        if (tooLong.Length == tokens.Length)
+            return new FtsQueryDiagnostics(AllTokensFilteredByLengthReason, tooLong);
+
+        return FtsQueryDiagnostics.None;
+    }
+
     /// <summary>
     /// Build a single FTS5 phrase token. A trailing user-supplied `*` is preserved as a prefix
     /// shorthand; otherwise the token is quoted as a literal phrase. When <paramref name="prefix"/>
@@ -269,10 +292,13 @@ FROM fts_chunks
         return new QueryCountResult(count, fileCount);
     }
 
-    private static string NormalizeLiteralSearchQuery(string query, string? lang) =>
-        string.Equals(lang, "csharp", StringComparison.OrdinalIgnoreCase)
-            ? CSharpVerbatimNameNormalizer.Normalize(query)
-            : query;
+    private static string NormalizeLiteralSearchQuery(string query, string? lang)
+    {
+        var normalized = query.Normalize(NormalizationForm.FormC);
+        return string.Equals(lang, "csharp", StringComparison.OrdinalIgnoreCase)
+            ? CSharpVerbatimNameNormalizer.Normalize(normalized)
+            : normalized;
+    }
 
     internal static string ValidateRawFtsQuery(string query)
     {

diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs
@@ -3430,6 +3430,18 @@ public static List<FileIssue> ValidateContent(string relativePath, byte[] rawByt
             });
         }
 
+        var longFtsTokenLine = FindOversizeFtsTokenLine(content, CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength);
+        if (longFtsTokenLine > 0)
+        {
+            issues.Add(new FileIssue
+            {
+                Path = relativePath,
+                Kind = "fts_token_too_long",
+                Line = longFtsTokenLine,
+                Message = $"Line {longFtsTokenLine} contains an FTS5 unicode61 token longer than {CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength} characters; that token is not searchable through FTS",
+            });
+        }
+
         return issues;
     }
 
@@ -3598,6 +3610,43 @@ private static int FindOversizeLine(string content, int maxLineLength)
         return 0;
     }
 
+    private static int FindOversizeFtsTokenLine(string content, int maxTokenLength)
+    {
+        if (string.IsNullOrEmpty(content))
+            return 0;
+
+        var lineNumber = 1;
+        var tokenLength = 0;
+        foreach (var rune in content.EnumerateRunes())
+        {
+            if (rune.Value == '\n')
+            {
+                lineNumber++;
+                tokenLength = 0;
+                continue;
+            }
+
+            if (IsLikelyUnicode61TokenRune(rune))
+            {
+                tokenLength++;
+                if (tokenLength > maxTokenLength)
+                    return lineNumber;
+            }
+            else
+            {
+                tokenLength = 0;
+            }
+        }
+
+        return 0;
+    }
+
+    private static bool IsLikelyUnicode61TokenRune(Rune rune)
+        => rune.Value == '_'
+            || Rune.IsLetter(rune)
+            || Rune.IsDigit(rune)
+            || Rune.GetUnicodeCategory(rune) == UnicodeCategory.NonSpacingMark;
+
     /// <summary>
     /// Count U+FFFD replacement characters in decoded content.
     /// デコード済みcontent内のU+FFFD置換文字数を計上する。

diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs
@@ -146,6 +146,18 @@ private static void AddFreshnessHint(JsonObject payload, DbReader reader)
             payload["freshness_degraded_reason"] = freshness.FreshnessDegradedReason;
     }
 
+    private static void AddFtsQueryDiagnostics(JsonObject payload, FtsQueryDiagnostics diagnostics)
+    {
+        if (!diagnostics.HasDegradation)
+            return;
+
+        payload["query_degraded_reason"] = diagnostics.QueryDegradedReason;
+        var dropped = new JsonArray();
+        foreach (var token in diagnostics.TokensDropped)
+            dropped.Add(token);
+        payload["tokens_dropped"] = dropped;
+    }
+
     private static void AddExactZeroHint(JsonObject payload, ExactZeroHintResult? exactZeroHint)
     {
         if (exactZeroHint == null)
@@ -855,10 +867,13 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args)
                 payload["rawQuery"] = rawQuery;
                 payload["path"] = PathEcho(pathPatterns);
                 payload["excludeTests"] = excludeTests;
+                if (countResults.Count == 0)
+                    AddFtsQueryDiagnostics(payload, DbReader.AnalyzeFtsQuery(query, rawQuery, prefix, lang));
                 return CreateToolResult(id, $"Counted {countResults.Count} search result(s).", payload);
             }
 
             var results = reader.Search(query, FetchLimitForEnvelope(limit), lang, rawQuery, pathPatterns, excludePaths, excludeTests, deduplicate, since, exact, prefix);
+            var ftsDiagnostics = DbReader.AnalyzeFtsQuery(query, rawQuery, prefix, lang);
             var truncated = TrimToRequestedLimit(results, limit);
             if (results.Count == 0)
             {
@@ -872,6 +887,7 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args)
                     ["excludeTests"] = excludeTests,
                     ["results"] = new JsonArray()
                 };
+                AddFtsQueryDiagnostics(payload, ftsDiagnostics);
                 AddResultEnvelope(payload, 0, 0, truncated: false);
                 AddRecoveryHint(
                     payload,

diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs
@@ -22,6 +22,14 @@ public class SearchResult
 
 public readonly record struct QueryCountResult(int Count, int FileCount, bool IncludesSql = false);
 
+public sealed record FtsQueryDiagnostics(
+    [property: JsonPropertyName("query_degraded_reason")] string? QueryDegradedReason,
+    [property: JsonPropertyName("tokens_dropped")] IReadOnlyList<string> TokensDropped)
+{
+    public static FtsQueryDiagnostics None { get; } = new(null, []);
+    public bool HasDegradation => QueryDegradedReason != null;
+}
+
 public enum ReferenceRankMode
 {
     Weighted,

diff --git a/tests/CodeIndex.Tests/DbReaderTests.cs b/tests/CodeIndex.Tests/DbReaderTests.cs
@@ -128,6 +128,27 @@ public void CountSearchResults_NormalizesJavascriptLangSpelling()
         Assert.Equal(1, counts.FileCount);
     }
 
+    [Fact]
+    public void AnalyzeFtsQuery_AllTokensTooLong_ReturnsDegradedReason()
+    {
+        var query = new string('x', DbReader.FtsUnicode61MaxTokenLength + 1);
+
+        var diagnostics = DbReader.AnalyzeFtsQuery(query);
+
+        Assert.Equal(DbReader.AllTokensFilteredByLengthReason, diagnostics.QueryDegradedReason);
+        Assert.Equal([query], diagnostics.TokensDropped);
+    }
+
+    [Fact]
+    public void Search_ExplicitPrefixMatchesLatinDiacriticToken()
+    {
+        InsertIndexedFile("src/cafe.md", "markdown", "menu café_au_lait\n");
+
+        var results = _reader.Search("café*", lang: "markdown");
+
+        Assert.Contains(results, r => r.Path == "src/cafe.md");
+    }
+
     [Theory]
     [InlineData("rowid:authenticate", "rowid:")]
     [InlineData("title:authenticate", "title:")]

diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs
@@ -4646,6 +4646,20 @@ public void ValidateContent_NoOversizeLine_DoesNotEmitLineTooLongIssue()
         Assert.DoesNotContain(issues, i => i.Kind == "line_too_long");
     }
 
+    [Fact]
+    public void ValidateContent_OversizeFtsToken_EmitsFtsTokenTooLongIssue()
+    {
+        var token = new string('x', CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength + 1);
+        var content = "ok\nconst value = " + token + ";\n";
+        var raw = System.Text.Encoding.UTF8.GetBytes(content);
+
+        var issues = FileIndexer.ValidateContent("generated.js", raw, content);
+
+        var issue = Assert.Single(issues, i => i.Kind == "fts_token_too_long");
+        Assert.Equal(2, issue.Line);
+        Assert.Contains("not searchable through FTS", issue.Message);
+    }
+
     [Fact]
     public void SymbolExtractor_Extract_OversizeLine_ReturnsEmpty()
     {