Fix file count and symbol query normalization performance

changelog.d/unreleased/1827.fixed.md

@@ -0,0 +1,17 @@
+---
+category: fixed
+issues:
+  - 1827
+affected:
+  - src/CodeIndex/Database/DbReader.cs
+  - src/CodeIndex/Database/DbReader.FilesStatus.cs
+  - tests/CodeIndex.Tests/DbReaderTests.cs
+---
+
+## English
+
+- **File metadata queries no longer run per-row count subqueries (#1827)** — `files` and exact file lookup now use grouped count joins for symbol and reference totals, avoiding repeated correlated scans on large indexes.
+
+## 日本語
+
+- **ファイルメタデータ取得で行ごとの count サブクエリを実行しないようにしました (#1827)** — `files` と完全一致のファイル取得は symbol / reference 件数を集約 JOIN で取得し、大規模 index での相関 scan の繰り返しを避けます。

changelog.d/unreleased/1954.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+issues:
+  - 1954
+affected:
+  - src/CodeIndex/Database/DbSymbolReader.cs
+  - tests/CodeIndex.Tests/DbReaderTests.cs
+---
+
+## English
+
+- **Symbol query normalization now reuses normalized multi-query inputs (#1954)** — recursive symbol searches and counts skip a second normalization/materialization pass after the query list has already been normalized.
+
+## 日本語
+
+- **複数 symbol query の正規化済み入力を再利用するようにしました (#1954)** — 再帰的な symbol search / count は、query list がすでに正規化済みの場合に二度目の正規化・materialize を行いません。

src/CodeIndex/Database/DbReader.FilesStatus.cs

@@ -407,14 +407,28 @@ FROM chunks c
     {
         using var cmd = _conn.CreateCommand();
         cmd.CommandText = $@"
+            WITH file_match AS (
+                SELECT f.id, f.path, f.lang, f.size, f.lines,
+                       {GetFileColumnSql("checksum")} AS checksum,
+                       {GetFileColumnSql("modified")} AS modified,
+                       {GetFileColumnSql("indexed_at")} AS indexed_at
+                FROM files f
+                WHERE f.path = @path
+            )
             SELECT f.path, f.lang, f.size, f.lines,
-                   (SELECT COUNT(*) FROM symbols WHERE file_id = f.id) AS symbol_count,
-                   {ReferenceCountByFileSubquery} AS reference_count,
-                   {GetFileColumnSql("checksum")} AS checksum,
-                   {GetFileColumnSql("modified")} AS modified,
-                   {GetFileColumnSql("indexed_at")} AS indexed_at
-            FROM files f
-            WHERE f.path = @path";
+                   COALESCE(symbol_counts.symbol_count, 0) AS symbol_count,
+                   {FileReferenceCountSql} AS reference_count,
+                   f.checksum,
+                   f.modified,
+                   f.indexed_at
+            FROM file_match f
+            LEFT JOIN (
+                SELECT s.file_id, COUNT(*) AS symbol_count
+                FROM symbols s
+                JOIN file_match file_set ON file_set.id = s.file_id
+                GROUP BY s.file_id
+            ) AS symbol_counts ON symbol_counts.file_id = f.id
+            {BuildFileReferenceCountJoinSql("file_match")}";
         cmd.Parameters.AddWithValue("@path", path);
 
         using var reader = cmd.ExecuteTrackedReader();

src/CodeIndex/Database/DbReader.cs

@@ -878,12 +878,20 @@ private HashSet<string> LoadHotspotFamilyReadyLanguages(SqliteConnection conn)
         }
     }
 
-    // Reference-count subquery that gracefully degrades to 0 when symbol_references is absent
-    // (legacy read-only DBs where TryMigrateForRead could not create the table).
-    // symbol_references が無いレガシー read-only DB では 0 にフォールバックする。
-    private string ReferenceCountByFileSubquery =>
+    private string BuildFileReferenceCountJoinSql(string fileSetCteName) =>
         _hasReferencesTable
-            ? "(SELECT COUNT(*) FROM symbol_references WHERE file_id = f.id)"
+            ? $@"
+            LEFT JOIN (
+                SELECT r.file_id, COUNT(*) AS reference_count
+                FROM symbol_references r
+                JOIN {fileSetCteName} file_set ON file_set.id = r.file_id
+                GROUP BY r.file_id
+            ) AS reference_counts ON reference_counts.file_id = f.id"
+            : string.Empty;
+
+    private string FileReferenceCountSql =>
+        _hasReferencesTable
+            ? "COALESCE(reference_counts.reference_count, 0)"
             : "0";
 
     // Script-style top-level code emits reference rows without a container symbol.
@@ -1410,14 +1418,13 @@ public List<FileResult> ListFiles(string? query = null, int limit = 20, string?
         using var cmd = _conn.CreateCommand();
 
         var sql = $@"
-            SELECT f.path, f.lang, f.size, f.lines,
-                   (SELECT COUNT(*) FROM symbols WHERE file_id = f.id) AS symbol_count,
-                   {ReferenceCountByFileSubquery} AS reference_count,
-                   {GetFileColumnSql("checksum")} AS checksum,
-                   {GetFileColumnSql("modified")} AS modified,
-                   {GetFileColumnSql("indexed_at")} AS indexed_at
-            FROM files f
-            WHERE 1=1";
+            WITH file_page AS (
+                SELECT f.id, f.path, f.lang, f.size, f.lines,
+                       {GetFileColumnSql("checksum")} AS checksum,
+                       {GetFileColumnSql("modified")} AS modified,
+                       {GetFileColumnSql("indexed_at")} AS indexed_at
+                FROM files f
+                WHERE 1=1";
 
         if (query != null)
             sql += " AND f.path LIKE @query ESCAPE '\\'";
@@ -1428,6 +1435,24 @@ FROM files f
         AppendPathFilters(ref sql, pathPatterns, excludePathPatterns, excludeTests);
         sql += $" ORDER BY {PathBucketOrder}, f.path LIMIT @limit";
 
+        sql += $@"
+            )
+            SELECT f.path, f.lang, f.size, f.lines,
+                   COALESCE(symbol_counts.symbol_count, 0) AS symbol_count,
+                   {FileReferenceCountSql} AS reference_count,
+                   f.checksum,
+                   f.modified,
+                   f.indexed_at
+            FROM file_page f
+            LEFT JOIN (
+                SELECT s.file_id, COUNT(*) AS symbol_count
+                FROM symbols s
+                JOIN file_page file_set ON file_set.id = s.file_id
+                GROUP BY s.file_id
+            ) AS symbol_counts ON symbol_counts.file_id = f.id
+            {BuildFileReferenceCountJoinSql("file_page")}
+            ORDER BY {PathBucketOrder}, f.path";
+
         cmd.CommandText = sql;
         if (query != null)
             cmd.Parameters.AddWithValue("@query", $"%{EscapeLikeQuery(query)}%");

src/CodeIndex/Database/DbSymbolReader.cs

@@ -71,6 +71,34 @@ public partial class DbReader
     private const int UnusedPublicCandidateBudget = 2048;
     private const string SymbolLanguageFileIdFilter = " AND s.file_id IN (SELECT id FROM files WHERE lang = @lang)";
 
+    private sealed class NormalizedSymbolSearchQueryList : List<string>
+    {
+        public NormalizedSymbolSearchQueryList(IEnumerable<string> queries)
+            : base(queries)
+        {
+        }
+    }
+
+    private static IReadOnlyList<string>? NormalizeSymbolSearchQueries(IReadOnlyList<string>? queries, string? lang, bool exact)
+    {
+        if (queries == null)
+            return null;
+        if (queries is NormalizedSymbolSearchQueryList)
+            return queries;
+
+        var seen = new HashSet<string>(StringComparer.Ordinal);
+        var normalized = new List<string>();
+        foreach (var query in queries)
+        {
+            var value = NormalizeSymbolSearchQueryForSymbolSearch(query, lang, exact) ?? query ?? string.Empty;
+            if (value.Length == 0 || !seen.Add(value))
+                continue;
+            normalized.Add(value);
+        }
+
+        return new NormalizedSymbolSearchQueryList(normalized);
+    }
+
     private void AppendVisibilityFilters(ref string sql, IReadOnlyList<string>? visibilityFilters, IReadOnlyList<string>? excludeVisibilityFilters)
     {
         if (visibilityFilters is { Count: > 0 })
@@ -204,7 +232,7 @@ public int CountSearchSymbols(string? query = null, int limit = 20, string? kind
 
     public bool AnySearchSymbols(IReadOnlyList<string>? queries, string? kind = null, string? lang = null, IReadOnlyList<string>? pathPatterns = null, IReadOnlyList<string>? excludePathPatterns = null, bool excludeTests = false, DateTime? since = null, bool exact = false, IReadOnlyList<string>? visibilityFilters = null, IReadOnlyList<string>? excludeVisibilityFilters = null)
     {
-        var validQueries = queries?.Select(query => NormalizeSymbolSearchQueryForSymbolSearch(query, lang, exact) ?? query ?? string.Empty).Where(q => !string.IsNullOrEmpty(q)).Distinct().ToList();
+        var validQueries = NormalizeSymbolSearchQueries(queries, lang, exact);
         if (validQueries == null || validQueries.Count == 0)
             return CountSearchSymbols(validQueries, 1, kind, lang, pathPatterns, excludePathPatterns, excludeTests, since, exact) > 0;
 
@@ -222,7 +250,7 @@ public int CountSearchSymbols(IReadOnlyList<string>? queries, int limit = 20, st
         if (HasVisibilityFilters(visibilityFilters, excludeVisibilityFilters))
             return SearchSymbols(queries, limit, kind, lang, pathPatterns, excludePathPatterns, excludeTests, since, exact, visibilityFilters, excludeVisibilityFilters).Count;
 
-        var validQueries = queries?.Select(query => NormalizeSymbolSearchQueryForSymbolSearch(query, lang, exact) ?? query ?? string.Empty).Where(q => !string.IsNullOrEmpty(q)).Distinct().ToList();
+        var validQueries = NormalizeSymbolSearchQueries(queries, lang, exact);
         if (validQueries != null && validQueries.Count > 1)
             return SearchSymbols(validQueries, limit, kind, lang, pathPatterns, excludePathPatterns, excludeTests, since, exact, visibilityFilters, excludeVisibilityFilters).Count;
 
@@ -324,7 +352,7 @@ FROM symbols s
                 JOIN files f ON s.file_id = f.id
                 WHERE 1=1";
 
-        var effectiveQueries = queries?.Select(query => NormalizeSymbolSearchQueryForSymbolSearch(query, lang, exact) ?? query ?? string.Empty).Where(q => !string.IsNullOrEmpty(q)).Distinct().ToList();
+        var effectiveQueries = NormalizeSymbolSearchQueries(queries, lang, exact);
         if (effectiveQueries != null && effectiveQueries.Count > 0)
         {
             var orClauses = exact
@@ -419,7 +447,7 @@ public List<SymbolResult> SearchSymbols(IReadOnlyList<string>? queries, int limi
         // public `limit` contract stays "Max total results", not per-name.
         // 複数名指定: 名前ごとに独立検索して候補プールを確保した上で、round-robin で統合し、
         // 最終的に全体で `limit` 件に収める。`limit` は従来どおり「合計の上限」。
-        var validQueries = queries?.Select(query => NormalizeSymbolSearchQueryForSymbolSearch(query, lang, exact) ?? query ?? string.Empty).Where(q => !string.IsNullOrEmpty(q)).Distinct().ToList();
+        var validQueries = NormalizeSymbolSearchQueries(queries, lang, exact);
         if (validQueries != null && validQueries.Count > 1)
         {
             var perName = new List<List<SymbolResult>>(validQueries.Count);
@@ -467,7 +495,7 @@ FROM symbols s
             JOIN files f ON s.file_id = f.id
             WHERE 1=1";
 
-        var effectiveQueries = queries?.Where(q => !string.IsNullOrEmpty(q)).Distinct().ToList();
+        var effectiveQueries = validQueries;
         if (effectiveQueries != null && effectiveQueries.Count > 0)
         {
             // --exact: Unicode-aware equality when FoldReady (#86), else ASCII COLLATE NOCASE.

tests/CodeIndex.Tests/DbReaderTests.cs

@@ -250,6 +250,33 @@ GROUP BY r.symbol_name
         Assert.Contains("idx_symbol_refs_name_kind", planAfterAnalyze);
     }
 
+    [Fact]
+    public void FileCountHelpers_UseGroupedReferenceCounts()
+    {
+        var joinSql = InvokePrivateStringMethod(_reader, "BuildFileReferenceCountJoinSql", "file_page");
+        var countSql = GetPrivateStringProperty(_reader, "FileReferenceCountSql");
+
+        Assert.Contains("GROUP BY r.file_id", joinSql, StringComparison.Ordinal);
+        Assert.Contains("JOIN file_page file_set ON file_set.id = r.file_id", joinSql, StringComparison.Ordinal);
+        Assert.DoesNotContain("WHERE file_id = f.id", joinSql, StringComparison.OrdinalIgnoreCase);
+        Assert.Equal("COALESCE(reference_counts.reference_count, 0)", countSql);
+    }
+
+    [Fact]
+    public void NormalizeSymbolSearchQueries_SkipsAlreadyNormalizedInput()
+    {
+        var method = typeof(DbReader).GetMethod(
+            "NormalizeSymbolSearchQueries",
+            BindingFlags.Static | BindingFlags.NonPublic);
+        Assert.NotNull(method);
+
+        var normalized = Assert.IsAssignableFrom<IReadOnlyList<string>>(method!.Invoke(null, [new[] { "module.exports.fetchData", "module.exports.fetchData" }, "javascript", false]));
+        var secondPass = Assert.IsAssignableFrom<IReadOnlyList<string>>(method.Invoke(null, [normalized, "javascript", false]));
+
+        Assert.Same(normalized, secondPass);
+        Assert.Equal(["fetchData"], normalized);
+    }
+
     [Theory]
     [InlineData("js")]
     [InlineData("JS")]
@@ -418,6 +445,20 @@ private string ExplainQueryPlan(string sql)
         return plan.ToString();
     }
 
+    private static string GetPrivateStringProperty(DbReader reader, string name)
+    {
+        var property = typeof(DbReader).GetProperty(name, BindingFlags.Instance | BindingFlags.NonPublic);
+        Assert.NotNull(property);
+        return Assert.IsType<string>(property!.GetValue(reader));
+    }
+
+    private static string InvokePrivateStringMethod(DbReader reader, string name, params object[] args)
+    {
+        var method = typeof(DbReader).GetMethod(name, BindingFlags.Instance | BindingFlags.NonPublic);
+        Assert.NotNull(method);
+        return Assert.IsType<string>(method!.Invoke(reader, args));
+    }
+
     private SqliteCommand CreateSearchReferencesCommandForSql(string query)
     {
         var method = typeof(DbReader).GetMethod(