From 4183dddb273a80f7b62fbfa4484e02a653a1dafa Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 18:58:05 +0900 Subject: [PATCH 1/3] Fix file count query plans (#1827) --- changelog.d/unreleased/1827.fixed.md | 17 ++++++++++++ .../Database/DbReader.FilesStatus.cs | 10 +++++-- src/CodeIndex/Database/DbReader.cs | 27 ++++++++++++++----- tests/CodeIndex.Tests/DbReaderTests.cs | 18 +++++++++++++ 4 files changed, 63 insertions(+), 9 deletions(-) create mode 100644 changelog.d/unreleased/1827.fixed.md diff --git a/changelog.d/unreleased/1827.fixed.md b/changelog.d/unreleased/1827.fixed.md new file mode 100644 index 0000000000..69ccdfb08d --- /dev/null +++ b/changelog.d/unreleased/1827.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 1827 +affected: + - src/CodeIndex/Database/DbReader.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - tests/CodeIndex.Tests/DbReaderTests.cs +--- + +## English + +- **File metadata queries no longer run per-row count subqueries (#1827)** — `files` and exact file lookup now use grouped count joins for symbol and reference totals, avoiding repeated correlated scans on large indexes. + +## 日本語 + +- **ファイルメタデータ取得で行ごとの count サブクエリを実行しないようにしました (#1827)** — `files` と完全一致のファイル取得は symbol / reference 件数を集約 JOIN で取得し、大規模 index での相関 scan の繰り返しを避けます。 diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index f00a8de90a..4691e181db 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -408,12 +408,18 @@ FROM chunks c using var cmd = _conn.CreateCommand(); cmd.CommandText = $@" SELECT f.path, f.lang, f.size, f.lines, - (SELECT COUNT(*) FROM symbols WHERE file_id = f.id) AS symbol_count, - {ReferenceCountByFileSubquery} AS reference_count, + COALESCE(symbol_counts.symbol_count, 0) AS symbol_count, + {FileReferenceCountSql} AS reference_count, {GetFileColumnSql("checksum")} AS checksum, {GetFileColumnSql("modified")} AS modified, {GetFileColumnSql("indexed_at")} AS indexed_at FROM files f + LEFT JOIN ( + SELECT file_id, COUNT(*) AS symbol_count + FROM symbols + GROUP BY file_id + ) AS symbol_counts ON symbol_counts.file_id = f.id + {FileReferenceCountJoinSql} WHERE f.path = @path"; cmd.Parameters.AddWithValue("@path", path); diff --git a/src/CodeIndex/Database/DbReader.cs b/src/CodeIndex/Database/DbReader.cs index 16c965eef4..c42cf77a2f 100644 --- a/src/CodeIndex/Database/DbReader.cs +++ b/src/CodeIndex/Database/DbReader.cs @@ -878,12 +878,19 @@ private HashSet LoadHotspotFamilyReadyLanguages(SqliteConnection conn) } } - // Reference-count subquery that gracefully degrades to 0 when symbol_references is absent - // (legacy read-only DBs where TryMigrateForRead could not create the table). - // symbol_references が無いレガシー read-only DB では 0 にフォールバックする。 - private string ReferenceCountByFileSubquery => + private string FileReferenceCountJoinSql => _hasReferencesTable - ? "(SELECT COUNT(*) FROM symbol_references WHERE file_id = f.id)" + ? @" + LEFT JOIN ( + SELECT file_id, COUNT(*) AS reference_count + FROM symbol_references + GROUP BY file_id + ) AS reference_counts ON reference_counts.file_id = f.id" + : string.Empty; + + private string FileReferenceCountSql => + _hasReferencesTable + ? "COALESCE(reference_counts.reference_count, 0)" : "0"; // Script-style top-level code emits reference rows without a container symbol. @@ -1411,12 +1418,18 @@ public List ListFiles(string? query = null, int limit = 20, string? var sql = $@" SELECT f.path, f.lang, f.size, f.lines, - (SELECT COUNT(*) FROM symbols WHERE file_id = f.id) AS symbol_count, - {ReferenceCountByFileSubquery} AS reference_count, + COALESCE(symbol_counts.symbol_count, 0) AS symbol_count, + {FileReferenceCountSql} AS reference_count, {GetFileColumnSql("checksum")} AS checksum, {GetFileColumnSql("modified")} AS modified, {GetFileColumnSql("indexed_at")} AS indexed_at FROM files f + LEFT JOIN ( + SELECT file_id, COUNT(*) AS symbol_count + FROM symbols + GROUP BY file_id + ) AS symbol_counts ON symbol_counts.file_id = f.id + {FileReferenceCountJoinSql} WHERE 1=1"; if (query != null) diff --git a/tests/CodeIndex.Tests/DbReaderTests.cs b/tests/CodeIndex.Tests/DbReaderTests.cs index 31e3a9d96f..68eb340a26 100644 --- a/tests/CodeIndex.Tests/DbReaderTests.cs +++ b/tests/CodeIndex.Tests/DbReaderTests.cs @@ -250,6 +250,17 @@ GROUP BY r.symbol_name Assert.Contains("idx_symbol_refs_name_kind", planAfterAnalyze); } + [Fact] + public void FileCountHelpers_UseGroupedReferenceCounts() + { + var joinSql = GetPrivateStringProperty(_reader, "FileReferenceCountJoinSql"); + var countSql = GetPrivateStringProperty(_reader, "FileReferenceCountSql"); + + Assert.Contains("GROUP BY file_id", joinSql, StringComparison.Ordinal); + Assert.DoesNotContain("WHERE file_id = f.id", joinSql, StringComparison.OrdinalIgnoreCase); + Assert.Equal("COALESCE(reference_counts.reference_count, 0)", countSql); + } + [Theory] [InlineData("js")] [InlineData("JS")] @@ -418,6 +429,13 @@ private string ExplainQueryPlan(string sql) return plan.ToString(); } + private static string GetPrivateStringProperty(DbReader reader, string name) + { + var property = typeof(DbReader).GetProperty(name, BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(property); + return Assert.IsType(property!.GetValue(reader)); + } + private SqliteCommand CreateSearchReferencesCommandForSql(string query) { var method = typeof(DbReader).GetMethod( From 85764d8fc309ac35bc93e8c2390ff40212450f40 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 19:02:07 +0900 Subject: [PATCH 2/3] Avoid repeated symbol query normalization (#1954) --- changelog.d/unreleased/1954.fixed.md | 16 ++++++++++ src/CodeIndex/Database/DbSymbolReader.cs | 38 ++++++++++++++++++++---- tests/CodeIndex.Tests/DbReaderTests.cs | 15 ++++++++++ 3 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 changelog.d/unreleased/1954.fixed.md diff --git a/changelog.d/unreleased/1954.fixed.md b/changelog.d/unreleased/1954.fixed.md new file mode 100644 index 0000000000..f7b7c81732 --- /dev/null +++ b/changelog.d/unreleased/1954.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1954 +affected: + - src/CodeIndex/Database/DbSymbolReader.cs + - tests/CodeIndex.Tests/DbReaderTests.cs +--- + +## English + +- **Symbol query normalization now reuses normalized multi-query inputs (#1954)** — recursive symbol searches and counts skip a second normalization/materialization pass after the query list has already been normalized. + +## 日本語 + +- **複数 symbol query の正規化済み入力を再利用するようにしました (#1954)** — 再帰的な symbol search / count は、query list がすでに正規化済みの場合に二度目の正規化・materialize を行いません。 diff --git a/src/CodeIndex/Database/DbSymbolReader.cs b/src/CodeIndex/Database/DbSymbolReader.cs index 22319c74e7..2247bed88a 100644 --- a/src/CodeIndex/Database/DbSymbolReader.cs +++ b/src/CodeIndex/Database/DbSymbolReader.cs @@ -71,6 +71,34 @@ public partial class DbReader private const int UnusedPublicCandidateBudget = 2048; private const string SymbolLanguageFileIdFilter = " AND s.file_id IN (SELECT id FROM files WHERE lang = @lang)"; + private sealed class NormalizedSymbolSearchQueryList : List + { + public NormalizedSymbolSearchQueryList(IEnumerable queries) + : base(queries) + { + } + } + + private static IReadOnlyList? NormalizeSymbolSearchQueries(IReadOnlyList? queries, string? lang, bool exact) + { + if (queries == null) + return null; + if (queries is NormalizedSymbolSearchQueryList) + return queries; + + var seen = new HashSet(StringComparer.Ordinal); + var normalized = new List(); + foreach (var query in queries) + { + var value = NormalizeSymbolSearchQueryForSymbolSearch(query, lang, exact) ?? query ?? string.Empty; + if (value.Length == 0 || !seen.Add(value)) + continue; + normalized.Add(value); + } + + return new NormalizedSymbolSearchQueryList(normalized); + } + private void AppendVisibilityFilters(ref string sql, IReadOnlyList? visibilityFilters, IReadOnlyList? excludeVisibilityFilters) { if (visibilityFilters is { Count: > 0 }) @@ -204,7 +232,7 @@ public int CountSearchSymbols(string? query = null, int limit = 20, string? kind public bool AnySearchSymbols(IReadOnlyList? queries, string? kind = null, string? lang = null, IReadOnlyList? pathPatterns = null, IReadOnlyList? excludePathPatterns = null, bool excludeTests = false, DateTime? since = null, bool exact = false, IReadOnlyList? visibilityFilters = null, IReadOnlyList? excludeVisibilityFilters = null) { - var validQueries = queries?.Select(query => NormalizeSymbolSearchQueryForSymbolSearch(query, lang, exact) ?? query ?? string.Empty).Where(q => !string.IsNullOrEmpty(q)).Distinct().ToList(); + var validQueries = NormalizeSymbolSearchQueries(queries, lang, exact); if (validQueries == null || validQueries.Count == 0) return CountSearchSymbols(validQueries, 1, kind, lang, pathPatterns, excludePathPatterns, excludeTests, since, exact) > 0; @@ -222,7 +250,7 @@ public int CountSearchSymbols(IReadOnlyList? queries, int limit = 20, st if (HasVisibilityFilters(visibilityFilters, excludeVisibilityFilters)) return SearchSymbols(queries, limit, kind, lang, pathPatterns, excludePathPatterns, excludeTests, since, exact, visibilityFilters, excludeVisibilityFilters).Count; - var validQueries = queries?.Select(query => NormalizeSymbolSearchQueryForSymbolSearch(query, lang, exact) ?? query ?? string.Empty).Where(q => !string.IsNullOrEmpty(q)).Distinct().ToList(); + var validQueries = NormalizeSymbolSearchQueries(queries, lang, exact); if (validQueries != null && validQueries.Count > 1) return SearchSymbols(validQueries, limit, kind, lang, pathPatterns, excludePathPatterns, excludeTests, since, exact, visibilityFilters, excludeVisibilityFilters).Count; @@ -324,7 +352,7 @@ FROM symbols s JOIN files f ON s.file_id = f.id WHERE 1=1"; - var effectiveQueries = queries?.Select(query => NormalizeSymbolSearchQueryForSymbolSearch(query, lang, exact) ?? query ?? string.Empty).Where(q => !string.IsNullOrEmpty(q)).Distinct().ToList(); + var effectiveQueries = NormalizeSymbolSearchQueries(queries, lang, exact); if (effectiveQueries != null && effectiveQueries.Count > 0) { var orClauses = exact @@ -419,7 +447,7 @@ public List SearchSymbols(IReadOnlyList? queries, int limi // public `limit` contract stays "Max total results", not per-name. // 複数名指定: 名前ごとに独立検索して候補プールを確保した上で、round-robin で統合し、 // 最終的に全体で `limit` 件に収める。`limit` は従来どおり「合計の上限」。 - var validQueries = queries?.Select(query => NormalizeSymbolSearchQueryForSymbolSearch(query, lang, exact) ?? query ?? string.Empty).Where(q => !string.IsNullOrEmpty(q)).Distinct().ToList(); + var validQueries = NormalizeSymbolSearchQueries(queries, lang, exact); if (validQueries != null && validQueries.Count > 1) { var perName = new List>(validQueries.Count); @@ -467,7 +495,7 @@ FROM symbols s JOIN files f ON s.file_id = f.id WHERE 1=1"; - var effectiveQueries = queries?.Where(q => !string.IsNullOrEmpty(q)).Distinct().ToList(); + var effectiveQueries = validQueries; if (effectiveQueries != null && effectiveQueries.Count > 0) { // --exact: Unicode-aware equality when FoldReady (#86), else ASCII COLLATE NOCASE. diff --git a/tests/CodeIndex.Tests/DbReaderTests.cs b/tests/CodeIndex.Tests/DbReaderTests.cs index 68eb340a26..cd1f7336bf 100644 --- a/tests/CodeIndex.Tests/DbReaderTests.cs +++ b/tests/CodeIndex.Tests/DbReaderTests.cs @@ -261,6 +261,21 @@ public void FileCountHelpers_UseGroupedReferenceCounts() Assert.Equal("COALESCE(reference_counts.reference_count, 0)", countSql); } + [Fact] + public void NormalizeSymbolSearchQueries_SkipsAlreadyNormalizedInput() + { + var method = typeof(DbReader).GetMethod( + "NormalizeSymbolSearchQueries", + BindingFlags.Static | BindingFlags.NonPublic); + Assert.NotNull(method); + + var normalized = Assert.IsAssignableFrom>(method!.Invoke(null, [new[] { "module.exports.fetchData", "module.exports.fetchData" }, "javascript", false])); + var secondPass = Assert.IsAssignableFrom>(method.Invoke(null, [normalized, "javascript", false])); + + Assert.Same(normalized, secondPass); + Assert.Equal(["fetchData"], normalized); + } + [Theory] [InlineData("js")] [InlineData("JS")] From df87ccef71d994adc51c4d21f8ef50c3da4ebaff Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 19:24:56 +0900 Subject: [PATCH 3/3] Scope file count aggregation to candidates (#1827) --- .../Database/DbReader.FilesStatus.cs | 26 ++++++---- src/CodeIndex/Database/DbReader.cs | 50 ++++++++++++------- tests/CodeIndex.Tests/DbReaderTests.cs | 12 ++++- 3 files changed, 58 insertions(+), 30 deletions(-) diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index 4691e181db..6c967520cf 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -407,20 +407,28 @@ FROM chunks c { using var cmd = _conn.CreateCommand(); cmd.CommandText = $@" + WITH file_match AS ( + SELECT f.id, f.path, f.lang, f.size, f.lines, + {GetFileColumnSql("checksum")} AS checksum, + {GetFileColumnSql("modified")} AS modified, + {GetFileColumnSql("indexed_at")} AS indexed_at + FROM files f + WHERE f.path = @path + ) SELECT f.path, f.lang, f.size, f.lines, COALESCE(symbol_counts.symbol_count, 0) AS symbol_count, {FileReferenceCountSql} AS reference_count, - {GetFileColumnSql("checksum")} AS checksum, - {GetFileColumnSql("modified")} AS modified, - {GetFileColumnSql("indexed_at")} AS indexed_at - FROM files f + f.checksum, + f.modified, + f.indexed_at + FROM file_match f LEFT JOIN ( - SELECT file_id, COUNT(*) AS symbol_count - FROM symbols - GROUP BY file_id + SELECT s.file_id, COUNT(*) AS symbol_count + FROM symbols s + JOIN file_match file_set ON file_set.id = s.file_id + GROUP BY s.file_id ) AS symbol_counts ON symbol_counts.file_id = f.id - {FileReferenceCountJoinSql} - WHERE f.path = @path"; + {BuildFileReferenceCountJoinSql("file_match")}"; cmd.Parameters.AddWithValue("@path", path); using var reader = cmd.ExecuteTrackedReader(); diff --git a/src/CodeIndex/Database/DbReader.cs b/src/CodeIndex/Database/DbReader.cs index c42cf77a2f..194723af36 100644 --- a/src/CodeIndex/Database/DbReader.cs +++ b/src/CodeIndex/Database/DbReader.cs @@ -878,13 +878,14 @@ private HashSet LoadHotspotFamilyReadyLanguages(SqliteConnection conn) } } - private string FileReferenceCountJoinSql => + private string BuildFileReferenceCountJoinSql(string fileSetCteName) => _hasReferencesTable - ? @" + ? $@" LEFT JOIN ( - SELECT file_id, COUNT(*) AS reference_count - FROM symbol_references - GROUP BY file_id + SELECT r.file_id, COUNT(*) AS reference_count + FROM symbol_references r + JOIN {fileSetCteName} file_set ON file_set.id = r.file_id + GROUP BY r.file_id ) AS reference_counts ON reference_counts.file_id = f.id" : string.Empty; @@ -1417,20 +1418,13 @@ public List ListFiles(string? query = null, int limit = 20, string? using var cmd = _conn.CreateCommand(); var sql = $@" - SELECT f.path, f.lang, f.size, f.lines, - COALESCE(symbol_counts.symbol_count, 0) AS symbol_count, - {FileReferenceCountSql} AS reference_count, - {GetFileColumnSql("checksum")} AS checksum, - {GetFileColumnSql("modified")} AS modified, - {GetFileColumnSql("indexed_at")} AS indexed_at - FROM files f - LEFT JOIN ( - SELECT file_id, COUNT(*) AS symbol_count - FROM symbols - GROUP BY file_id - ) AS symbol_counts ON symbol_counts.file_id = f.id - {FileReferenceCountJoinSql} - WHERE 1=1"; + WITH file_page AS ( + SELECT f.id, f.path, f.lang, f.size, f.lines, + {GetFileColumnSql("checksum")} AS checksum, + {GetFileColumnSql("modified")} AS modified, + {GetFileColumnSql("indexed_at")} AS indexed_at + FROM files f + WHERE 1=1"; if (query != null) sql += " AND f.path LIKE @query ESCAPE '\\'"; @@ -1441,6 +1435,24 @@ GROUP BY file_id AppendPathFilters(ref sql, pathPatterns, excludePathPatterns, excludeTests); sql += $" ORDER BY {PathBucketOrder}, f.path LIMIT @limit"; + sql += $@" + ) + SELECT f.path, f.lang, f.size, f.lines, + COALESCE(symbol_counts.symbol_count, 0) AS symbol_count, + {FileReferenceCountSql} AS reference_count, + f.checksum, + f.modified, + f.indexed_at + FROM file_page f + LEFT JOIN ( + SELECT s.file_id, COUNT(*) AS symbol_count + FROM symbols s + JOIN file_page file_set ON file_set.id = s.file_id + GROUP BY s.file_id + ) AS symbol_counts ON symbol_counts.file_id = f.id + {BuildFileReferenceCountJoinSql("file_page")} + ORDER BY {PathBucketOrder}, f.path"; + cmd.CommandText = sql; if (query != null) cmd.Parameters.AddWithValue("@query", $"%{EscapeLikeQuery(query)}%"); diff --git a/tests/CodeIndex.Tests/DbReaderTests.cs b/tests/CodeIndex.Tests/DbReaderTests.cs index cd1f7336bf..6bccbba887 100644 --- a/tests/CodeIndex.Tests/DbReaderTests.cs +++ b/tests/CodeIndex.Tests/DbReaderTests.cs @@ -253,10 +253,11 @@ GROUP BY r.symbol_name [Fact] public void FileCountHelpers_UseGroupedReferenceCounts() { - var joinSql = GetPrivateStringProperty(_reader, "FileReferenceCountJoinSql"); + var joinSql = InvokePrivateStringMethod(_reader, "BuildFileReferenceCountJoinSql", "file_page"); var countSql = GetPrivateStringProperty(_reader, "FileReferenceCountSql"); - Assert.Contains("GROUP BY file_id", joinSql, StringComparison.Ordinal); + Assert.Contains("GROUP BY r.file_id", joinSql, StringComparison.Ordinal); + Assert.Contains("JOIN file_page file_set ON file_set.id = r.file_id", joinSql, StringComparison.Ordinal); Assert.DoesNotContain("WHERE file_id = f.id", joinSql, StringComparison.OrdinalIgnoreCase); Assert.Equal("COALESCE(reference_counts.reference_count, 0)", countSql); } @@ -451,6 +452,13 @@ private static string GetPrivateStringProperty(DbReader reader, string name) return Assert.IsType(property!.GetValue(reader)); } + private static string InvokePrivateStringMethod(DbReader reader, string name, params object[] args) + { + var method = typeof(DbReader).GetMethod(name, BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(method); + return Assert.IsType(method!.Invoke(reader, args)); + } + private SqliteCommand CreateSearchReferencesCommandForSql(string query) { var method = typeof(DbReader).GetMethod(