diff --git a/changelog.d/unreleased/2528.fixed.md b/changelog.d/unreleased/2528.fixed.md new file mode 100644 index 0000000000..996d14658e --- /dev/null +++ b/changelog.d/unreleased/2528.fixed.md @@ -0,0 +1,19 @@ +--- +category: fixed +issues: + - 2528 +affected: + - src/CodeIndex/Database/DbContext.cs + - src/CodeIndex/Database/DbWriter.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs + - tests/CodeIndex.Tests/DatabaseTests.cs +--- + +## English + +- **Index refresh no longer stalls on large C# test files with many references (#2528)** — `cdidx index` now uses folded-name reverse-call indexes when refreshing mutual-recursion flags, avoiding expensive full reference-table scans after files such as `QueryCommandRunnerTests.cs`. + +## 日本語 + +- **参照数の多い大きな C# テストファイルで index 更新が止まったように見える問題を修正 (#2528)** — `cdidx index` は相互再帰フラグ更新時に折り畳み名の逆呼び出し index を使うようになり、`QueryCommandRunnerTests.cs` のようなファイル更新後の高コストな参照テーブル全体走査を避けます。 diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index 0f98cbf5c2..30dce2f0e0 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -1372,6 +1372,7 @@ value TEXT Execute("CREATE INDEX IF NOT EXISTS idx_symbols_visibility ON symbols(visibility)"); Execute("CREATE INDEX IF NOT EXISTS idx_symbol_refs_name_kind ON symbol_references(symbol_name, reference_kind)"); Execute("CREATE INDEX IF NOT EXISTS idx_symbol_refs_name_file ON symbol_references(symbol_name, file_id)"); + Execute("CREATE INDEX IF NOT EXISTS idx_symbol_refs_mutual_folded ON symbol_references(container_name_folded, symbol_name_folded, reference_kind, is_self_reference)"); Execute("CREATE INDEX IF NOT EXISTS idx_reference_lines_file_line ON reference_lines(file_id, line)"); Execute("CREATE INDEX IF NOT EXISTS idx_symbol_refs_reference_line ON symbol_references(reference_line_id)"); // Case-insensitive exact-match indexes for `references --exact` / `callers --exact` / `callees --exact` (#83). diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 8d093d4e70..3d07822807 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -993,7 +993,6 @@ public void InsertReferences(IReadOnlyList references) { if (references.Count == 0) return; - var referenceLineIds = new Dictionary<(long FileId, int Line), long>(); int rowsPerStatement = GetRowsPerInsertStatement(columnCount: 13); for (int i = 0; i < references.Count; i += rowsPerStatement) { @@ -1003,18 +1002,7 @@ public void InsertReferences(IReadOnlyList references) // symbol_references share one rollback boundary; without it a mid-chunk failure // under an outer transaction would orphan committed reference_lines (#1518). using var transaction = BeginTransaction(); - - using var lineCmd = _conn.CreateCommand(); - lineCmd.CommandText = @" - INSERT INTO reference_lines (file_id, line, context) - VALUES (@fid, @line, @context) - ON CONFLICT(file_id, line) DO UPDATE SET - context = excluded.context - RETURNING id"; - var pReferenceLineFid = lineCmd.Parameters.Add("@fid", SqliteType.Integer); - var pReferenceLineNumber = lineCmd.Parameters.Add("@line", SqliteType.Integer); - var pReferenceLineContext = lineCmd.Parameters.Add("@context", SqliteType.Text); - lineCmd.Prepare(); + var referenceLineIds = UpsertReferenceLines(references, i, end); using var cmd = _conn.CreateCommand(); var sql = new StringBuilder(); @@ -1030,15 +1018,7 @@ INSERT INTO symbol_references ( for (int j = i; j < end; j++) { var reference = references[j]; - var referenceLineKey = (reference.FileId, reference.Line); - if (!referenceLineIds.TryGetValue(referenceLineKey, out var referenceLineId)) - { - pReferenceLineFid.Value = reference.FileId; - pReferenceLineNumber.Value = reference.Line; - pReferenceLineContext.Value = reference.Context; - referenceLineId = (long)lineCmd.ExecuteScalar()!; - referenceLineIds[referenceLineKey] = referenceLineId; - } + var referenceLineId = referenceLineIds[(reference.FileId, reference.Line)]; if (j > i) sql.Append(", "); @@ -1072,6 +1052,74 @@ INSERT INTO symbol_references ( RefreshMutualRecursionFlags(); } + private Dictionary<(long FileId, int Line), long> UpsertReferenceLines(IReadOnlyList references, int start, int end) + { + var contextsByLine = new Dictionary<(long FileId, int Line), string>(); + for (int i = start; i < end; i++) + { + var reference = references[i]; + contextsByLine[(reference.FileId, reference.Line)] = reference.Context; + } + + var rows = contextsByLine.ToArray(); + int rowsPerStatement = GetRowsPerInsertStatement(columnCount: 3); + for (int i = 0; i < rows.Length; i += rowsPerStatement) + { + int batchEnd = Math.Min(i + rowsPerStatement, rows.Length); + using var cmd = _conn.CreateCommand(); + var sql = new StringBuilder(); + sql.Append("INSERT INTO reference_lines (file_id, line, context) VALUES "); + for (int j = i; j < batchEnd; j++) + { + if (j > i) + sql.Append(", "); + var suffix = j - i; + var ((fileId, line), context) = rows[j]; + sql.Append($"(@fid{suffix}, @line{suffix}, @context{suffix})"); + cmd.Parameters.Add($"@fid{suffix}", SqliteType.Integer).Value = fileId; + cmd.Parameters.Add($"@line{suffix}", SqliteType.Integer).Value = line; + cmd.Parameters.Add($"@context{suffix}", SqliteType.Text).Value = context; + } + + sql.Append(" ON CONFLICT(file_id, line) DO UPDATE SET context = excluded.context"); + cmd.CommandText = sql.ToString(); + cmd.ExecuteNonQuery(); + } + + var fileIds = contextsByLine.Keys.Select(key => key.FileId).Distinct().ToArray(); + var lineIds = new Dictionary<(long FileId, int Line), long>(); + int fileIdsPerStatement = GetRowsPerInsertStatement(columnCount: 1); + for (int i = 0; i < fileIds.Length; i += fileIdsPerStatement) + { + int fileEnd = Math.Min(i + fileIdsPerStatement, fileIds.Length); + using var cmd = _conn.CreateCommand(); + var parameters = new List(fileEnd - i); + for (int j = i; j < fileEnd; j++) + { + var parameterName = $"@fid{j - i}"; + parameters.Add(parameterName); + cmd.Parameters.Add(parameterName, SqliteType.Integer).Value = fileIds[j]; + } + + cmd.CommandText = $@" + SELECT id, file_id, line + FROM reference_lines + WHERE file_id IN ({string.Join(", ", parameters)})"; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + var id = reader.GetInt64(0); + var fileId = reader.GetInt64(1); + var line = reader.GetInt32(2); + var key = (fileId, line); + if (contextsByLine.ContainsKey(key)) + lineIds[key] = id; + } + } + + return lineIds; + } + private void RefreshMutualRecursionFlags() { using var cmd = _conn.CreateCommand(); @@ -1084,13 +1132,32 @@ AND r.container_name IS NOT NULL AND r.container_name <> '' AND r.symbol_name IS NOT NULL AND r.symbol_name <> '' - AND EXISTS ( - SELECT 1 - FROM symbol_references AS reverse - WHERE reverse.is_self_reference = 0 - AND reverse.reference_kind IN ('call', 'instantiate', 'subscribe', 'unsubscribe', 'razor_event_binding') - AND reverse.container_name = r.symbol_name COLLATE NOCASE - AND reverse.symbol_name = r.container_name COLLATE NOCASE + AND ( + ( + r.container_name_folded IS NOT NULL + AND r.container_name_folded <> '' + AND r.symbol_name_folded IS NOT NULL + AND r.symbol_name_folded <> '' + AND EXISTS ( + SELECT 1 + FROM symbol_references AS reverse + WHERE reverse.is_self_reference = 0 + AND reverse.reference_kind IN ('call', 'instantiate', 'subscribe', 'unsubscribe', 'razor_event_binding') + AND reverse.container_name_folded = r.symbol_name_folded + AND reverse.symbol_name_folded = r.container_name_folded + ) + ) + OR ( + (r.container_name_folded IS NULL OR r.symbol_name_folded IS NULL) + AND EXISTS ( + SELECT 1 + FROM symbol_references AS reverse + WHERE reverse.is_self_reference = 0 + AND reverse.reference_kind IN ('call', 'instantiate', 'subscribe', 'unsubscribe', 'razor_event_binding') + AND reverse.container_name = r.symbol_name COLLATE NOCASE + AND reverse.symbol_name = r.container_name COLLATE NOCASE + ) + ) ) THEN 1 ELSE 0 diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs index 7159ec22f9..912f39fb3a 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs @@ -756,7 +756,11 @@ private static bool HasCSharpQuoteRun(string line, int index, int requiredLength return GetCSharpQuoteRunLength(line, index) >= requiredLength; } - private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindCSharpBraceRange(string[] lines, int startIndex, int startColumn = 0) + private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindCSharpBraceRange( + string[] lines, + int startIndex, + int startColumn = 0, + bool linesAreSanitized = false) { int depth = 0; bool opened = false; @@ -793,9 +797,17 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindCSharpBra for (int i = startIndex; i < lines.Length; i++) { - var lexedLine = LexCSharpLine(lines[i], lexState); - lexState = lexedLine.EndState; - var sanitizedLine = lexedLine.SanitizedLine; + string sanitizedLine; + if (linesAreSanitized) + { + sanitizedLine = lines[i]; + } + else + { + var lexedLine = LexCSharpLine(lines[i], lexState); + lexState = lexedLine.EndState; + sanitizedLine = lexedLine.SanitizedLine; + } var scanLine = i == startIndex && startColumn > 0 && startColumn < sanitizedLine.Length ? sanitizedLine[startColumn..] : i == startIndex && startColumn >= sanitizedLine.Length diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index e93e36a2fd..9d6a9a35ef 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -2773,7 +2773,9 @@ public static List Extract(long fileId, string? lang, string conte ? (i + 1, null, null) : scalaBracelessClassEndLine.HasValue ? (scalaBracelessClassEndLine.Value + 1, null, null) - : ResolveRange(rangeLines, i, pattern.BodyStyle, lang, absoluteStartColumn); + : lang == "csharp" && pattern.BodyStyle == BodyStyle.Brace && csharpMatchLines != null + ? FindCSharpBraceRange(csharpMatchLines, i, absoluteStartColumn, linesAreSanitized: true) + : ResolveRange(rangeLines, i, pattern.BodyStyle, lang, absoluteStartColumn); if (fortranContinuationCandidate != null) endLine = Math.Max(endLine, fortranContinuationCandidate.Value.LastConsumedLineIndex + 1); var startLine = i + 1; @@ -10038,7 +10040,7 @@ private static bool IsCppTemplateSpecializationSymbol( private static void ExtractRustAssociatedTypeDefaultSymbols(long fileId, string[] lines, string[] structuralLines, List symbols) { var traits = symbols - .Where(symbol => symbol.Kind == "interface" + .Where(symbol => symbol.Kind is "interface" or "protocol" && symbol.BodyStartLine is > 0 && symbol.BodyEndLine is > 0) .OrderBy(symbol => symbol.StartLine) @@ -10070,7 +10072,7 @@ private static void ExtractRustAssociatedTypeDefaultSymbols(long fileId, string[ StartColumn = nameGroup.Index, EndLine = lineNumber, Signature = lines[lineIndex].Trim(), - ContainerKind = trait.Kind, + ContainerKind = "interface", ContainerName = trait.Name, ContainerQualifiedName = trait.ContainerQualifiedName, Visibility = match.Groups["visibility"].Success ? match.Groups["visibility"].Value : null, diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 0ae07dd82e..6aed2edd98 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -44,6 +44,60 @@ public void InitializeSchema_CreatesAllTables() Assert.Contains("fts_chunks", tables); } + [Fact] + public void InitializeSchema_CreatesFoldedMutualReferenceIndex() + { + using var cmd = _db.Connection.CreateCommand(); + cmd.CommandText = "SELECT name FROM sqlite_master WHERE type='index' AND name='idx_symbol_refs_mutual_folded'"; + + Assert.Equal("idx_symbol_refs_mutual_folded", (string?)cmd.ExecuteScalar()); + } + + [Fact] + public void InsertReferences_UsesFoldedNamesForMutualRecursion() + { + var fileId = _writer.UpsertFile(new FileRecord + { + Path = "src/app.cs", + Lang = "csharp", + Size = 100, + Lines = 4, + Modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc), + Checksum = "abc", + }); + + _writer.InsertReferences( + [ + new ReferenceRecord + { + FileId = fileId, + SymbolName = "Run", + ReferenceKind = "call", + Line = 1, + Column = 1, + Context = "Start();", + ContainerKind = "function", + ContainerName = "Start", + }, + new ReferenceRecord + { + FileId = fileId, + SymbolName = "Start", + ReferenceKind = "call", + Line = 2, + Column = 1, + Context = "Run();", + ContainerKind = "function", + ContainerName = "Run", + }, + ]); + + using var cmd = _db.Connection.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM symbol_references WHERE is_mutual_recursion = 1"; + + Assert.Equal(2L, (long)cmd.ExecuteScalar()!); + } + [Fact] public void OptimizeFts_ResetsIncrementalWriteCounterAndStampsTime() {