Fix index refresh stalls on large C# files

changelog.d/unreleased/2528.fixed.md

@@ -0,0 +1,19 @@
+---
+category: fixed
+issues:
+  - 2528
+affected:
+  - src/CodeIndex/Database/DbContext.cs
+  - src/CodeIndex/Database/DbWriter.cs
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs
+  - tests/CodeIndex.Tests/DatabaseTests.cs
+---
+
+## English
+
+- **Index refresh no longer stalls on large C# test files with many references (#2528)** — `cdidx index` now uses folded-name reverse-call indexes when refreshing mutual-recursion flags, avoiding expensive full reference-table scans after files such as `QueryCommandRunnerTests.cs`.
+
+## 日本語
+
+- **参照数の多い大きな C# テストファイルで index 更新が止まったように見える問題を修正 (#2528)** — `cdidx index` は相互再帰フラグ更新時に折り畳み名の逆呼び出し index を使うようになり、`QueryCommandRunnerTests.cs` のようなファイル更新後の高コストな参照テーブル全体走査を避けます。

src/CodeIndex/Database/DbContext.cs

@@ -1372,6 +1372,7 @@ value  TEXT
         Execute("CREATE INDEX IF NOT EXISTS idx_symbols_visibility      ON symbols(visibility)");
         Execute("CREATE INDEX IF NOT EXISTS idx_symbol_refs_name_kind   ON symbol_references(symbol_name, reference_kind)");
         Execute("CREATE INDEX IF NOT EXISTS idx_symbol_refs_name_file   ON symbol_references(symbol_name, file_id)");
+        Execute("CREATE INDEX IF NOT EXISTS idx_symbol_refs_mutual_folded ON symbol_references(container_name_folded, symbol_name_folded, reference_kind, is_self_reference)");
         Execute("CREATE INDEX IF NOT EXISTS idx_reference_lines_file_line ON reference_lines(file_id, line)");
         Execute("CREATE INDEX IF NOT EXISTS idx_symbol_refs_reference_line ON symbol_references(reference_line_id)");
         // Case-insensitive exact-match indexes for `references --exact` / `callers --exact` / `callees --exact` (#83).

src/CodeIndex/Database/DbWriter.cs

@@ -993,7 +993,6 @@ public void InsertReferences(IReadOnlyList<ReferenceRecord> references)
     {
         if (references.Count == 0) return;
 
-        var referenceLineIds = new Dictionary<(long FileId, int Line), long>();
         int rowsPerStatement = GetRowsPerInsertStatement(columnCount: 13);
         for (int i = 0; i < references.Count; i += rowsPerStatement)
         {
@@ -1003,18 +1002,7 @@ public void InsertReferences(IReadOnlyList<ReferenceRecord> references)
             // symbol_references share one rollback boundary; without it a mid-chunk failure
             // under an outer transaction would orphan committed reference_lines (#1518).
             using var transaction = BeginTransaction();
-
-            using var lineCmd = _conn.CreateCommand();
-            lineCmd.CommandText = @"
-                INSERT INTO reference_lines (file_id, line, context)
-                VALUES (@fid, @line, @context)
-                ON CONFLICT(file_id, line) DO UPDATE SET
-                    context = excluded.context
-                RETURNING id";
-            var pReferenceLineFid = lineCmd.Parameters.Add("@fid", SqliteType.Integer);
-            var pReferenceLineNumber = lineCmd.Parameters.Add("@line", SqliteType.Integer);
-            var pReferenceLineContext = lineCmd.Parameters.Add("@context", SqliteType.Text);
-            lineCmd.Prepare();
+            var referenceLineIds = UpsertReferenceLines(references, i, end);
 
             using var cmd = _conn.CreateCommand();
             var sql = new StringBuilder();
@@ -1030,15 +1018,7 @@ INSERT INTO symbol_references (
             for (int j = i; j < end; j++)
             {
                 var reference = references[j];
-                var referenceLineKey = (reference.FileId, reference.Line);
-                if (!referenceLineIds.TryGetValue(referenceLineKey, out var referenceLineId))
-                {
-                    pReferenceLineFid.Value = reference.FileId;
-                    pReferenceLineNumber.Value = reference.Line;
-                    pReferenceLineContext.Value = reference.Context;
-                    referenceLineId = (long)lineCmd.ExecuteScalar()!;
-                    referenceLineIds[referenceLineKey] = referenceLineId;
-                }
+                var referenceLineId = referenceLineIds[(reference.FileId, reference.Line)];
 
                 if (j > i)
                     sql.Append(", ");
@@ -1072,6 +1052,74 @@ INSERT INTO symbol_references (
         RefreshMutualRecursionFlags();
     }
 
+    private Dictionary<(long FileId, int Line), long> UpsertReferenceLines(IReadOnlyList<ReferenceRecord> references, int start, int end)
+    {
+        var contextsByLine = new Dictionary<(long FileId, int Line), string>();
+        for (int i = start; i < end; i++)
+        {
+            var reference = references[i];
+            contextsByLine[(reference.FileId, reference.Line)] = reference.Context;
+        }
+
+        var rows = contextsByLine.ToArray();
+        int rowsPerStatement = GetRowsPerInsertStatement(columnCount: 3);
+        for (int i = 0; i < rows.Length; i += rowsPerStatement)
+        {
+            int batchEnd = Math.Min(i + rowsPerStatement, rows.Length);
+            using var cmd = _conn.CreateCommand();
+            var sql = new StringBuilder();
+            sql.Append("INSERT INTO reference_lines (file_id, line, context) VALUES ");
+            for (int j = i; j < batchEnd; j++)
+            {
+                if (j > i)
+                    sql.Append(", ");
+                var suffix = j - i;
+                var ((fileId, line), context) = rows[j];
+                sql.Append($"(@fid{suffix}, @line{suffix}, @context{suffix})");
+                cmd.Parameters.Add($"@fid{suffix}", SqliteType.Integer).Value = fileId;
+                cmd.Parameters.Add($"@line{suffix}", SqliteType.Integer).Value = line;
+                cmd.Parameters.Add($"@context{suffix}", SqliteType.Text).Value = context;
+            }
+
+            sql.Append(" ON CONFLICT(file_id, line) DO UPDATE SET context = excluded.context");
+            cmd.CommandText = sql.ToString();
+            cmd.ExecuteNonQuery();
+        }
+
+        var fileIds = contextsByLine.Keys.Select(key => key.FileId).Distinct().ToArray();
+        var lineIds = new Dictionary<(long FileId, int Line), long>();
+        int fileIdsPerStatement = GetRowsPerInsertStatement(columnCount: 1);
+        for (int i = 0; i < fileIds.Length; i += fileIdsPerStatement)
+        {
+            int fileEnd = Math.Min(i + fileIdsPerStatement, fileIds.Length);
+            using var cmd = _conn.CreateCommand();
+            var parameters = new List<string>(fileEnd - i);
+            for (int j = i; j < fileEnd; j++)
+            {
+                var parameterName = $"@fid{j - i}";
+                parameters.Add(parameterName);
+                cmd.Parameters.Add(parameterName, SqliteType.Integer).Value = fileIds[j];
+            }
+
+            cmd.CommandText = $@"
+                SELECT id, file_id, line
+                FROM reference_lines
+                WHERE file_id IN ({string.Join(", ", parameters)})";
+            using var reader = cmd.ExecuteReader();
+            while (reader.Read())
+            {
+                var id = reader.GetInt64(0);
+                var fileId = reader.GetInt64(1);
+                var line = reader.GetInt32(2);
+                var key = (fileId, line);
+                if (contextsByLine.ContainsKey(key))
+                    lineIds[key] = id;
+            }
+        }
+
+        return lineIds;
+    }
+
     private void RefreshMutualRecursionFlags()
     {
         using var cmd = _conn.CreateCommand();
@@ -1084,13 +1132,32 @@ AND r.container_name IS NOT NULL
                  AND r.container_name <> ''
                  AND r.symbol_name IS NOT NULL
                  AND r.symbol_name <> ''
-                 AND EXISTS (
-                    SELECT 1
-                    FROM symbol_references AS reverse
-                    WHERE reverse.is_self_reference = 0
-                      AND reverse.reference_kind IN ('call', 'instantiate', 'subscribe', 'unsubscribe', 'razor_event_binding')
-                      AND reverse.container_name = r.symbol_name COLLATE NOCASE
-                      AND reverse.symbol_name = r.container_name COLLATE NOCASE
+                 AND (
+                    (
+                        r.container_name_folded IS NOT NULL
+                        AND r.container_name_folded <> ''
+                        AND r.symbol_name_folded IS NOT NULL
+                        AND r.symbol_name_folded <> ''
+                        AND EXISTS (
+                            SELECT 1
+                            FROM symbol_references AS reverse
+                            WHERE reverse.is_self_reference = 0
+                              AND reverse.reference_kind IN ('call', 'instantiate', 'subscribe', 'unsubscribe', 'razor_event_binding')
+                              AND reverse.container_name_folded = r.symbol_name_folded
+                              AND reverse.symbol_name_folded = r.container_name_folded
+                        )
+                    )
+                    OR (
+                        (r.container_name_folded IS NULL OR r.symbol_name_folded IS NULL)
+                        AND EXISTS (
+                            SELECT 1
+                            FROM symbol_references AS reverse
+                            WHERE reverse.is_self_reference = 0
+                              AND reverse.reference_kind IN ('call', 'instantiate', 'subscribe', 'unsubscribe', 'razor_event_binding')
+                              AND reverse.container_name = r.symbol_name COLLATE NOCASE
+                              AND reverse.symbol_name = r.container_name COLLATE NOCASE
+                        )
+                    )
                  )
                 THEN 1
                 ELSE 0

src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs

@@ -756,7 +756,11 @@ private static bool HasCSharpQuoteRun(string line, int index, int requiredLength
         return GetCSharpQuoteRunLength(line, index) >= requiredLength;
     }
 
-    private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindCSharpBraceRange(string[] lines, int startIndex, int startColumn = 0)
+    private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindCSharpBraceRange(
+        string[] lines,
+        int startIndex,
+        int startColumn = 0,
+        bool linesAreSanitized = false)
     {
         int depth = 0;
         bool opened = false;
@@ -793,9 +797,17 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindCSharpBra
 
         for (int i = startIndex; i < lines.Length; i++)
         {
-            var lexedLine = LexCSharpLine(lines[i], lexState);
-            lexState = lexedLine.EndState;
-            var sanitizedLine = lexedLine.SanitizedLine;
+            string sanitizedLine;
+            if (linesAreSanitized)
+            {
+                sanitizedLine = lines[i];
+            }
+            else
+            {
+                var lexedLine = LexCSharpLine(lines[i], lexState);
+                lexState = lexedLine.EndState;
+                sanitizedLine = lexedLine.SanitizedLine;
+            }
             var scanLine = i == startIndex && startColumn > 0 && startColumn < sanitizedLine.Length
                 ? sanitizedLine[startColumn..]
                 : i == startIndex && startColumn >= sanitizedLine.Length

src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs

@@ -2773,7 +2773,9 @@ public static List<SymbolRecord> Extract(long fileId, string? lang, string conte
                             ? (i + 1, null, null)
                             : scalaBracelessClassEndLine.HasValue
                                     ? (scalaBracelessClassEndLine.Value + 1, null, null)
-                                    : ResolveRange(rangeLines, i, pattern.BodyStyle, lang, absoluteStartColumn);
+                                    : lang == "csharp" && pattern.BodyStyle == BodyStyle.Brace && csharpMatchLines != null
+                                        ? FindCSharpBraceRange(csharpMatchLines, i, absoluteStartColumn, linesAreSanitized: true)
+                                        : ResolveRange(rangeLines, i, pattern.BodyStyle, lang, absoluteStartColumn);
                     if (fortranContinuationCandidate != null)
                         endLine = Math.Max(endLine, fortranContinuationCandidate.Value.LastConsumedLineIndex + 1);
                     var startLine = i + 1;
@@ -10038,7 +10040,7 @@ private static bool IsCppTemplateSpecializationSymbol(
     private static void ExtractRustAssociatedTypeDefaultSymbols(long fileId, string[] lines, string[] structuralLines, List<SymbolRecord> symbols)
     {
         var traits = symbols
-            .Where(symbol => symbol.Kind == "interface"
+            .Where(symbol => symbol.Kind is "interface" or "protocol"
                 && symbol.BodyStartLine is > 0
                 && symbol.BodyEndLine is > 0)
             .OrderBy(symbol => symbol.StartLine)
@@ -10070,7 +10072,7 @@ private static void ExtractRustAssociatedTypeDefaultSymbols(long fileId, string[
                             StartColumn = nameGroup.Index,
                             EndLine = lineNumber,
                             Signature = lines[lineIndex].Trim(),
-                            ContainerKind = trait.Kind,
+                            ContainerKind = "interface",
                             ContainerName = trait.Name,
                             ContainerQualifiedName = trait.ContainerQualifiedName,
                             Visibility = match.Groups["visibility"].Success ? match.Groups["visibility"].Value : null,

tests/CodeIndex.Tests/DatabaseTests.cs

@@ -44,6 +44,60 @@ public void InitializeSchema_CreatesAllTables()
         Assert.Contains("fts_chunks", tables);
     }
 
+    [Fact]
+    public void InitializeSchema_CreatesFoldedMutualReferenceIndex()
+    {
+        using var cmd = _db.Connection.CreateCommand();
+        cmd.CommandText = "SELECT name FROM sqlite_master WHERE type='index' AND name='idx_symbol_refs_mutual_folded'";
+
+        Assert.Equal("idx_symbol_refs_mutual_folded", (string?)cmd.ExecuteScalar());
+    }
+
+    [Fact]
+    public void InsertReferences_UsesFoldedNamesForMutualRecursion()
+    {
+        var fileId = _writer.UpsertFile(new FileRecord
+        {
+            Path = "src/app.cs",
+            Lang = "csharp",
+            Size = 100,
+            Lines = 4,
+            Modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc),
+            Checksum = "abc",
+        });
+
+        _writer.InsertReferences(
+        [
+            new ReferenceRecord
+            {
+                FileId = fileId,
+                SymbolName = "Run",
+                ReferenceKind = "call",
+                Line = 1,
+                Column = 1,
+                Context = "Start();",
+                ContainerKind = "function",
+                ContainerName = "Start",
+            },
+            new ReferenceRecord
+            {
+                FileId = fileId,
+                SymbolName = "Start",
+                ReferenceKind = "call",
+                Line = 2,
+                Column = 1,
+                Context = "Run();",
+                ContainerKind = "function",
+                ContainerName = "Run",
+            },
+        ]);
+
+        using var cmd = _db.Connection.CreateCommand();
+        cmd.CommandText = "SELECT COUNT(*) FROM symbol_references WHERE is_mutual_recursion = 1";
+
+        Assert.Equal(2L, (long)cmd.ExecuteScalar()!);
+    }
+
     [Fact]
     public void OptimizeFts_ResetsIncrementalWriteCounterAndStampsTime()
     {