Fix indexing liveness for slow extraction

changelog.d/unreleased/2627.fixed.md

@@ -0,0 +1,17 @@
+---
+category: fixed
+issues:
+  - 2627
+affected:
+  - src/CodeIndex/Cli/IndexCommandRunner.cs
+  - src/CodeIndex/Cli/RuntimeSafety.cs
+  - src/CodeIndex/Indexer/References/ReferenceExtractor.cs
+---
+
+## English
+
+- **Clearer long-running extraction progress during indexing (#2627)** — `cdidx index --json` liveness output now includes the current per-file phase, and C# pattern-reference regexes use bounded matching so unusually expensive source/test files no longer look like an undifferentiated one-path stall.
+
+## 日本語
+
+- **index 中の長時間抽出 progress を明確化しました (#2627)** — `cdidx index --json` の liveness output が現在のファイル内 phase を表示し、C# pattern-reference regex には bounded matching を適用したため、非常に重い source/test file が単なる 1 path の停止のように見え続ける状態を避けます。

src/CodeIndex/Cli/IndexCommandRunner.cs

@@ -1,6 +1,7 @@
 using System.Diagnostics;
 using System.Collections.Concurrent;
 using System.Runtime.InteropServices;
+using System.Text.RegularExpressions;
 using System.Text.Json;
 using CodeIndex.Database;
 using CodeIndex.Indexer;
@@ -38,6 +39,7 @@ public static int Run(string[] indexArgs, JsonSerializerOptions jsonOptions) =>
 
     internal static int Run(string[] indexArgs, JsonSerializerOptions jsonOptions, CancellationTokenSource? cancellationForTesting)
     {
+        RuntimeSafety.Configure();
         var options = ParseArgs(indexArgs);
         var jsonContext = CliJsonSerializerContextFactory.Create(jsonOptions);
         using var ownedCancellation = cancellationForTesting == null ? new CancellationTokenSource() : null;
@@ -2082,15 +2084,18 @@ void ThrowIfUpdateCancelled()
                     writer.PurgeStaleFilesSharingDirectoryAndStem(projectRoot, record.Path);
                 WriteProjectRootOnce();
                 var fileId = writer.UpsertFile(record);
+                currentUpdatePath = FormatIndexPhasePath(relPath, "chunking");
                 var chunks = ChunkSplitter.Split(fileId, content);
                 writer.InsertChunks(chunks);
+                currentUpdatePath = FormatIndexPhasePath(relPath, "symbols");
                 var symbols = SymbolExtractor.Extract(fileId, record.Lang, content, absPath, Path.GetFullPath(options.ProjectPath!));
                 SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(absPath, record.Lang));
                 var fileContext = new FileContext(projectRoot, record.Path, absPath, record.Lang);
                 postExtractionHooks.OnSymbolsExtracted(fileContext, symbols);
                 symbolsDroppedByKindFilter += options.SymbolKindFilter.Apply(symbols);
                 FileIndexer.ValidateSymbolLineRanges(record, symbols);
                 writer.InsertSymbols(symbols);
+                currentUpdatePath = FormatIndexPhasePath(relPath, "references");
                 var references = ReferenceExtractor.Extract(
                     fileId,
                     record.Lang,
@@ -2101,8 +2106,10 @@ void ThrowIfUpdateCancelled()
                 postExtractionHooks.OnReferencesExtracted(fileContext, references);
                 writer.InsertReferences(references);
                 // Validate content for encoding issues / エンコーディング問題を検証
+                currentUpdatePath = FormatIndexPhasePath(relPath, "validating");
                 var issues = FileIndexer.ValidateContent(record.Path, rawBytes, content);
                 writer.InsertIssues(fileId, issues);
+                currentUpdatePath = FormatIndexPhasePath(relPath, "committing");
                 writer.ClearBatchInProgress();
                 txn.Commit();
 
@@ -2183,11 +2190,12 @@ void ThrowIfUpdateCancelled()
                 GlobalToolLog.Error($"index_update_file_failed path={CollapseLineBreaks(relPath)}\n{GlobalToolLog.FormatExceptionChain(ex)}");
 
                 errors++;
-                errorList.Add(new CliJsonMessage(relPath, ex.Message));
+                var errorMessage = FormatIndexFileException(ex);
+                errorList.Add(new CliJsonMessage(relPath, errorMessage));
                 if (!options.Json)
                 {
                     PauseUpdateSpinnerForConsoleWrite();
-                    Console.Error.WriteLine(FormatPerFileErrorLine("ERR ", relPath, ex));
+                    Console.Error.WriteLine(FormatPerFileErrorLine("ERR ", relPath, ex, errorMessage));
                     ResumeUpdateSpinnerAfterConsoleWrite();
                 }
             }
@@ -2767,7 +2775,26 @@ private static void StampWorkspacePathCaseSensitivity(DbWriter writer, string pr
     // 複数行メッセージが疑似スタック行を注入できないようにする。詳細診断は
     // `cdidx report` / `CDIDX_DEBUG` で取得する (#1578)。
     internal static string FormatPerFileErrorLine(string label, string path, Exception ex) =>
-        $"  [{label}] {CollapseLineBreaks(path)}: {CollapseLineBreaks(ex.Message)}";
+        FormatPerFileErrorLine(label, path, ex, FormatIndexFileException(ex));
+
+    internal static string FormatPerFileErrorLine(string label, string path, Exception ex, string message) =>
+        $"  [{label}] {CollapseLineBreaks(path)}: {CollapseLineBreaks(message)}";
+
+    internal static string FormatIndexFileException(Exception ex) =>
+        ex is RegexMatchTimeoutException timeoutException
+            ? RuntimeSafety.FormatRegexTimeout(timeoutException)
+            : ex.Message;
+
+    internal static string FormatIndexPhasePath(string path, string phase) =>
+        $"{path} ({phase})";
+
+    internal static string? GetJsonIndexHeartbeatPath(string? currentFile, IEnumerable<string> activeExtractionPhases)
+    {
+        if (!string.IsNullOrEmpty(currentFile))
+            return currentFile;
+
+        return activeExtractionPhases.FirstOrDefault(static phase => !string.IsNullOrEmpty(phase));
+    }
 
     private static string CollapseLineBreaks(string value)
     {
@@ -3241,13 +3268,14 @@ void ThrowIfFullScanCancelled(int filesProcessed, int? filesTotal)
         int processed = 0, skipped = 0, warnings = warningList.Count, errors = errorList.Count;
         var symbolsDroppedByKindFilter = 0;
 
-          var interactiveIndexSpinner = !options.Json && !options.Quiet && ConsoleUi.ShouldUseInteractiveConsole();
+        var interactiveIndexSpinner = !options.Json && !options.Quiet && ConsoleUi.ShouldUseInteractiveConsole();
         var redirectedIndexingMessagePrinted = false;
         var indexProgressVisible = false;
         var reusedHotspotFamilyLanguages = new HashSet<string>(StringComparer.Ordinal);
         var skippedSymbolExtractorLanguages = new HashSet<string>(StringComparer.Ordinal);
         var lastJsonProgressAt = Stopwatch.GetTimestamp();
         string? currentJsonIndexFile = null;
+        var activeJsonExtractionPhases = new ConcurrentDictionary<int, string>();
         CancellationTokenSource? jsonHeartbeatCts = null;
         Task? jsonHeartbeatTask = null;
         using var postExtractionHooks = PostExtractionHookRunner.DiscoverDefault();
@@ -3362,7 +3390,9 @@ void StartJsonHeartbeatIfNeeded()
                     if (token.IsCancellationRequested)
                         break;
 
-                    var file = currentJsonIndexFile;
+                    var file = GetJsonIndexHeartbeatPath(
+                        currentJsonIndexFile,
+                        activeJsonExtractionPhases.OrderBy(static kvp => kvp.Key).Select(static kvp => kvp.Value));
                     var fileSuffix = string.IsNullOrEmpty(file) ? string.Empty : $": {file}";
                     Console.Error.WriteLine($"cdidx: still indexing {processed:N0}/{files.Count:N0} file(s){fileSuffix}...");
                 }
@@ -3429,7 +3459,7 @@ void StopJsonHeartbeat()
             using var extractionResults = new BlockingCollection<FullScanFileWorkItem>(Math.Max(1, extractionParallelism * 4));
             var nextFileIndex = -1;
             var workers = Enumerable.Range(0, extractionParallelism)
-                .Select(_ => Task.Factory.StartNew(() =>
+                .Select(workerIndex => Task.Factory.StartNew(() =>
                 {
                     while (true)
                     {
@@ -3441,23 +3471,29 @@ void StopJsonHeartbeat()
                         var filePath = files[fileIndex];
                         try
                         {
+                            var relativeFilePath = FileIndexer.NormalizePathSeparators(Path.GetRelativePath(projectRoot, filePath));
+                            activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(relativeFilePath, "reading");
                             var (record, content, rawBytes, warning) = indexer.BuildRecordWithRawBytes(filePath);
                             IReadOnlyList<ChunkRecord>? chunks = null;
                             IReadOnlyList<SymbolRecord>? symbols = null;
                             IReadOnlyList<ReferenceRecord>? references = null;
                             IReadOnlyList<FileIssue>? issues = null;
                             if (parallelizeExtraction)
                             {
+                                activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "chunking");
                                 chunks = ChunkSplitter.Split(0, content);
+                                activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "symbols");
                                 symbols = SymbolExtractor.Extract(0, record.Lang, content, filePath, Path.GetFullPath(options.ProjectPath!));
                                 SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(filePath, record.Lang));
+                                activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "references");
                                 references = ReferenceExtractor.Extract(
                                     0,
                                     record.Lang,
                                     content,
                                     symbols,
                                     record.Path,
                                     record.Lang == "csharp" ? csharpWorkspace.Symbols : null);
+                                activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "validating");
                                 issues = FileIndexer.ValidateContent(record.Path, rawBytes, content);
                             }
                             extractionResults.Add(
@@ -3483,6 +3519,10 @@ void StopJsonHeartbeat()
                         {
                             extractionResults.Add(FullScanFileWorkItem.Failure(filePath, ex), cancellationToken);
                         }
+                        finally
+                        {
+                            activeJsonExtractionPhases.TryRemove(workerIndex, out _);
+                        }
                     }
                 }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Default))
                 .ToArray();
@@ -3602,10 +3642,12 @@ void StopJsonHeartbeat()
                     using var txn = writer.BeginTransaction();
                     writer.PurgeStaleFilesSharingChecksum(projectRoot, record.Path, record.Checksum);
                     var fileId = writer.UpsertFile(record);
+                    currentJsonIndexFile = FormatIndexPhasePath(record.Path, "chunking");
                     var chunks = item.Chunks == null
                         ? ChunkSplitter.Split(fileId, item.Content!)
                         : ReassignChunkFileIds(item.Chunks, fileId);
                     writer.InsertChunks(chunks);
+                    currentJsonIndexFile = FormatIndexPhasePath(record.Path, "symbols");
                     var symbols = item.Symbols == null
                         ? SymbolExtractor.Extract(fileId, record.Lang, item.Content!, item.FilePath, Path.GetFullPath(options.ProjectPath!))
                         : ReassignSymbolFileIds(item.Symbols, fileId);
@@ -3618,6 +3660,7 @@ void StopJsonHeartbeat()
                     symbols = (IReadOnlyList<SymbolRecord>)mutableSymbols;
                     FileIndexer.ValidateSymbolLineRanges(record, symbols);
                     writer.InsertSymbols(symbols);
+                    currentJsonIndexFile = FormatIndexPhasePath(record.Path, "references");
                     var references = item.References == null
                         ? ReferenceExtractor.Extract(
                             fileId,
@@ -3629,8 +3672,10 @@ void StopJsonHeartbeat()
                         : ReassignReferenceFileIds(item.References, fileId);
                     postExtractionHooks.OnReferencesExtracted(fileContext, AsMutableList(references));
                     writer.InsertReferences(references);
+                    currentJsonIndexFile = FormatIndexPhasePath(record.Path, "validating");
                     var issues = item.Issues ?? FileIndexer.ValidateContent(record.Path, item.RawBytes!, item.Content!);
                     writer.InsertIssues(fileId, issues);
+                    currentJsonIndexFile = FormatIndexPhasePath(record.Path, "committing");
                     WriteProjectRootOnce();
                     txn.Commit();
 
@@ -3640,12 +3685,13 @@ void StopJsonHeartbeat()
                 {
                     GlobalToolLog.Error($"index_file_failed path={CollapseLineBreaks(item.FilePath)}\n{GlobalToolLog.FormatExceptionChain(ex)}");
                     errors++;
-                    errorList.Add(new CliJsonMessage(item.FilePath, ex.Message));
+                    var errorMessage = FormatIndexFileException(ex);
+                    errorList.Add(new CliJsonMessage(item.FilePath, errorMessage));
                     if (!options.Json)
                     {
                         PauseIndexSpinnerForConsoleWrite();
                         ConsoleUi.ClearProgressLine();
-                        Console.Error.WriteLine(FormatPerFileErrorLine("ERR ", item.FilePath, ex));
+                        Console.Error.WriteLine(FormatPerFileErrorLine("ERR ", item.FilePath, ex, errorMessage));
                         ResumeIndexSpinnerAfterConsoleWrite();
                     }
                 }

src/CodeIndex/Cli/RuntimeSafety.cs

@@ -0,0 +1,17 @@
+using System.Text.RegularExpressions;
+
+namespace CodeIndex.Cli;
+
+internal static class RuntimeSafety
+{
+    internal static readonly TimeSpan RegexMatchTimeout = TimeSpan.FromSeconds(2);
+
+    public static void Configure()
+    {
+        AppDomain.CurrentDomain.SetData("REGEX_DEFAULT_MATCH_TIMEOUT", RegexMatchTimeout);
+    }
+
+    public static string FormatRegexTimeout(RegexMatchTimeoutException ex)
+        => $"Regex extraction timed out after {ex.MatchTimeout.TotalSeconds:0.###}s while indexing this file. "
+           + "The file was skipped so indexing can finish; please report the file or reduce the pathological pattern input.";
+}

src/CodeIndex/Indexer/References/ReferenceExtractor.cs

@@ -11,6 +11,7 @@ namespace CodeIndex.Indexer;
 /// </summary>
 public static partial class ReferenceExtractor
 {
+    private static readonly TimeSpan ExtractionRegexTimeout = TimeSpan.FromSeconds(2);
     // THREAD-SAFETY: Reference extraction is stateless per call. Shared Regex instances and
     // lookup tables are initialized once and then read concurrently; language-specific state
     // must be created per extraction call (for example via CreateState helpers) rather than
@@ -572,7 +573,8 @@ private static bool IsFunctionLikeSymbolKind(string kind)
     // `is` / `is not` / `as` の型位置 (`o is Base`, `o is not Base`, `o as Base`)。
     private static readonly Regex CSharpIsAsTypeTestRegex = new(
         $@"(?<![\w$])(?:is\s+(?:not\s+)?|as\s+)(?<type>{CSharpTypeExpressionPattern})",
-        RegexOptions.Compiled);
+        RegexOptions.Compiled,
+        ExtractionRegexTimeout);
     internal static readonly Regex CSharpTrailingIsAsTypePatternIntroRegex = new(
         @"(?<![\w$])(?:is(?:\s+not)?|as)\s*$",
         RegexOptions.Compiled);
@@ -597,7 +599,8 @@ private static bool IsFunctionLikeSymbolKind(string kind)
         RegexOptions.Compiled);
     private static readonly Regex CSharpTypeExpressionAtCursorRegex = new(
         $@"\G(?<type>{CSharpTypeExpressionPattern})",
-        RegexOptions.Compiled);
+        RegexOptions.Compiled,
+        ExtractionRegexTimeout);
     // C# XML-doc cross-reference (`<see cref="Base.Do"/>`, `<seealso cref="ILogger.Log"/>`).
     // C# XML doc の `<see cref="Base.Do"/>` / `<seealso cref="ILogger.Log"/>`。
     private static readonly Regex CSharpDocCrefRegex = new(

src/CodeIndex/Program.cs

@@ -6,4 +6,5 @@
 // Windows のコンソールは既定で OEM コードページを使用するため、Unicode 文字が文字化けします。
 Console.OutputEncoding = Encoding.UTF8;
 ConsoleUi.EnsureConsoleWritersSynchronized();
+RuntimeSafety.Configure();
 return ProgramRunner.Run(args);

tests/CodeIndex.Tests/IndexCommandRunnerTests.cs

@@ -1,4 +1,5 @@
 using System.Text.Json;
+using System.Text.RegularExpressions;
 using System.Runtime.Versioning;
 using System.Runtime.InteropServices;
 using CodeIndex.Cli;
@@ -48,6 +49,46 @@ public void ParseArgs_HelpFlagSetsShowHelp()
         Assert.Null(options.ProjectPath);
     }
 
+    [Fact]
+    public void FormatIndexFileException_RegexTimeout_UsesBoundedExtractionMessage()
+    {
+        var ex = new RegexMatchTimeoutException("raw-sensitive-content", "raw-sensitive-pattern", TimeSpan.FromSeconds(2));
+
+        var message = IndexCommandRunner.FormatIndexFileException(ex);
+
+        Assert.Contains("Regex extraction timed out after 2s", message);
+        Assert.Contains("file was skipped", message);
+        Assert.DoesNotContain("raw-sensitive", message);
+    }
+
+    [Fact]
+    public void FormatIndexPhasePath_AppendsPhaseSuffixForJsonLiveness()
+    {
+        var message = IndexCommandRunner.FormatIndexPhasePath("src/App.cs", "references");
+
+        Assert.Equal("src/App.cs (references)", message);
+    }
+
+    [Fact]
+    public void GetJsonIndexHeartbeatPath_UsesWorkerPhaseWhenMainThreadIsIdle()
+    {
+        var message = IndexCommandRunner.GetJsonIndexHeartbeatPath(
+            currentFile: null,
+            activeExtractionPhases: ["src/App.cs (references)"]);
+
+        Assert.Equal("src/App.cs (references)", message);
+    }
+
+    [Fact]
+    public void GetJsonIndexHeartbeatPath_PrefersMainThreadPhaseWhenCommittingResults()
+    {
+        var message = IndexCommandRunner.GetJsonIndexHeartbeatPath(
+            "src/App.cs (committing)",
+            ["src/Other.cs (references)"]);
+
+        Assert.Equal("src/App.cs (committing)", message);
+    }
+
     [Fact]
     public void Run_NullByteFile_SkipsWithoutPersistingPartialRows()
     {