diff --git a/changelog.d/unreleased/2665.fixed.md b/changelog.d/unreleased/2665.fixed.md new file mode 100644 index 0000000000..4591b05684 --- /dev/null +++ b/changelog.d/unreleased/2665.fixed.md @@ -0,0 +1,20 @@ +--- +category: fixed +issues: + - 2665 + - 2680 + - 2683 +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Cli/CommandErrorCodes.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +--- + +## English + +- **Full index refresh now fails with a bounded extraction-stall diagnostic (#2665, #2680, #2683)** — full scans detect when extraction stops making progress and report the active file/phase instead of waiting indefinitely on a large source file. + +## 日本語 + +- **フルインデックス更新が抽出停止時に有界な診断で失敗するようになりました (#2665, #2680, #2683)** — フルスキャンで抽出の進捗が止まった場合、巨大なソースファイルで待ち続ける代わりに、処理中のファイルとフェーズを報告します。 diff --git a/src/CodeIndex/Cli/CommandErrorCodes.cs b/src/CodeIndex/Cli/CommandErrorCodes.cs index b0e636a358..2c33191a50 100644 --- a/src/CodeIndex/Cli/CommandErrorCodes.cs +++ b/src/CodeIndex/Cli/CommandErrorCodes.cs @@ -50,4 +50,7 @@ internal static class CommandErrorCodes /// The user interrupted the command with Ctrl-C / SIGINT. public const string Interrupted = "E012_INTERRUPTED"; + + /// Index extraction made no forward progress within the bounded stall timeout. + public const string IndexExtractionStalled = "E013_INDEX_EXTRACTION_STALLED"; } diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index daf07cfff1..dbd9c22f99 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -22,6 +22,7 @@ public static class IndexCommandRunner private const string SymbolKindFilterMetaKey = "index_symbol_kind_filter"; private const int ScanCheckpointVersion = 1; private const string ScanCheckpointFileName = "scan-checkpoint.json"; + private static readonly TimeSpan IndexExtractionStallTimeout = TimeSpan.FromMinutes(5); private sealed record ScanCheckpoint( int Version, @@ -30,6 +31,7 @@ private sealed record ScanCheckpoint( internal static Action? FullScanWritePhaseStartedForTesting { get; set; } internal static Action? FullScanExtractionSchedulingForTesting { get; set; } + internal static Func? IndexExtractionStallTimeoutForTesting { get; set; } internal static Action? HotspotFamilyUpdateRestampReadyForCommitForTesting { get; set; } internal static Func IsInputRedirectedForTesting { get; set; } = () => Console.IsInputRedirected; internal static Func ReadLineForTesting { get; set; } = Console.ReadLine; @@ -543,6 +545,10 @@ int WriteDryRunInterrupted() => WriteCommandError( { return WriteInterruptedResult(options.Json, jsonOptions, ex.FilesProcessed, ex.FilesTotal); } + catch (IndexExtractionStalledException ex) + { + return WriteExtractionStalledResult(options.Json, jsonOptions, ex); + } catch (Exception ex) when (IsDatabaseFilesystemError(ex)) { return WriteDatabaseFilesystemError(options.Json, jsonOptions, resolvedDbPath, ex); @@ -2112,7 +2118,14 @@ void ThrowIfUpdateCancelled() var chunks = ChunkSplitter.Split(fileId, content); writer.InsertChunks(chunks); currentUpdatePath = FormatIndexPhasePath(relPath, "symbols"); - var symbols = SymbolExtractor.Extract(fileId, record.Lang, content, absPath, Path.GetFullPath(options.ProjectPath!), cancellationToken); + var symbols = ExtractSymbolsWithStallTimeout( + fileId, + record.Lang, + content, + absPath, + Path.GetFullPath(options.ProjectPath!), + currentUpdatePath, + cancellationToken); SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(absPath, record.Lang)); var fileContext = new FileContext(projectRoot, record.Path, absPath, record.Lang); postExtractionHooks.OnSymbolsExtracted(fileContext, symbols); @@ -2143,6 +2156,10 @@ void ThrowIfUpdateCancelled() ThrowIfUpdateCancelled(); WriteUpdateVerboseStatus($" [OK ] {relPath} ({chunks.Count} chunks, {symbols.Count} symbols, {references.Count} refs)"); } + catch (IndexExtractionStalledException) + { + throw; + } catch (Exception ex) { if (ex is FileIndexer.BinaryFileSkippedException) @@ -2841,9 +2858,18 @@ internal static string FormatPerFileErrorLine(string label, string path, Excepti $" [{label}] {CollapseLineBreaks(path)}: {CollapseLineBreaks(message)}"; internal static string FormatIndexFileException(Exception ex) => - ex is RegexMatchTimeoutException timeoutException - ? RuntimeSafety.FormatRegexTimeout(timeoutException) - : ex.Message; + ex switch + { + RegexMatchTimeoutException timeoutException => RuntimeSafety.FormatRegexTimeout(timeoutException), + IndexExtractionStalledException stalledException => FormatExtractionStalledMessage(stalledException), + _ => ex.Message, + }; + + private static string FormatExtractionStalledMessage(IndexExtractionStalledException ex) + { + var pathSuffix = string.IsNullOrWhiteSpace(ex.ActivePath) ? string.Empty : $" Last active phase: {ex.ActivePath}."; + return $"Index extraction made no progress for {ConsoleUi.FormatDuration(ex.Timeout)}.{pathSuffix}"; + } internal static string FormatIndexPhasePath(string path, string phase) => $"{path} ({phase})"; @@ -2856,6 +2882,98 @@ internal static string FormatIndexPhasePath(string path, string phase) => return activeExtractionPhases.FirstOrDefault(static phase => !string.IsNullOrEmpty(phase)); } + internal static bool TryGetFullScanExtractionStallPath( + int filesProcessed, + int filesTotal, + TimeSpan timeout, + long lastProgressTimestamp, + string? currentFile, + IEnumerable activeExtractionPhases, + out string? activePath) + { + activePath = null; + if (filesTotal <= 0 || filesProcessed >= filesTotal || timeout <= TimeSpan.Zero) + return false; + + if (Stopwatch.GetElapsedTime(lastProgressTimestamp) < timeout) + return false; + + activePath = GetJsonIndexHeartbeatPath(currentFile, activeExtractionPhases); + return true; + } + + private static void ThrowIfFullScanExtractionStalled( + int filesProcessed, + int filesTotal, + TimeSpan timeout, + long lastProgressTimestamp, + string? currentFile, + ConcurrentDictionary activeExtractionPhases, + Action cancelStalledWork) + { + if (!TryGetFullScanExtractionStallPath( + filesProcessed, + filesTotal, + timeout, + lastProgressTimestamp, + currentFile, + activeExtractionPhases.OrderBy(static kvp => kvp.Key).Select(static kvp => kvp.Value), + out var activePath)) + { + return; + } + + cancelStalledWork(); + throw new IndexExtractionStalledException(filesProcessed, filesTotal, timeout, activePath); + } + + private static List ExtractSymbolsWithStallTimeout( + long fileId, + string? lang, + string content, + string filePath, + string projectRoot, + string phasePath, + CancellationToken cancellationToken) + { + var timeout = IndexExtractionStallTimeoutForTesting?.Invoke() ?? IndexExtractionStallTimeout; + if (timeout <= TimeSpan.Zero) + return SymbolExtractor.Extract(fileId, lang, content, filePath, projectRoot, cancellationToken); + + using var extractionCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var extractionToken = extractionCts.Token; + var task = Task.Run( + () => SymbolExtractor.Extract(fileId, lang, content, filePath, projectRoot, extractionToken), + CancellationToken.None); + try + { + if (task.Wait(timeout, cancellationToken)) + return task.GetAwaiter().GetResult(); + } + catch (AggregateException ex) when (ex.InnerExceptions.Count == 1) + { + throw ex.InnerExceptions[0]; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + + extractionCts.Cancel(); + try + { + task.Wait(TimeSpan.FromSeconds(1)); + } + catch (AggregateException ex) when (ex.InnerExceptions.All(inner => inner is OperationCanceledException or TaskCanceledException)) + { + } + catch (OperationCanceledException) + { + } + + throw new IndexExtractionStalledException(0, null, timeout, phasePath); + } + private static string CollapseLineBreaks(string value) { if (string.IsNullOrEmpty(value)) @@ -2902,6 +3020,19 @@ private static int WriteInterruptedResult(bool json, JsonSerializerOptions jsonO CommandErrorCodes.Interrupted); } + private static int WriteExtractionStalledResult(bool json, JsonSerializerOptions jsonOptions, IndexExtractionStalledException ex) + { + var totalSuffix = ex.FilesTotal is > 0 ? $" of {ex.FilesTotal.Value:N0}" : string.Empty; + var pathSuffix = string.IsNullOrWhiteSpace(ex.ActivePath) ? string.Empty : $" Last active phase: {ex.ActivePath}."; + return WriteCommandError( + json, + jsonOptions, + $"Index extraction made no progress for {ConsoleUi.FormatDuration(ex.Timeout)} ({ex.FilesProcessed:N0}{totalSuffix} files processed).{pathSuffix}", + CommandExitCodes.CancelledBySignal, + "Rerun with `--verbose` to inspect progress, lower `--parallelism`, or file a bug with the reported active phase.", + CommandErrorCodes.IndexExtractionStalled); + } + internal static bool HandleIndexCancelKeyPress(CancellationTokenSource cancellation, ref bool firstCancelHandled) { if (!firstCancelHandled && !cancellation.IsCancellationRequested) @@ -3529,13 +3660,15 @@ void StopJsonHeartbeat() } using var extractionResults = new BlockingCollection(Math.Max(1, extractionParallelism * 4)); + using var extractionStallCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var extractionCancellationToken = extractionStallCts.Token; var nextFileIndex = -1; var workers = Enumerable.Range(0, extractionParallelism) .Select(workerIndex => Task.Factory.StartNew(() => { while (true) { - cancellationToken.ThrowIfCancellationRequested(); + extractionCancellationToken.ThrowIfCancellationRequested(); var fileIndex = Interlocked.Increment(ref nextFileIndex); if (fileIndex >= files.Count) break; @@ -3545,7 +3678,7 @@ void StopJsonHeartbeat() { var relativeFilePath = FileIndexer.NormalizePathSeparators(Path.GetRelativePath(projectRoot, filePath)); activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(relativeFilePath, "reading"); - var (record, content, rawBytes, warning) = indexer.BuildRecordWithRawBytes(filePath, cancellationToken); + var (record, content, rawBytes, warning) = indexer.BuildRecordWithRawBytes(filePath, extractionCancellationToken); IReadOnlyList? chunks = null; IReadOnlyList? symbols = null; IReadOnlyList? references = null; @@ -3555,7 +3688,14 @@ void StopJsonHeartbeat() activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "chunking"); chunks = ChunkSplitter.Split(0, content); activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "symbols"); - symbols = SymbolExtractor.Extract(0, record.Lang, content, filePath, Path.GetFullPath(options.ProjectPath!), cancellationToken); + symbols = ExtractSymbolsWithStallTimeout( + 0, + record.Lang, + content, + filePath, + Path.GetFullPath(options.ProjectPath!), + activeJsonExtractionPhases[workerIndex], + extractionCancellationToken); SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(filePath, record.Lang)); activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "references"); references = ReferenceExtractor.Extract( @@ -3565,21 +3705,21 @@ void StopJsonHeartbeat() symbols, record.Path, record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - cancellationToken); + extractionCancellationToken); activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "validating"); issues = FileIndexer.ValidateContent(record.Path, rawBytes, content); } extractionResults.Add( FullScanFileWorkItem.Success(filePath, record, content, rawBytes, warning, chunks, symbols, references, issues), - cancellationToken); + extractionCancellationToken); } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + catch (OperationCanceledException) when (extractionCancellationToken.IsCancellationRequested) { throw; } catch (FileIndexer.BinaryFileSkippedException ex) { - extractionResults.Add(FullScanFileWorkItem.Skipped(filePath, ex.Message), cancellationToken); + extractionResults.Add(FullScanFileWorkItem.Skipped(filePath, ex.Message), extractionCancellationToken); } catch (FileIndexer.FileTooLargeSkippedException ex) { @@ -3593,18 +3733,18 @@ void StopJsonHeartbeat() }; extractionResults.Add( FullScanFileWorkItem.Success(filePath, record, string.Empty, [], ex.Message, [], [], [], [issue]), - cancellationToken); + extractionCancellationToken); } catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException) { var relativePath = FileIndexer.NormalizePathSeparators(Path.GetRelativePath(projectRoot, filePath)); extractionResults.Add( FullScanFileWorkItem.Skipped(filePath, $"{relativePath}: skipped because it was deleted during indexing."), - cancellationToken); + extractionCancellationToken); } catch (Exception ex) { - extractionResults.Add(FullScanFileWorkItem.Failure(filePath, ex), cancellationToken); + extractionResults.Add(FullScanFileWorkItem.Failure(filePath, ex), extractionCancellationToken); } finally { @@ -3623,14 +3763,30 @@ void StopJsonHeartbeat() TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + var extractionStallTimeout = IndexExtractionStallTimeoutForTesting?.Invoke() ?? IndexExtractionStallTimeout; + var lastExtractionProgressAt = Stopwatch.GetTimestamp(); while (!extractionResults.IsCompleted) { ThrowIfFullScanCancelled(processed, files.Count); if (!extractionResults.TryTake(out var item, millisecondsTimeout: 100)) + { + ThrowIfFullScanExtractionStalled( + processed, + files.Count, + extractionStallTimeout, + lastExtractionProgressAt, + currentJsonIndexFile, + activeJsonExtractionPhases, + extractionStallCts.Cancel); continue; + } + lastExtractionProgressAt = Stopwatch.GetTimestamp(); currentJsonIndexFile = FileIndexer.NormalizePathSeparators(Path.GetRelativePath(projectRoot, item.FilePath)); EnsureIndexingActivityVisible(); + if (item.Exception is IndexExtractionStalledException stalledException) + throw stalledException; + try { if (item.Exception != null) @@ -3736,7 +3892,14 @@ void StopJsonHeartbeat() writer.InsertChunks(chunks); currentJsonIndexFile = FormatIndexPhasePath(record.Path, "symbols"); var symbols = item.Symbols == null - ? SymbolExtractor.Extract(fileId, record.Lang, item.Content!, item.FilePath, Path.GetFullPath(options.ProjectPath!), cancellationToken) + ? ExtractSymbolsWithStallTimeout( + fileId, + record.Lang, + item.Content!, + item.FilePath, + Path.GetFullPath(options.ProjectPath!), + currentJsonIndexFile, + cancellationToken) : ReassignSymbolFileIds(item.Symbols, fileId); if (item.Symbols == null) SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(item.FilePath, record.Lang)); @@ -3769,6 +3932,10 @@ void StopJsonHeartbeat() WriteIndexVerboseStatus($" [OK ] {record.Path} ({chunks.Count} chunks, {symbols.Count} symbols, {references.Count} refs)"); } + catch (IndexExtractionStalledException) + { + throw; + } catch (Exception ex) { GlobalToolLog.Error($"index_file_failed path={CollapseLineBreaks(item.FilePath)}\n{GlobalToolLog.FormatExceptionChain(ex)}"); @@ -4767,6 +4934,23 @@ public IndexInterruptedException(int filesProcessed, int? filesTotal) public int? FilesTotal { get; } } + private sealed class IndexExtractionStalledException : Exception + { + public IndexExtractionStalledException(int filesProcessed, int? filesTotal, TimeSpan timeout, string? activePath) + : base("Index extraction stalled.") + { + FilesProcessed = filesProcessed; + FilesTotal = filesTotal; + Timeout = timeout; + ActivePath = activePath; + } + + public int FilesProcessed { get; } + public int? FilesTotal { get; } + public TimeSpan Timeout { get; } + public string? ActivePath { get; } + } + private sealed class CancelKeyPressRegistration(ConsoleCancelEventHandler handler) : IDisposable { public void Dispose() diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 3cc561af32..333658230b 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -3900,6 +3900,9 @@ public static List Extract(long fileId, string? lang, string conte if (restartPatternScanOffset >= 0) { + if (restartPatternScanOffset <= patternStartOffset) + break; + patternStartOffset = restartPatternScanOffset; continue; } diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 2ae20d0276..c98fd4d21b 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Text.Json; using System.Text.RegularExpressions; using System.Runtime.Versioning; @@ -418,6 +419,42 @@ public void ParseArgs_ForceFlag_SetsForce() Assert.Equal(Path.GetFullPath("."), options.ProjectPath); } + [Fact] + public void TryGetFullScanExtractionStallPath_ReportsActivePhaseAfterTimeout() + { + var staleTimestamp = Stopwatch.GetTimestamp() - Stopwatch.Frequency; + + var stalled = IndexCommandRunner.TryGetFullScanExtractionStallPath( + filesProcessed: 23, + filesTotal: 376, + timeout: TimeSpan.FromMilliseconds(1), + lastProgressTimestamp: staleTimestamp, + currentFile: null, + activeExtractionPhases: ["src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs (symbols)"], + out var activePath); + + Assert.True(stalled); + Assert.Equal("src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs (symbols)", activePath); + } + + [Fact] + public void TryGetFullScanExtractionStallPath_DoesNotReportWhenComplete() + { + var staleTimestamp = Stopwatch.GetTimestamp() - Stopwatch.Frequency; + + var stalled = IndexCommandRunner.TryGetFullScanExtractionStallPath( + filesProcessed: 376, + filesTotal: 376, + timeout: TimeSpan.FromMilliseconds(1), + lastProgressTimestamp: staleTimestamp, + currentFile: "src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs", + activeExtractionPhases: ["src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs (symbols)"], + out var activePath); + + Assert.False(stalled); + Assert.Null(activePath); + } + [Fact] public void ParseArgs_YesFlag_SetsYes() {