Fix symbol extraction timeout isolation

changelog.d/unreleased/3048.fixed.md

@@ -0,0 +1,18 @@
+---
+category: fixed
+issues:
+  - 3048
+affected:
+  - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs
+  - src/CodeIndex/Cli/IndexCommandRunner.Update.cs
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs
+  - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs
+---
+
+## English
+
+- **Full-scan symbol extraction timeouts now stop isolated extractor work (#3048)** — full-scan symbol extraction runs through a reusable worker process and kills that worker on timeout, so a timed-out extractor cannot keep consuming CPU inside the indexing process after `cdidx index` reports the stall.
+
+## 日本語
+
+- **full-scan の symbol extraction timeout が分離された抽出処理を停止するようになりました (#3048)** — full-scan の symbol extraction は再利用可能な worker process 経由で実行され、timeout 時にはその worker を kill するため、`cdidx index` が stall を報告した後に timeout した extractor が indexing process 内で CPU を消費し続けることを防ぎます。

src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs

@@ -105,44 +105,22 @@ private static List<SymbolRecord> ExtractSymbolsWithStallTimeout(
         string filePath,
         string projectRoot,
         string phasePath,
+        SymbolExtractionWorkerClient worker,
         CancellationToken cancellationToken)
     {
         var timeout = IndexExtractionStallTimeoutForTesting?.Invoke() ?? IndexExtractionStallTimeout;
         if (timeout <= TimeSpan.Zero)
             return SymbolExtractor.Extract(fileId, lang, content, filePath, projectRoot, cancellationToken);
 
-        using var extractionCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
-        var extractionToken = extractionCts.Token;
-        var task = Task.Run(
-            () => SymbolExtractor.Extract(fileId, lang, content, filePath, projectRoot, extractionToken),
-            CancellationToken.None);
-        try
-        {
-            if (task.Wait(timeout, cancellationToken))
-                return task.GetAwaiter().GetResult();
-        }
-        catch (AggregateException ex) when (ex.InnerExceptions.Count == 1)
-        {
-            throw ex.InnerExceptions[0];
-        }
-        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
-        {
-            throw;
-        }
-
-        extractionCts.Cancel();
-        try
-        {
-            task.Wait(TimeSpan.FromSeconds(1));
-        }
-        catch (AggregateException ex) when (ex.InnerExceptions.All(inner => inner is OperationCanceledException or TaskCanceledException))
-        {
-        }
-        catch (OperationCanceledException)
-        {
-        }
+        cancellationToken.ThrowIfCancellationRequested();
+        var result = worker.Invoke(fileId, lang, content, filePath, projectRoot, timeout, cancellationToken);
+        cancellationToken.ThrowIfCancellationRequested();
+        if (result.TimedOut)
+            throw new IndexExtractionStalledException(0, null, timeout, phasePath);
+        if (!result.Success)
+            throw new InvalidOperationException(result.WorkerError ?? "isolated symbol extraction worker failed.");
 
-        throw new IndexExtractionStalledException(0, null, timeout, phasePath);
+        return result.Symbols ?? [];
     }
 
     private static string CollapseLineBreaks(string value)
@@ -892,11 +870,13 @@ void StopJsonHeartbeat()
 
             using var extractionResults = new BlockingCollection<FullScanFileWorkItem>(Math.Max(1, extractionParallelism * 4));
             using var extractionStallCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+            using var mainSymbolExtractionWorker = new SymbolExtractionWorkerClient();
             var extractionCancellationToken = extractionStallCts.Token;
             var nextFileIndex = -1;
             var workers = Enumerable.Range(0, extractionParallelism)
                 .Select(workerIndex => Task.Factory.StartNew(() =>
                 {
+                    using var workerSymbolExtractionWorker = new SymbolExtractionWorkerClient();
                     while (true)
                     {
                         extractionCancellationToken.ThrowIfCancellationRequested();
@@ -926,6 +906,7 @@ void StopJsonHeartbeat()
                                     filePath,
                                     Path.GetFullPath(options.ProjectPath!),
                                     activeJsonExtractionPhases[workerIndex],
+                                    workerSymbolExtractionWorker,
                                     extractionCancellationToken);
                                 if (symbols.Count > options.MaxSymbolsPerFile)
                                 {
@@ -1145,6 +1126,7 @@ void StopJsonHeartbeat()
                             item.FilePath,
                             Path.GetFullPath(options.ProjectPath!),
                             currentJsonIndexFile,
+                            mainSymbolExtractionWorker,
                             cancellationToken)
                         : ReassignSymbolFileIds(item.Symbols, fileId);
                     if (symbols.Count > options.MaxSymbolsPerFile)

src/CodeIndex/Cli/IndexCommandRunner.Update.cs

@@ -317,6 +317,7 @@ void ThrowIfUpdateCancelled()
             () => currentUpdatePath == null
                 ? $"{updated + removed + skipped:N0}/{targetPaths.Count:N0} files processed"
                 : $"{updated + removed + skipped:N0}/{targetPaths.Count:N0} files processed, current {currentUpdatePath}");
+        using var symbolExtractionWorker = new SymbolExtractionWorkerClient();
         try
         {
             foreach (var relPath in targetPaths)
@@ -667,6 +668,7 @@ void ThrowIfUpdateCancelled()
                         absPath,
                         Path.GetFullPath(options.ProjectPath!),
                         currentUpdatePath,
+                        symbolExtractionWorker,
                         cancellationToken);
                     if (symbols.Count > options.MaxSymbolsPerFile)
                     {

src/CodeIndex/Cli/ProgramRunner.cs

@@ -54,6 +54,9 @@ internal static int Run(
         Action? beforeDispatchForTesting = null,
         CancellationToken cancellationToken = default)
     {
+        if (SymbolExtractionWorker.TryRunCommand(args, Console.In, Console.Out, Console.Error, out var symbolWorkerExitCode))
+            return symbolWorkerExitCode;
+
         if (PostExtractionHookCallbackWorker.TryRunCommand(args, Console.In, Console.Out, Console.Error, out var hookWorkerExitCode))
             return hookWorkerExitCode;
 

src/CodeIndex/Indexer/Hooks/PostExtractionHookCallbackWorker.cs

@@ -4,6 +4,7 @@
 using System.Runtime.Versioning;
 using System.Text;
 using System.Text.Json;
+using System.Text.Json.Serialization;
 using CodeIndex.Models;
 
 namespace CodeIndex.Indexer.Hooks;
@@ -336,10 +337,7 @@ internal static class PostExtractionHookCallbackWorker
 {
     internal const string CommandName = "__cdidx-post-extraction-hook-callback";
     internal const int WorkerKillWaitMilliseconds = 5000;
-    internal static readonly JsonSerializerOptions JsonOptions = new()
-    {
-        PropertyNameCaseInsensitive = true,
-    };
+    internal static readonly JsonSerializerOptions JsonOptions = PostExtractionHookCallbackWorkerJsonContext.Default.Options;
 
     internal static bool TryRunCommand(
         string[] args,
@@ -363,26 +361,40 @@ internal static bool TryCreateStartInfo(
         out ProcessStartInfo startInfo,
         out string error)
     {
-        var runnerAssemblyPath = typeof(PostExtractionHookCallbackWorker).Assembly.Location;
+        return TryCreateStartInfo(
+            hook,
+            Environment.ProcessPath,
+            ResolveCurrentRunnerAssemblyPath(),
+            out startInfo,
+            out error);
+    }
+
+    internal static bool TryCreateStartInfo(
+        PostExtractionHookInfo hook,
+        string? currentProcessPath,
+        string? runnerAssemblyPath,
+        out ProcessStartInfo startInfo,
+        out string error)
+    {
+        startInfo = CreateStartInfo();
+        if (ShouldStartCurrentExecutable(currentProcessPath, runnerAssemblyPath))
+        {
+            startInfo.FileName = currentProcessPath!;
+            startInfo.ArgumentList.Add(CommandName);
+            startInfo.ArgumentList.Add(hook.AssemblyPath);
+            startInfo.ArgumentList.Add(hook.TypeName);
+            error = string.Empty;
+            return true;
+        }
+
         if (string.IsNullOrWhiteSpace(runnerAssemblyPath))
         {
             startInfo = new ProcessStartInfo();
             error = "could not resolve the cdidx assembly path for isolated hook callback execution.";
             return false;
         }
 
-        startInfo = new ProcessStartInfo
-        {
-            FileName = ResolveDotnetHostPath(),
-            UseShellExecute = false,
-            RedirectStandardInput = true,
-            RedirectStandardOutput = true,
-            RedirectStandardError = true,
-            StandardInputEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
-            StandardOutputEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
-            StandardErrorEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
-            CreateNoWindow = true,
-        };
+        startInfo.FileName = ResolveDotnetHostPath();
         startInfo.ArgumentList.Add(runnerAssemblyPath);
         startInfo.ArgumentList.Add(CommandName);
         startInfo.ArgumentList.Add(hook.AssemblyPath);
@@ -523,6 +535,48 @@ private static string ResolveDotnetHostPath()
         return "dotnet";
     }
 
+    private static ProcessStartInfo CreateStartInfo()
+        => new()
+        {
+            UseShellExecute = false,
+            RedirectStandardInput = true,
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+            StandardInputEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
+            StandardOutputEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
+            StandardErrorEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
+            CreateNoWindow = true,
+        };
+
+    private static bool ShouldStartCurrentExecutable(string? currentProcessPath, string? runnerAssemblyPath)
+    {
+        if (string.IsNullOrWhiteSpace(currentProcessPath) || IsDotnetHostPath(currentProcessPath))
+            return false;
+
+        var processName = Path.GetFileNameWithoutExtension(currentProcessPath);
+        var appName = typeof(PostExtractionHookCallbackWorker).Assembly.GetName().Name;
+        if (!string.IsNullOrWhiteSpace(appName)
+            && string.Equals(processName, appName, StringComparison.OrdinalIgnoreCase))
+        {
+            return true;
+        }
+
+        return string.IsNullOrWhiteSpace(runnerAssemblyPath);
+    }
+
+    private static string? ResolveCurrentRunnerAssemblyPath()
+    {
+        var assemblyName = typeof(PostExtractionHookCallbackWorker).Assembly.GetName().Name;
+        if (string.IsNullOrWhiteSpace(assemblyName))
+            return null;
+
+        var candidate = Path.Combine(AppContext.BaseDirectory, assemblyName + ".dll");
+        return File.Exists(candidate) ? candidate : null;
+    }
+
+    private static bool IsDotnetHostPath(string path)
+        => string.Equals(Path.GetFileNameWithoutExtension(path), "dotnet", StringComparison.OrdinalIgnoreCase);
+
     private static void ApplyCurrentRuntimeRollForward(ProcessStartInfo startInfo)
     {
         var targetMajor = GetRunnerTargetFrameworkMajor();
@@ -570,3 +624,8 @@ internal sealed record WorkerResponse(
         string? CallbackError,
         string? WorkerError);
 }
+
+[JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)]
+[JsonSerializable(typeof(PostExtractionHookCallbackWorker.WorkerRequest))]
+[JsonSerializable(typeof(PostExtractionHookCallbackWorker.WorkerResponse))]
+internal partial class PostExtractionHookCallbackWorkerJsonContext : JsonSerializerContext;