diff --git a/changelog.d/unreleased/3048.fixed.md b/changelog.d/unreleased/3048.fixed.md new file mode 100644 index 0000000000..c28cbd761e --- /dev/null +++ b/changelog.d/unreleased/3048.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3048 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/IndexCommandRunner.Update.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +--- + +## English + +- **Full-scan symbol extraction timeouts now stop isolated extractor work (#3048)** — full-scan symbol extraction runs through a reusable worker process and kills that worker on timeout, so a timed-out extractor cannot keep consuming CPU inside the indexing process after `cdidx index` reports the stall. + +## 日本語 + +- **full-scan の symbol extraction timeout が分離された抽出処理を停止するようになりました (#3048)** — full-scan の symbol extraction は再利用可能な worker process 経由で実行され、timeout 時にはその worker を kill するため、`cdidx index` が stall を報告した後に timeout した extractor が indexing process 内で CPU を消費し続けることを防ぎます。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 1a4f084f21..21b8a41e7e 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -105,44 +105,22 @@ private static List ExtractSymbolsWithStallTimeout( string filePath, string projectRoot, string phasePath, + SymbolExtractionWorkerClient worker, CancellationToken cancellationToken) { var timeout = IndexExtractionStallTimeoutForTesting?.Invoke() ?? IndexExtractionStallTimeout; if (timeout <= TimeSpan.Zero) return SymbolExtractor.Extract(fileId, lang, content, filePath, projectRoot, cancellationToken); - using var extractionCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - var extractionToken = extractionCts.Token; - var task = Task.Run( - () => SymbolExtractor.Extract(fileId, lang, content, filePath, projectRoot, extractionToken), - CancellationToken.None); - try - { - if (task.Wait(timeout, cancellationToken)) - return task.GetAwaiter().GetResult(); - } - catch (AggregateException ex) when (ex.InnerExceptions.Count == 1) - { - throw ex.InnerExceptions[0]; - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - - extractionCts.Cancel(); - try - { - task.Wait(TimeSpan.FromSeconds(1)); - } - catch (AggregateException ex) when (ex.InnerExceptions.All(inner => inner is OperationCanceledException or TaskCanceledException)) - { - } - catch (OperationCanceledException) - { - } + cancellationToken.ThrowIfCancellationRequested(); + var result = worker.Invoke(fileId, lang, content, filePath, projectRoot, timeout, cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + if (result.TimedOut) + throw new IndexExtractionStalledException(0, null, timeout, phasePath); + if (!result.Success) + throw new InvalidOperationException(result.WorkerError ?? "isolated symbol extraction worker failed."); - throw new IndexExtractionStalledException(0, null, timeout, phasePath); + return result.Symbols ?? []; } private static string CollapseLineBreaks(string value) @@ -892,11 +870,13 @@ void StopJsonHeartbeat() using var extractionResults = new BlockingCollection(Math.Max(1, extractionParallelism * 4)); using var extractionStallCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + using var mainSymbolExtractionWorker = new SymbolExtractionWorkerClient(); var extractionCancellationToken = extractionStallCts.Token; var nextFileIndex = -1; var workers = Enumerable.Range(0, extractionParallelism) .Select(workerIndex => Task.Factory.StartNew(() => { + using var workerSymbolExtractionWorker = new SymbolExtractionWorkerClient(); while (true) { extractionCancellationToken.ThrowIfCancellationRequested(); @@ -926,6 +906,7 @@ void StopJsonHeartbeat() filePath, Path.GetFullPath(options.ProjectPath!), activeJsonExtractionPhases[workerIndex], + workerSymbolExtractionWorker, extractionCancellationToken); if (symbols.Count > options.MaxSymbolsPerFile) { @@ -1145,6 +1126,7 @@ void StopJsonHeartbeat() item.FilePath, Path.GetFullPath(options.ProjectPath!), currentJsonIndexFile, + mainSymbolExtractionWorker, cancellationToken) : ReassignSymbolFileIds(item.Symbols, fileId); if (symbols.Count > options.MaxSymbolsPerFile) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index 9b34d485bb..0aba188406 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -317,6 +317,7 @@ void ThrowIfUpdateCancelled() () => currentUpdatePath == null ? $"{updated + removed + skipped:N0}/{targetPaths.Count:N0} files processed" : $"{updated + removed + skipped:N0}/{targetPaths.Count:N0} files processed, current {currentUpdatePath}"); + using var symbolExtractionWorker = new SymbolExtractionWorkerClient(); try { foreach (var relPath in targetPaths) @@ -667,6 +668,7 @@ void ThrowIfUpdateCancelled() absPath, Path.GetFullPath(options.ProjectPath!), currentUpdatePath, + symbolExtractionWorker, cancellationToken); if (symbols.Count > options.MaxSymbolsPerFile) { diff --git a/src/CodeIndex/Cli/ProgramRunner.cs b/src/CodeIndex/Cli/ProgramRunner.cs index b4d1dbf78b..d76f4986b0 100644 --- a/src/CodeIndex/Cli/ProgramRunner.cs +++ b/src/CodeIndex/Cli/ProgramRunner.cs @@ -54,6 +54,9 @@ internal static int Run( Action? beforeDispatchForTesting = null, CancellationToken cancellationToken = default) { + if (SymbolExtractionWorker.TryRunCommand(args, Console.In, Console.Out, Console.Error, out var symbolWorkerExitCode)) + return symbolWorkerExitCode; + if (PostExtractionHookCallbackWorker.TryRunCommand(args, Console.In, Console.Out, Console.Error, out var hookWorkerExitCode)) return hookWorkerExitCode; diff --git a/src/CodeIndex/Indexer/Hooks/PostExtractionHookCallbackWorker.cs b/src/CodeIndex/Indexer/Hooks/PostExtractionHookCallbackWorker.cs index 4d012a1672..c91f381947 100644 --- a/src/CodeIndex/Indexer/Hooks/PostExtractionHookCallbackWorker.cs +++ b/src/CodeIndex/Indexer/Hooks/PostExtractionHookCallbackWorker.cs @@ -4,6 +4,7 @@ using System.Runtime.Versioning; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using CodeIndex.Models; namespace CodeIndex.Indexer.Hooks; @@ -336,10 +337,7 @@ internal static class PostExtractionHookCallbackWorker { internal const string CommandName = "__cdidx-post-extraction-hook-callback"; internal const int WorkerKillWaitMilliseconds = 5000; - internal static readonly JsonSerializerOptions JsonOptions = new() - { - PropertyNameCaseInsensitive = true, - }; + internal static readonly JsonSerializerOptions JsonOptions = PostExtractionHookCallbackWorkerJsonContext.Default.Options; internal static bool TryRunCommand( string[] args, @@ -363,7 +361,32 @@ internal static bool TryCreateStartInfo( out ProcessStartInfo startInfo, out string error) { - var runnerAssemblyPath = typeof(PostExtractionHookCallbackWorker).Assembly.Location; + return TryCreateStartInfo( + hook, + Environment.ProcessPath, + ResolveCurrentRunnerAssemblyPath(), + out startInfo, + out error); + } + + internal static bool TryCreateStartInfo( + PostExtractionHookInfo hook, + string? currentProcessPath, + string? runnerAssemblyPath, + out ProcessStartInfo startInfo, + out string error) + { + startInfo = CreateStartInfo(); + if (ShouldStartCurrentExecutable(currentProcessPath, runnerAssemblyPath)) + { + startInfo.FileName = currentProcessPath!; + startInfo.ArgumentList.Add(CommandName); + startInfo.ArgumentList.Add(hook.AssemblyPath); + startInfo.ArgumentList.Add(hook.TypeName); + error = string.Empty; + return true; + } + if (string.IsNullOrWhiteSpace(runnerAssemblyPath)) { startInfo = new ProcessStartInfo(); @@ -371,18 +394,7 @@ internal static bool TryCreateStartInfo( return false; } - startInfo = new ProcessStartInfo - { - FileName = ResolveDotnetHostPath(), - UseShellExecute = false, - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true, - StandardInputEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), - StandardOutputEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), - StandardErrorEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), - CreateNoWindow = true, - }; + startInfo.FileName = ResolveDotnetHostPath(); startInfo.ArgumentList.Add(runnerAssemblyPath); startInfo.ArgumentList.Add(CommandName); startInfo.ArgumentList.Add(hook.AssemblyPath); @@ -523,6 +535,48 @@ private static string ResolveDotnetHostPath() return "dotnet"; } + private static ProcessStartInfo CreateStartInfo() + => new() + { + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + StandardInputEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), + StandardOutputEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), + StandardErrorEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), + CreateNoWindow = true, + }; + + private static bool ShouldStartCurrentExecutable(string? currentProcessPath, string? runnerAssemblyPath) + { + if (string.IsNullOrWhiteSpace(currentProcessPath) || IsDotnetHostPath(currentProcessPath)) + return false; + + var processName = Path.GetFileNameWithoutExtension(currentProcessPath); + var appName = typeof(PostExtractionHookCallbackWorker).Assembly.GetName().Name; + if (!string.IsNullOrWhiteSpace(appName) + && string.Equals(processName, appName, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return string.IsNullOrWhiteSpace(runnerAssemblyPath); + } + + private static string? ResolveCurrentRunnerAssemblyPath() + { + var assemblyName = typeof(PostExtractionHookCallbackWorker).Assembly.GetName().Name; + if (string.IsNullOrWhiteSpace(assemblyName)) + return null; + + var candidate = Path.Combine(AppContext.BaseDirectory, assemblyName + ".dll"); + return File.Exists(candidate) ? candidate : null; + } + + private static bool IsDotnetHostPath(string path) + => string.Equals(Path.GetFileNameWithoutExtension(path), "dotnet", StringComparison.OrdinalIgnoreCase); + private static void ApplyCurrentRuntimeRollForward(ProcessStartInfo startInfo) { var targetMajor = GetRunnerTargetFrameworkMajor(); @@ -570,3 +624,8 @@ internal sealed record WorkerResponse( string? CallbackError, string? WorkerError); } + +[JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)] +[JsonSerializable(typeof(PostExtractionHookCallbackWorker.WorkerRequest))] +[JsonSerializable(typeof(PostExtractionHookCallbackWorker.WorkerResponse))] +internal partial class PostExtractionHookCallbackWorkerJsonContext : JsonSerializerContext; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs new file mode 100644 index 0000000000..dcd05587e9 --- /dev/null +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs @@ -0,0 +1,671 @@ +using System.Diagnostics; +using System.Reflection; +using System.Runtime.Versioning; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using CodeIndex.Models; + +namespace CodeIndex.Indexer; + +internal sealed record SymbolExtractionWorkerResult( + bool Success, + bool TimedOut, + string? WorkerError, + long DurationMs, + List? Symbols); + +internal sealed class SymbolExtractionWorkerClient : IDisposable +{ + private readonly object gate = new(); + private Process? process; + private StringBuilder stderr = new(); + private bool disposed; + + internal SymbolExtractionWorkerResult Invoke( + long fileId, + string? lang, + string content, + string filePath, + string projectRoot, + TimeSpan callbackBudget, + CancellationToken cancellationToken = default) + { + lock (gate) + { + ObjectDisposedException.ThrowIf(disposed, this); + var stopwatch = Stopwatch.StartNew(); + if (!EnsureStarted(out var startError)) + { + stopwatch.Stop(); + return Failure(startError, stopwatch.ElapsedMilliseconds); + } + + var request = new SymbolExtractionWorker.WorkerRequest(fileId, lang, content, filePath, projectRoot); + var requestJson = JsonSerializer.Serialize(request, SymbolExtractionWorker.JsonOptions); + var waitMilliseconds = GetRemainingWaitMilliseconds(stopwatch, callbackBudget); + if (waitMilliseconds <= 0) + { + KillWorker(); + stopwatch.Stop(); + return TimedOut(stopwatch.ElapsedMilliseconds); + } + + Task responseTask; + Task sendTask; + try + { + responseTask = process!.StandardOutput.ReadLineAsync(); + sendTask = SendRequestAsync(process.StandardInput, requestJson); + } + catch (Exception ex) + { + KillWorker(); + stopwatch.Stop(); + return Failure($"failed to send symbol extraction request: {ex.Message}", stopwatch.ElapsedMilliseconds); + } + + if (!WaitForTask(sendTask, waitMilliseconds, cancellationToken, out var sendException)) + { + KillWorker(); + stopwatch.Stop(); + return TimedOut(stopwatch.ElapsedMilliseconds); + } + + if (sendException != null) + { + KillWorker(); + stopwatch.Stop(); + return Failure($"failed to send symbol extraction request: {sendException.Message}", stopwatch.ElapsedMilliseconds); + } + + waitMilliseconds = GetRemainingWaitMilliseconds(stopwatch, callbackBudget); + if (waitMilliseconds <= 0 || !WaitForTask(responseTask, waitMilliseconds, cancellationToken, out var responseException)) + { + KillWorker(); + stopwatch.Stop(); + return TimedOut(stopwatch.ElapsedMilliseconds); + } + + if (responseException != null) + { + KillWorker(); + stopwatch.Stop(); + return Failure($"failed to read symbol extraction response: {responseException.Message}", stopwatch.ElapsedMilliseconds); + } + + if (CallbackBudgetExceeded(stopwatch, callbackBudget)) + { + KillWorker(); + stopwatch.Stop(); + return TimedOut(stopwatch.ElapsedMilliseconds); + } + + stopwatch.Stop(); + var responseJson = responseTask.GetAwaiter().GetResult(); + if (responseJson == null) + { + var workerError = BuildWorkerExitError(process, stderr.ToString(), "worker exited before returning a response."); + ClearExitedWorker(); + return Failure(workerError, stopwatch.ElapsedMilliseconds); + } + + SymbolExtractionWorker.WorkerResponse? response; + try + { + response = JsonSerializer.Deserialize( + responseJson, + SymbolExtractionWorker.JsonOptions); + } + catch (JsonException ex) + { + KillWorker(); + return Failure($"worker returned invalid JSON: {ex.Message}", stopwatch.ElapsedMilliseconds); + } + + if (response == null) + return Failure("worker returned an empty response.", stopwatch.ElapsedMilliseconds); + ForwardCapturedStderr(response.CapturedStderr); + if (!string.IsNullOrWhiteSpace(response.WorkerError)) + return Failure(response.WorkerError, stopwatch.ElapsedMilliseconds); + if (response.Symbols == null) + return Failure("worker response omitted symbols.", stopwatch.ElapsedMilliseconds); + + return new SymbolExtractionWorkerResult( + Success: true, + TimedOut: false, + WorkerError: null, + DurationMs: stopwatch.ElapsedMilliseconds, + Symbols: response.Symbols); + } + } + + public void Dispose() + { + lock (gate) + { + if (disposed) + return; + + disposed = true; + if (process == null) + return; + + try + { + process.StandardInput.Close(); + } + catch + { + // Best effort: disposal should not throw after indexing has completed. + } + + if (!WaitForWorkerExit(process, 1000)) + KillWorker(); + else + ClearExitedWorker(); + } + } + + private bool EnsureStarted(out string error) + { + if (process is { HasExited: false }) + { + error = string.Empty; + return true; + } + + ClearExitedWorker(); + stderr = new StringBuilder(); + if (!SymbolExtractionWorker.TryCreateStartInfo(out var startInfo, out error)) + return false; + + var next = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + next.ErrorDataReceived += (_, eventArgs) => + { + if (eventArgs.Data != null) + stderr.AppendLine(eventArgs.Data); + }; + + try + { + if (!next.Start()) + { + error = "symbol extraction worker process did not start."; + next.Dispose(); + return false; + } + + next.BeginErrorReadLine(); + process = next; + error = string.Empty; + return true; + } + catch (Exception ex) + { + error = $"failed to start symbol extraction worker process: {ex.Message}"; + next.Dispose(); + return false; + } + } + + private void KillWorker() + { + if (process == null) + return; + + SymbolExtractionWorker.TryKillProcess(process); + ClearExitedWorker(); + } + + private void ClearExitedWorker() + { + if (process == null) + return; + + process.Dispose(); + process = null; + } + + private static SymbolExtractionWorkerResult Failure(string? message, long durationMs) + => new( + Success: false, + TimedOut: false, + WorkerError: string.IsNullOrWhiteSpace(message) ? "isolated symbol extraction worker failed." : message, + DurationMs: Math.Max(0, durationMs), + Symbols: null); + + private static SymbolExtractionWorkerResult TimedOut(long durationMs) + => new( + Success: false, + TimedOut: true, + WorkerError: null, + DurationMs: Math.Max(0, durationMs), + Symbols: null); + + private static async Task SendRequestAsync(TextWriter input, string requestJson) + { + await input.WriteLineAsync(requestJson).ConfigureAwait(false); + await input.FlushAsync().ConfigureAwait(false); + } + + private bool WaitForTask(Task task, int milliseconds, CancellationToken cancellationToken, out Exception? exception) + { + try + { + if (!task.Wait(milliseconds, cancellationToken)) + { + exception = null; + return false; + } + + exception = null; + return true; + } + catch (AggregateException ex) + { + exception = ex.GetBaseException(); + return true; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + KillWorker(); + throw; + } + catch (Exception ex) + { + exception = ex; + return true; + } + } + + private static int GetRemainingWaitMilliseconds(Stopwatch stopwatch, TimeSpan callbackBudget) + { + var remainingMilliseconds = callbackBudget.TotalMilliseconds - stopwatch.Elapsed.TotalMilliseconds; + if (remainingMilliseconds <= 0) + return 0; + + return Math.Max(1, (int)Math.Ceiling(Math.Min(remainingMilliseconds, int.MaxValue))); + } + + private static bool CallbackBudgetExceeded(Stopwatch stopwatch, TimeSpan callbackBudget) + => stopwatch.Elapsed > callbackBudget; + + private static string BuildWorkerExitError(Process? process, string stderr, string fallback) + { + var exitCodeText = process == null + ? "unknown" + : process.ExitCode.ToString(System.Globalization.CultureInfo.InvariantCulture); + var detail = !string.IsNullOrWhiteSpace(stderr) ? stderr.Trim() : fallback; + return $"worker exited with code {exitCodeText}: {detail}"; + } + + private static bool WaitForWorkerExit(Process process, int milliseconds) + { + try + { + return process.WaitForExit(milliseconds); + } + catch + { + return false; + } + } + + private static void ForwardCapturedStderr(string? capturedStderr) + { + if (!string.IsNullOrEmpty(capturedStderr)) + Console.Error.Write(capturedStderr); + } +} + +internal static class SymbolExtractionWorker +{ + internal const string CommandName = "__cdidx-symbol-extraction"; + internal const int WorkerKillWaitMilliseconds = 5000; + internal const string DelayEnvironmentVariable = "CDIDX_TEST_SYMBOL_EXTRACTION_WORKER_DELAY_MS"; + internal const string CompletionPathEnvironmentVariable = "CDIDX_TEST_SYMBOL_EXTRACTION_WORKER_DONE_PATH"; + internal const string ConsoleStdoutEnvironmentVariable = "CDIDX_TEST_SYMBOL_EXTRACTION_WORKER_STDOUT"; + private const int CapturedConsoleMaxChars = 32 * 1024; + internal static readonly JsonSerializerOptions JsonOptions = SymbolExtractionWorkerJsonContext.Default.Options; + + internal static bool TryRunCommand( + string[] args, + TextReader input, + TextWriter output, + TextWriter error, + out int exitCode) + { + if (args.Length == 0 || !StringComparer.Ordinal.Equals(args[0], CommandName)) + { + exitCode = 0; + return false; + } + + exitCode = RunCommand(args, input, output, error); + return true; + } + + internal static bool TryCreateStartInfo(out ProcessStartInfo startInfo, out string error) + { + return TryCreateStartInfo( + Environment.ProcessPath, + ResolveCurrentRunnerAssemblyPath(), + out startInfo, + out error); + } + + internal static bool TryCreateStartInfo( + string? currentProcessPath, + string? runnerAssemblyPath, + out ProcessStartInfo startInfo, + out string error) + { + startInfo = CreateStartInfo(); + if (ShouldStartCurrentExecutable(currentProcessPath, runnerAssemblyPath)) + { + startInfo.FileName = currentProcessPath!; + startInfo.ArgumentList.Add(CommandName); + error = string.Empty; + return true; + } + + if (string.IsNullOrWhiteSpace(runnerAssemblyPath)) + { + startInfo = new ProcessStartInfo(); + error = "could not resolve the cdidx assembly path for isolated symbol extraction."; + return false; + } + + startInfo.FileName = ResolveDotnetHostPath(); + startInfo.ArgumentList.Add(runnerAssemblyPath); + startInfo.ArgumentList.Add(CommandName); + ApplyCurrentRuntimeRollForward(startInfo); + + error = string.Empty; + return true; + } + + private static ProcessStartInfo CreateStartInfo() + => new() + { + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + StandardInputEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), + StandardOutputEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), + StandardErrorEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), + CreateNoWindow = true, + }; + + private static bool ShouldStartCurrentExecutable(string? currentProcessPath, string? runnerAssemblyPath) + { + if (string.IsNullOrWhiteSpace(currentProcessPath) || IsDotnetHostPath(currentProcessPath)) + return false; + + var processName = Path.GetFileNameWithoutExtension(currentProcessPath); + var appName = typeof(SymbolExtractionWorker).Assembly.GetName().Name; + if (!string.IsNullOrWhiteSpace(appName) + && string.Equals(processName, appName, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return string.IsNullOrWhiteSpace(runnerAssemblyPath); + } + + internal static void TryKillProcess(Process process) + { + try + { + if (!process.HasExited) + process.Kill(entireProcessTree: true); + } + catch + { + // Best effort: timeout reporting must not fail because cleanup failed. + } + + try + { + process.WaitForExit(WorkerKillWaitMilliseconds); + } + catch + { + // Best effort: the parent continues with the timeout diagnostic. + } + } + + private static int RunCommand(string[] args, TextReader input, TextWriter output, TextWriter error) + { + if (args.Length != 1) + { + error.WriteLine("symbol extraction worker does not accept positional arguments."); + return 2; + } + + try + { + string? requestJson; + while ((requestJson = input.ReadLine()) != null) + { + WorkerResponse response; + try + { + var request = JsonSerializer.Deserialize(requestJson, JsonOptions) + ?? throw new InvalidOperationException("worker request was empty."); + response = InvokeInsideWorker(request); + } + catch (Exception ex) + { + response = new WorkerResponse(null, ex.Message, null); + } + + output.WriteLine(JsonSerializer.Serialize(response, JsonOptions)); + output.Flush(); + } + + return 0; + } + catch (Exception ex) + { + error.WriteLine(ex.Message); + return 1; + } + } + + private static WorkerResponse InvokeInsideWorker(WorkerRequest request) + { + var originalOut = Console.Out; + var originalError = Console.Error; + using var capturedOut = new BoundedTextWriter(CapturedConsoleMaxChars); + using var capturedError = new BoundedTextWriter(CapturedConsoleMaxChars); + try + { + Console.SetOut(capturedOut); + Console.SetError(capturedError); + WriteConsoleOutputForTestingIfRequested(); + DelayForTestingIfRequested(); + var symbols = SymbolExtractor.Extract( + request.FileId, + request.Lang, + request.Content, + request.FilePath, + request.ProjectRoot, + CancellationToken.None); + return new WorkerResponse(symbols, null, capturedError.GetCapturedText()); + } + catch (Exception ex) + { + return new WorkerResponse(null, ex.Message, capturedError.GetCapturedText()); + } + finally + { + Console.SetOut(originalOut); + Console.SetError(originalError); + } + } + + private static void WriteConsoleOutputForTestingIfRequested() + { + var stdout = Environment.GetEnvironmentVariable(ConsoleStdoutEnvironmentVariable); + if (!string.IsNullOrEmpty(stdout)) + Console.Out.WriteLine(stdout); + } + + private static void DelayForTestingIfRequested() + { + var raw = Environment.GetEnvironmentVariable(DelayEnvironmentVariable); + if (!int.TryParse(raw, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var milliseconds) + || milliseconds <= 0) + { + return; + } + + Thread.Sleep(milliseconds); + var completionPath = Environment.GetEnvironmentVariable(CompletionPathEnvironmentVariable); + if (!string.IsNullOrWhiteSpace(completionPath)) + File.WriteAllText(completionPath, "completed"); + } + + private static string ResolveDotnetHostPath() + { + var dotnetHostPath = Environment.GetEnvironmentVariable("DOTNET_HOST_PATH"); + if (!string.IsNullOrWhiteSpace(dotnetHostPath)) + return dotnetHostPath; + + var processPath = Environment.ProcessPath; + if (!string.IsNullOrWhiteSpace(processPath) && IsDotnetHostPath(processPath)) + { + return processPath; + } + + return "dotnet"; + } + + private static string? ResolveCurrentRunnerAssemblyPath() + { + var assemblyName = typeof(SymbolExtractionWorker).Assembly.GetName().Name; + if (string.IsNullOrWhiteSpace(assemblyName)) + return null; + + var candidate = Path.Combine(AppContext.BaseDirectory, assemblyName + ".dll"); + return File.Exists(candidate) ? candidate : null; + } + + private static bool IsDotnetHostPath(string path) + => string.Equals(Path.GetFileNameWithoutExtension(path), "dotnet", StringComparison.OrdinalIgnoreCase); + + private static void ApplyCurrentRuntimeRollForward(ProcessStartInfo startInfo) + { + var targetMajor = GetRunnerTargetFrameworkMajor(); + if (targetMajor.HasValue && Environment.Version.Major > targetMajor.Value) + startInfo.Environment["DOTNET_ROLL_FORWARD"] = "LatestMajor"; + } + + private static int? GetRunnerTargetFrameworkMajor() + { + var frameworkName = typeof(SymbolExtractionWorker) + .Assembly + .GetCustomAttribute() + ?.FrameworkName; + if (string.IsNullOrWhiteSpace(frameworkName)) + return null; + + const string versionPrefix = "Version=v"; + var versionIndex = frameworkName.IndexOf(versionPrefix, StringComparison.OrdinalIgnoreCase); + if (versionIndex < 0) + return null; + + var majorStart = versionIndex + versionPrefix.Length; + var majorEnd = frameworkName.IndexOf('.', majorStart); + var majorText = majorEnd < 0 + ? frameworkName[majorStart..] + : frameworkName[majorStart..majorEnd]; + return int.TryParse( + majorText, + System.Globalization.NumberStyles.Integer, + System.Globalization.CultureInfo.InvariantCulture, + out var major) + ? major + : null; + } + + internal sealed record WorkerRequest( + long FileId, + string? Lang, + string Content, + string FilePath, + string ProjectRoot); + + internal sealed record WorkerResponse( + List? Symbols, + string? WorkerError, + string? CapturedStderr); + + private sealed class BoundedTextWriter(int maxChars) : TextWriter + { + private readonly StringBuilder builder = new(); + private bool truncated; + + public override Encoding Encoding => Encoding.UTF8; + + public override void Write(char value) + { + if (builder.Length < maxChars) + { + builder.Append(value); + return; + } + + truncated = true; + } + + public override void Write(string? value) + { + if (string.IsNullOrEmpty(value)) + return; + + Append(value.AsSpan()); + } + + public override void Write(char[] buffer, int index, int count) + => Append(buffer.AsSpan(index, count)); + + internal string GetCapturedText() + { + if (!truncated) + return builder.ToString(); + + return builder + .AppendLine() + .Append("[cdidx] captured worker console output truncated.") + .ToString(); + } + + private void Append(ReadOnlySpan value) + { + var remaining = maxChars - builder.Length; + if (remaining <= 0) + { + truncated = true; + return; + } + + if (value.Length <= remaining) + { + builder.Append(value); + return; + } + + builder.Append(value[..remaining]); + truncated = true; + } + } +} + +[JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)] +[JsonSerializable(typeof(SymbolExtractionWorker.WorkerRequest))] +[JsonSerializable(typeof(SymbolExtractionWorker.WorkerResponse))] +internal partial class SymbolExtractionWorkerJsonContext : JsonSerializerContext; diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 46acc8ea1b..639ac84cdc 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -7,6 +7,7 @@ using CodeIndex.Database; using CodeIndex.Indexer; using CodeIndex.Indexer.Extensibility; +using CodeIndex.Indexer.Hooks; using CodeIndex.Models; using Microsoft.Data.Sqlite; @@ -162,6 +163,201 @@ public void Run_FilesMode_WhenSymbolExtractionStalls_ReportsStallInsteadOfInterr } } + [Fact] + public void SymbolExtractionWorker_TimeoutKillsWorkerBeforeDelayedExtractionContinues() + { + var projectRoot = CreateTempProject(); + lock (TestConsoleLock.Gate) + { + using var env = EnvironmentVariableScope.Capture( + SymbolExtractionWorker.DelayEnvironmentVariable, + SymbolExtractionWorker.CompletionPathEnvironmentVariable); + try + { + var completionPath = Path.Combine(projectRoot, "symbol-worker.done"); + env.Set(SymbolExtractionWorker.DelayEnvironmentVariable, "500"); + env.Set(SymbolExtractionWorker.CompletionPathEnvironmentVariable, completionPath); + + using var worker = new SymbolExtractionWorkerClient(); + var result = worker.Invoke( + 0, + "csharp", + "public class App { }\n", + Path.Combine(projectRoot, "App.cs"), + projectRoot, + TimeSpan.FromMilliseconds(50)); + + Assert.True(result.TimedOut); + Assert.False(result.Success); + AssertFileDoesNotAppear(completionPath, TimeSpan.FromMilliseconds(1000)); + } + finally + { + DeleteDirectory(projectRoot); + } + } + } + + [Fact] + public void SymbolExtractionWorker_CapturesStdoutAndForwardsStderrDiagnostics() + { + var projectRoot = CreateTempProject(); + lock (TestConsoleLock.Gate) + { + using var env = EnvironmentVariableScope.Capture(SymbolExtractionWorker.ConsoleStdoutEnvironmentVariable); + try + { + WriteSymbolWorkerPatternConfig( + projectRoot, + "language: \"toydsl\"\nextensions:\n - extension: \".toy\"\npatterns:\n - kind: \"class\"\n regex: \"^(a+)+$\"\n"); + env.Set(SymbolExtractionWorker.ConsoleStdoutEnvironmentVariable, "not-json-protocol"); + var slowLine = new string('a', 10_000) + "!"; + SymbolExtractionWorkerResult? result = null; + + var stderr = ConsoleCapture.CaptureError(() => + { + using var worker = new SymbolExtractionWorkerClient(); + result = worker.Invoke( + 0, + "toydsl", + slowLine, + Path.Combine(projectRoot, "demo.toy"), + projectRoot, + TimeSpan.FromSeconds(5)); + }); + + Assert.NotNull(result); + Assert.True(result.Success, result.WorkerError); + Assert.False(result.TimedOut); + Assert.Empty(result.Symbols!); + Assert.Contains("Pattern extractor", stderr, StringComparison.Ordinal); + Assert.Contains("timed out", stderr, StringComparison.Ordinal); + Assert.DoesNotContain("not-json-protocol", stderr, StringComparison.Ordinal); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + DeleteDirectory(projectRoot); + } + } + } + + [Fact] + public void SymbolExtractionWorker_StartInfo_UsesCurrentCdidxExecutableWhenAvailable() + { + var currentProcessPath = Path.Combine(Path.GetTempPath(), OperatingSystem.IsWindows() ? "cdidx.exe" : "cdidx"); + + var created = SymbolExtractionWorker.TryCreateStartInfo( + currentProcessPath, + runnerAssemblyPath: string.Empty, + out var startInfo, + out var error); + + Assert.True(created, error); + Assert.Equal(currentProcessPath, startInfo.FileName); + Assert.Equal([SymbolExtractionWorker.CommandName], startInfo.ArgumentList); + Assert.True(startInfo.RedirectStandardInput); + Assert.True(startInfo.RedirectStandardOutput); + Assert.True(startInfo.RedirectStandardError); + } + + [Fact] + public void SymbolExtractionWorker_StartInfo_UsesFrameworkDependentDllWhenProcessIsNotCdidx() + { + var currentProcessPath = Path.Combine(Path.GetTempPath(), OperatingSystem.IsWindows() ? "testhost.exe" : "testhost"); + var runnerAssemblyPath = Path.Combine(Path.GetTempPath(), "cdidx.dll"); + + var created = SymbolExtractionWorker.TryCreateStartInfo( + currentProcessPath, + runnerAssemblyPath, + out var startInfo, + out var error); + + Assert.True(created, error); + Assert.NotEqual(currentProcessPath, startInfo.FileName); + Assert.Equal([runnerAssemblyPath, SymbolExtractionWorker.CommandName], startInfo.ArgumentList); + } + + [Fact] + public void PostExtractionHookCallbackWorker_StartInfo_UsesCurrentCdidxExecutableWhenAvailable() + { + var hook = new PostExtractionHookInfo( + "demo", + Path.Combine(Path.GetTempPath(), "demo-hook.dll"), + "Demo.Hook"); + var currentProcessPath = Path.Combine(Path.GetTempPath(), OperatingSystem.IsWindows() ? "cdidx.exe" : "cdidx"); + + var created = PostExtractionHookCallbackWorker.TryCreateStartInfo( + hook, + currentProcessPath, + runnerAssemblyPath: string.Empty, + out var startInfo, + out var error); + + Assert.True(created, error); + Assert.Equal(currentProcessPath, startInfo.FileName); + Assert.Equal([PostExtractionHookCallbackWorker.CommandName, hook.AssemblyPath, hook.TypeName], startInfo.ArgumentList); + Assert.True(startInfo.RedirectStandardInput); + Assert.True(startInfo.RedirectStandardOutput); + Assert.True(startInfo.RedirectStandardError); + } + + [Fact] + public void PostExtractionHookCallbackWorker_StartInfo_UsesFrameworkDependentDllWhenProcessIsNotCdidx() + { + var hook = new PostExtractionHookInfo( + "demo", + Path.Combine(Path.GetTempPath(), "demo-hook.dll"), + "Demo.Hook"); + var currentProcessPath = Path.Combine(Path.GetTempPath(), OperatingSystem.IsWindows() ? "testhost.exe" : "testhost"); + var runnerAssemblyPath = Path.Combine(Path.GetTempPath(), "cdidx.dll"); + + var created = PostExtractionHookCallbackWorker.TryCreateStartInfo( + hook, + currentProcessPath, + runnerAssemblyPath, + out var startInfo, + out var error); + + Assert.True(created, error); + Assert.NotEqual(currentProcessPath, startInfo.FileName); + Assert.Equal([runnerAssemblyPath, PostExtractionHookCallbackWorker.CommandName, hook.AssemblyPath, hook.TypeName], startInfo.ArgumentList); + } + + [SkipOnMacOsArm64Fact] + public void Run_PublishedSingleFileBinary_IndexesWithIsolatedSymbolWorker() + { + var projectRoot = CreateTempProject(); + var publishDir = Path.Combine(Path.GetTempPath(), $"cdidx_single_file_publish_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(Path.GetTempPath(), $"cdidx_single_file_index_{Guid.NewGuid():N}.db"); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "App.cs"), + "public class PublishedSingleFileApp { public void Run() { } }\n"); + + var publishedCli = PublishTrimmedCli(publishDir, publishSingleFile: true); + + var (exitCode, stdout, stderr) = RunPublishedCli(publishedCli, projectRoot, projectRoot, "--db", dbPath, "--json", "--force"); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Contains("cdidx: scanning files...", stderr); + Assert.Contains("cdidx: preparing index writes...", stderr); + using (var document = JsonDocument.Parse(stdout)) + Assert.Equal("success", document.RootElement.GetProperty("status").GetString()); + Assert.Equal(1, CountRows(dbPath, "files")); + Assert.True(CountRows(dbPath, "symbols") >= 1); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + DeleteDirectory(publishDir); + if (File.Exists(dbPath)) + File.Delete(dbPath); + } + } + [Fact] public void GetJsonIndexHeartbeatPath_UsesWorkerPhaseWhenMainThreadIsIdle() { @@ -9775,7 +9971,7 @@ private static (int ExitCode, string StdOut, string StdErr) RunPublishedCli(stri return (process.ExitCode, stdOut, stdErr); } - private static string PublishTrimmedCli(string outputDir) + private static string PublishTrimmedCli(string outputDir, bool publishSingleFile = false) { Directory.CreateDirectory(outputDir); var buildOutputDir = Path.Combine(outputDir, "bin", "publish") + Path.DirectorySeparatorChar; @@ -9802,7 +9998,7 @@ private static string PublishTrimmedCli(string outputDir) psi.ArgumentList.Add(outputDir); psi.ArgumentList.Add("-p:PublishTrimmed=true"); psi.ArgumentList.Add("-p:SelfContained=true"); - psi.ArgumentList.Add("-p:PublishSingleFile=false"); + psi.ArgumentList.Add($"-p:PublishSingleFile={publishSingleFile.ToString().ToLowerInvariant()}"); psi.ArgumentList.Add($"-p:OutputPath={buildOutputDir}"); psi.ArgumentList.Add($"-p:IntermediateOutputPath={intermediateDir}"); psi.ArgumentList.Add($"-p:NuGetLockFilePath={lockFilePath}"); @@ -9882,6 +10078,25 @@ private static string GetBuiltCliDllPath() throw new InvalidOperationException("Could not locate built cdidx.dll from test output path / テスト出力パスから cdidx.dll を特定できませんでした"); } + private static void AssertFileDoesNotAppear(string path, TimeSpan duration) + { + var deadline = DateTimeOffset.UtcNow.Add(duration); + while (DateTimeOffset.UtcNow < deadline) + { + if (File.Exists(path)) + throw new InvalidOperationException("The timed-out symbol extraction worker continued running after the callback returned."); + + Thread.Sleep(25); + } + } + + private static void WriteSymbolWorkerPatternConfig(string projectRoot, string content) + { + var path = Path.Combine(projectRoot, ".cdidx", "patterns", "toydsl.yaml"); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, content); + } + private static string GetRepositoryRoot() { var dir = new DirectoryInfo(AppContext.BaseDirectory);