Handle file deletion races during indexing

changelog.d/unreleased/2044.fixed.md

@@ -0,0 +1,18 @@
+---
+category: fixed
+issues:
+  - 2044
+affected:
+  - src/CodeIndex/Indexer/Scanning/FileIndexer.cs
+  - src/CodeIndex/Cli/IndexCommandRunner.cs
+  - src/CodeIndex/Mcp/McpToolHandlers.cs
+  - tests/CodeIndex.Tests/FileIndexerTests.cs
+---
+
+## English
+
+- **Concurrent file deletions no longer fail indexing (#2044)** — files deleted after directory enumeration are now treated as non-fatal warnings and stale index rows are removed when possible.
+
+## 日本語
+
+- **並行削除されたファイルで index が失敗しないようにしました (#2044)** — directory enumeration 後に削除されたファイルは非致命的な warning として扱い、可能な場合は古い index 行を削除します。

src/CodeIndex/Cli/IndexCommandRunner.cs

@@ -1854,6 +1854,37 @@ void ThrowIfUpdateCancelled()
 
                 var indexability = FileIndexer.GetFileIndexability(absPath);
                 var detection = FileIndexer.TryDetectLanguage(absPath);
+                if (indexability == FileIndexer.FileProbeStatus.Missing || detection.Status == FileIndexer.FileProbeStatus.Missing)
+                {
+                    var message = $"{relPath}: skipped because it was deleted during indexing.";
+                    warnings++;
+                    warningList.Add(new CliJsonMessage(relPath, message));
+                    if (!options.Json && !options.Quiet)
+                    {
+                        PauseUpdateSpinnerForConsoleWrite();
+                        ConsoleUi.PrintWarning(message);
+                        ResumeUpdateSpinnerAfterConsoleWrite();
+                    }
+
+                    if (writer.HasFileAtPath(relPath))
+                    {
+                        DemoteReadinessOnce();
+                        using var deleteTxn = writer.BeginTransaction();
+                        if (writer.DeleteFileByPath(relPath))
+                        {
+                            WriteProjectRootOnce();
+                            deleteTxn.Commit();
+                            removed++;
+                            ftsMutated = true;
+                        }
+                    }
+                    else
+                    {
+                        skipped++;
+                    }
+                    continue;
+                }
+
                 if (indexability == FileIndexer.FileProbeStatus.ProbeFailed || detection.Status == FileIndexer.FileProbeStatus.ProbeFailed)
                 {
                     DemoteReadinessOnce();
@@ -2111,6 +2142,40 @@ void ThrowIfUpdateCancelled()
                     continue;
                 }
 
+                if (ex is FileNotFoundException or DirectoryNotFoundException)
+                {
+                    if (fileBatchMarked)
+                        writer.ClearBatchInProgress();
+
+                    var message = $"{relPath}: skipped because it was deleted during indexing.";
+                    warnings++;
+                    warningList.Add(new CliJsonMessage(relPath, message));
+                    if (!options.Json && !options.Quiet)
+                    {
+                        PauseUpdateSpinnerForConsoleWrite();
+                        ConsoleUi.PrintWarning(message);
+                        ResumeUpdateSpinnerAfterConsoleWrite();
+                    }
+
+                    if (writer.HasFileAtPath(relPath))
+                    {
+                        DemoteReadinessOnce();
+                        using var deleteTxn = writer.BeginTransaction();
+                        if (writer.DeleteFileByPath(relPath))
+                        {
+                            WriteProjectRootOnce();
+                            deleteTxn.Commit();
+                            removed++;
+                            ftsMutated = true;
+                        }
+                    }
+                    else
+                    {
+                        skipped++;
+                    }
+                    continue;
+                }
+
                 DemoteReadinessOnce();
                 if (fileBatchMarked)
                     writer.ClearBatchInProgress();
@@ -3401,6 +3466,13 @@ void StopJsonHeartbeat()
                         {
                             extractionResults.Add(FullScanFileWorkItem.Skipped(filePath, ex.Message), cancellationToken);
                         }
+                        catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
+                        {
+                            var relativePath = FileIndexer.NormalizePathSeparators(Path.GetRelativePath(projectRoot, filePath));
+                            extractionResults.Add(
+                                FullScanFileWorkItem.Skipped(filePath, $"{relativePath}: skipped because it was deleted during indexing."),
+                                cancellationToken);
+                        }
                         catch (Exception ex)
                         {
                             extractionResults.Add(FullScanFileWorkItem.Failure(filePath, ex), cancellationToken);

src/CodeIndex/Indexer/Scanning/FileIndexer.cs

@@ -20,6 +20,7 @@ internal enum FileProbeStatus
         Supported,
         Unsupported,
         ProbeFailed,
+        Missing,
     }
 
     internal readonly record struct LanguageDetectionResult(FileProbeStatus Status, string? Language);
@@ -1254,7 +1255,33 @@ internal static FileProbeStatus GetFileIndexability(string filePath)
         // File.GetAttributes は .NET 上で lstat 相当（symlink target を辿らない）なので、Windows でも Unix でも必要な判定になる。
         // Unix 側の stat() は symlink を辿るため、このガードが無いと symlink→通常ファイルが
         // Supported として通過してしまう。
-        if (HasSkippedAttributes(filePath))
+        FileAttributes attributes;
+        try
+        {
+            attributes = File.GetAttributes(LongPath.EnsureWindowsPrefix(filePath));
+        }
+        catch (FileNotFoundException)
+        {
+            return FileProbeStatus.Missing;
+        }
+        catch (DirectoryNotFoundException)
+        {
+            return FileProbeStatus.Missing;
+        }
+        catch (UnauthorizedAccessException)
+        {
+            return OperatingSystem.IsWindows()
+                ? FileProbeStatus.Supported
+                : FileProbeStatus.ProbeFailed;
+        }
+        catch (IOException)
+        {
+            return OperatingSystem.IsWindows()
+                ? FileProbeStatus.Supported
+                : FileProbeStatus.ProbeFailed;
+        }
+
+        if (HasSkippedAttributes(attributes))
             return FileProbeStatus.Unsupported;
 
         if (OperatingSystem.IsWindows())
@@ -1753,6 +1780,17 @@ private bool EnumerateDirectory(
                     // GetFileIndexability もファイル symlink / reparse point を拒否するため、
                     // update モード (--files / --commits) でも同じ skip 挙動が二重プローブ無しで効く。
                     var indexability = GetFileIndexability(file);
+                    if (indexability == FileProbeStatus.Missing)
+                    {
+                        var relativePath = ToRelativePath(file);
+                        errors.Add(new ScanError(
+                            relativePath,
+                            "Skipped file because it was deleted during scanning.",
+                            ScanIssueSeverity.Warning));
+                        nonIndexablePaths.Add(relativePath);
+                        continue;
+                    }
+
                     if (indexability == FileProbeStatus.ProbeFailed)
                     {
                         var relativePath = ToRelativePath(file);
@@ -1771,6 +1809,16 @@ private bool EnumerateDirectory(
                     // Include files with a known extension/filename or an extensionless recognized shebang
                     // 既知の拡張子・既知ファイル名、または拡張子なしで shebang を認識できるファイルを含める
                     var language = TryDetectLanguage(file);
+                    if (language.Status == FileProbeStatus.Missing)
+                    {
+                        errors.Add(new ScanError(
+                            relativeFile,
+                            "Skipped file because it was deleted during scanning.",
+                            ScanIssueSeverity.Warning));
+                        nonIndexablePaths.Add(relativeFile);
+                        continue;
+                    }
+
                     if (language.Status == FileProbeStatus.ProbeFailed)
                     {
                         errors.Add(new ScanError(relativeFile, "Could not probe file for indexability/language."));
@@ -3134,7 +3182,11 @@ internal static string ComputeChecksum(byte[] bytes)
     /// </summary>
     private static LanguageDetectionResult TryDetectLanguageFromShebang(string filePath)
     {
-        if (GetFileIndexability(filePath) != FileProbeStatus.Supported)
+        var indexability = GetFileIndexability(filePath);
+        if (indexability == FileProbeStatus.Missing)
+            return new LanguageDetectionResult(FileProbeStatus.Missing, null);
+
+        if (indexability != FileProbeStatus.Supported)
             return new LanguageDetectionResult(FileProbeStatus.Unsupported, null);
 
         try
@@ -3186,6 +3238,14 @@ private static LanguageDetectionResult TryDetectLanguageFromShebang(string fileP
                 ? new LanguageDetectionResult(FileProbeStatus.Supported, language)
                 : new LanguageDetectionResult(FileProbeStatus.Unsupported, null);
         }
+        catch (FileNotFoundException)
+        {
+            return new LanguageDetectionResult(FileProbeStatus.Missing, null);
+        }
+        catch (DirectoryNotFoundException)
+        {
+            return new LanguageDetectionResult(FileProbeStatus.Missing, null);
+        }
         catch (IOException)
         {
             return new LanguageDetectionResult(FileProbeStatus.ProbeFailed, null);

src/CodeIndex/Mcp/McpToolHandlers.cs

@@ -2489,6 +2489,27 @@ void WriteProjectRootOnce()
                     errors++;
                 }
             }
+            catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
+            {
+                if (fileBatchMarked)
+                    writer.ClearBatchInProgress();
+
+                try
+                {
+                    var relativePath = FileIndexer.NormalizePathSeparators(Path.GetRelativePath(projectPath, filePath));
+                    if (writer.HasFileAtPath(relativePath))
+                    {
+                        using var txn = writer.BeginTransaction();
+                        writer.DeleteFileByPath(relativePath);
+                        WriteProjectRootOnce();
+                        txn.Commit();
+                    }
+                }
+                catch
+                {
+                    errors++;
+                }
+            }
             catch
             {
                 if (fileBatchMarked)

tests/CodeIndex.Tests/FileIndexerTests.cs

@@ -2711,6 +2711,49 @@ public void ScanFiles_SkipsCaseInsensitiveDirectories()
         }
     }
 
+    [Fact]
+    public void ScanFilesDetailed_FileDeletedAfterEnumeration_RecordsWarning()
+    {
+        var tempDir = Path.Combine(Path.GetTempPath(), $"cdidx-delete-race-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(tempDir);
+        try
+        {
+            var scriptPath = Path.Combine(tempDir, "script");
+            File.WriteAllText(scriptPath, "#!/usr/bin/env python\nprint('hello')\n");
+
+            var indexer = new FileIndexer(
+                tempDir,
+                ignoreCase: false,
+                ignoreRuleRoot: null,
+                maxFileSizeBytes: null,
+                directoryIgnoreCaseProbe: _ => false,
+                enumerateFiles: dir => Path.GetFullPath(dir) == Path.GetFullPath(tempDir)
+                    ? DeleteBeforeProbe(scriptPath)
+                    : Directory.EnumerateFiles(dir));
+
+            var result = indexer.ScanFilesDetailed();
+
+            Assert.Empty(result.Files);
+            Assert.Contains("script", result.NonIndexablePaths);
+            var warning = Assert.Single(result.Errors);
+            Assert.Equal("script", warning.Path);
+            Assert.Equal(FileIndexer.ScanIssueSeverity.Warning, warning.Severity);
+            Assert.Contains("deleted during scanning", warning.Message, StringComparison.OrdinalIgnoreCase);
+            Assert.False(result.HadErrors);
+        }
+        finally
+        {
+            if (Directory.Exists(tempDir))
+                Directory.Delete(tempDir, true);
+        }
+
+        static IEnumerable<string> DeleteBeforeProbe(string path)
+        {
+            File.Delete(path);
+            yield return path;
+        }
+    }
+
     [Fact]
     public void ScanFiles_DescendsIntoSubmoduleHostedUnderSkipDir()
     {