From a29ac30dbf7bb9e21d9fb65e30502a841f027a6c Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 13:26:38 +0900 Subject: [PATCH 1/2] Fix FileIndexer race and symlink handling (#1654 #1655 #1656 #1711) --- DEVELOPER_GUIDE.md | 4 +- changelog.d/unreleased/1654.fixed.md | 16 ++ changelog.d/unreleased/1655.fixed.md | 16 ++ changelog.d/unreleased/1656.fixed.md | 18 ++ changelog.d/unreleased/1711.added.md | 20 ++ src/CodeIndex/Cli/ConsoleUi.cs | 3 +- .../Cli/IndexCommandRunner.DryRun.cs | 2 +- .../Cli/IndexCommandRunner.FullScan.cs | 2 + src/CodeIndex/Cli/IndexCommandRunner.Parse.cs | 26 ++- src/CodeIndex/Cli/IndexCommandRunner.cs | 3 +- src/CodeIndex/Cli/IndexWatchRunner.cs | 5 + src/CodeIndex/Cli/JsonOutputContracts.cs | 2 + src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 200 ++++++++++++++---- tests/CodeIndex.Tests/FileIndexerTests.cs | 110 ++++++++++ 14 files changed, 379 insertions(+), 48 deletions(-) create mode 100644 changelog.d/unreleased/1654.fixed.md create mode 100644 changelog.d/unreleased/1655.fixed.md create mode 100644 changelog.d/unreleased/1656.fixed.md create mode 100644 changelog.d/unreleased/1711.added.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 92188ccd62..19d04117a3 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -114,13 +114,13 @@ The lock files for projects with zero direct `PackageReference` entries (e.g. `t ### Indexing pipeline ``` -Directory scan / shared path filter (built-in skip lists + `.gitignore` / `.cdidxignore` + reparse/Windows Hidden/System attribute pruning) +Directory scan / shared path filter (built-in skip lists + `.gitignore` / `.cdidxignore` + directory symlink policy + reparse/Windows Hidden/System attribute pruning) → Parallel extraction workers (`--parallelism`, `CDIDX_INDEX_PARALLELISM`; default CPU count capped at 16) read UTF-8, split chunks, extract symbols/references, and validate content → Single SQLite writer checks unchanged-file reuse, UPSERTs file records, runs post-extraction hooks, and inserts chunks + symbols + references + issues in per-file transactions → Populate FTS5 index ``` -Scoped `--files` / `--commits` refreshes reuse the same path filter as full scans. Before scanning a nested project root, `FileIndexer` loads ignore files from the resolved ignore-rule root through each existing ancestor directory down to the project root's parent, then loads the project directory's own rules during the normal walk. Within each directory, `FileIndexer` loads `.gitignore` before `.cdidxignore`, appends both rule sets in that order, and honors later `!` patterns as re-includes. If an ancestor ignore directory cannot be read, scanning fails closed with a scan error instead of silently skipping those rules; `ScanFilesResult.AncestorIgnoreDirectories` records the resolved ancestor list for troubleshooting. If a commit-scoped refresh includes `.gitignore` or `.cdidxignore` changes, `IndexCommandRunner` falls back to a full scan so newly ignored files are purged safely. Malformed ignore lines are reported as scan errors and skipped instead of aborting the whole run. On Windows, files and directories with Hidden or System attributes are rejected before language detection; clear those attributes before indexing project-owned sources because ignore rules cannot re-include them. +Scoped `--files` / `--commits` refreshes reuse the same path filter as full scans. Before scanning a nested project root, `FileIndexer` loads ignore files from the resolved ignore-rule root through each existing ancestor directory down to the project root's parent, then loads the project directory's own rules during the normal walk. Within each directory, `FileIndexer` loads `.gitignore` before `.cdidxignore`, appends both rule sets in that order, and honors later `!` patterns as re-includes. If an ancestor ignore directory cannot be read, scanning fails closed with a scan error instead of silently skipping those rules; `ScanFilesResult.AncestorIgnoreDirectories` records the resolved ancestor list for troubleshooting. If a commit-scoped refresh includes `.gitignore` or `.cdidxignore` changes, `IndexCommandRunner` falls back to a full scan so newly ignored files are purged safely. Malformed ignore lines are reported as scan errors and skipped instead of aborting the whole run. Directory symlinks default to `--follow-symlinks none`; `internal` follows only targets that resolve under the workspace root, and `all` preserves the broad historical behavior. Dangling symlinks are counted and warned separately. On Windows, files and directories with Hidden or System attributes are rejected before language detection; clear those attributes before indexing project-owned sources because ignore rules cannot re-include them. Incremental refreshes that mutate `fts_chunks` increment `codeindex_meta.fts_incremental_writes_since_optimize`. When the counter reaches `DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold`, the update path runs `INSERT INTO fts_chunks(fts_chunks) VALUES('optimize')`, resets the counter, and stamps `fts_last_optimized_at`. Users can run the same maintenance directly with `cdidx optimize --db ` or `cdidx index --optimize`; this may briefly hold the writer lock on large indexes. diff --git a/changelog.d/unreleased/1654.fixed.md b/changelog.d/unreleased/1654.fixed.md new file mode 100644 index 0000000000..e0be3aa26f --- /dev/null +++ b/changelog.d/unreleased/1654.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1654 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs +--- + +## English + +- **File reads now retry once when mtime changes during indexing (#1654)** — `FileIndexer` rechecks `LastWriteTimeUtc` after reading content and retries once before persisting metadata, reducing stale mtime/content races. + +## 日本語 + +- **index 中に mtime が変わったファイル読み取りを 1 回 retry するようになりました (#1654)** — `FileIndexer` は content 読み取り後に `LastWriteTimeUtc` を再確認し、metadata 保存前に 1 回 retry することで stale mtime/content race を減らします。 diff --git a/changelog.d/unreleased/1655.fixed.md b/changelog.d/unreleased/1655.fixed.md new file mode 100644 index 0000000000..673a88bdbb --- /dev/null +++ b/changelog.d/unreleased/1655.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1655 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs +--- + +## English + +- **Mid-scan deletes remain non-fatal purge candidates (#1655)** — files that disappear during scan probing are recorded as skipped non-indexable paths with warnings, preserving directory purge authority instead of leaving orphan rows behind. + +## 日本語 + +- **scan 中に削除されたファイルを非 fatal な purge 候補として扱います (#1655)** — probe 中に消えたファイルは warning 付きの non-indexable path として記録され、directory purge の authority を失わず orphan row を残しにくくなります。 diff --git a/changelog.d/unreleased/1656.fixed.md b/changelog.d/unreleased/1656.fixed.md new file mode 100644 index 0000000000..9da5cec9b8 --- /dev/null +++ b/changelog.d/unreleased/1656.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 1656 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs +--- + +## English + +- **Dangling symlinks are now reported distinctly (#1656)** — directory symlink targets that cannot be resolved are warned as dangling symlinks and counted as `dangling_symlinks_skipped` in full-scan JSON summaries. + +## 日本語 + +- **dangling symlink を個別に報告するようになりました (#1656)** — 解決できない directory symlink target は dangling symlink として warning され、full-scan JSON summary の `dangling_symlinks_skipped` に計上されます。 diff --git a/changelog.d/unreleased/1711.added.md b/changelog.d/unreleased/1711.added.md new file mode 100644 index 0000000000..d08fc3f132 --- /dev/null +++ b/changelog.d/unreleased/1711.added.md @@ -0,0 +1,20 @@ +--- +category: added +issues: + - 1711 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - src/CodeIndex/Cli/IndexCommandRunner.Parse.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - src/CodeIndex/Cli/IndexWatchRunner.cs + - DEVELOPER_GUIDE.md + - tests/CodeIndex.Tests/FileIndexerTests.cs +--- + +## English + +- **Added `--follow-symlinks none|internal|all` for directory scans (#1711)** — indexing now defaults to not following directory symlinks, can opt into workspace-internal targets, or can opt into all targets explicitly. + +## 日本語 + +- **directory scan 向けに `--follow-symlinks none|internal|all` を追加しました (#1711)** — indexing は既定で directory symlink を辿らず、workspace 内 target のみ、または全 target を明示 opt-in できます。 diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index d9b935293e..4cf0ba5369 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -60,7 +60,7 @@ public static class ConsoleUi private static readonly (string Command, string Usage)[] CommandUsageLines = [ - ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), + ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--duration-format ] [--max-file-bytes ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), ("hooks", "cdidx hooks [--project ] [--force] [--json]"), ("backfill-fold", "cdidx backfill-fold [--db ] [--json]"), ("optimize", "cdidx optimize [--db ] [--json]"), @@ -776,6 +776,7 @@ private static void PrintFlagReference(Action WriteHelpLine) Console.WriteLine(" --duration-format Index elapsed time format: `auto` (default), `seconds`, or `hms`; JSON keeps raw elapsed_ms"); WriteHelpLine(" --max-file-bytes Index only files up to this size (default: 4MiB; also honors CDIDX_MAX_FILE_BYTES; accepts K/M/G suffixes)"); WriteHelpLine(" --parallelism Full-scan extraction workers (default: CPU count capped at 16; also honors CDIDX_INDEX_PARALLELISM)"); + WriteHelpLine(" --follow-symlinks Directory symlink policy: none (default), internal, or all"); WriteHelpLine(" --include-symbol-kind [,] Keep only matching symbol kinds during indexing"); WriteHelpLine(" --exclude-symbol-kind [,] Drop matching symbol kinds during indexing"); Console.WriteLine(" --commits [id ...] Update only files changed in the specified git commits (preferred after commits)"); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs index a620857c23..f3b48ebc6b 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs @@ -15,7 +15,7 @@ private static int RunDryRun( CancellationToken cancellationToken) { var projectPath = options.ProjectPath!; - var dryIndexer = new FileIndexer(projectPath, ignoreCase, ignoreRuleRoot, options.MaxFileSizeBytes); + var dryIndexer = new FileIndexer(projectPath, ignoreCase, ignoreRuleRoot, options.MaxFileSizeBytes, directoryIgnoreCaseProbe: null, symlinkPolicy: options.SymlinkPolicy); IReadOnlyList dryCandidates; var errorList = new List(); var dryScanErrorKeys = new HashSet(StringComparer.Ordinal); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 808ecd7335..64ce35e144 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -1325,6 +1325,7 @@ void StopJsonHeartbeat() FilesScanned = files.Count, FilesSkipped = skipped, FilesPurged = purged, + DanglingSymlinksSkipped = scanResult.DanglingSymlinks.Count, Warnings = warnings, Errors = errors, SymbolsDroppedByKindFilter = symbolsDroppedByKindFilter, @@ -1370,6 +1371,7 @@ void StopJsonHeartbeat() Console.WriteLine(ConsoleUi.FormatSummaryLine("Symbols", $"{totalSymbols:N0}", indent: " ")); Console.WriteLine(ConsoleUi.FormatSummaryLine("Refs", $"{totalReferences:N0}", indent: " ")); if (skipped > 0) Console.WriteLine(ConsoleUi.FormatSummaryLine("Skipped", $"{skipped:N0} (unchanged)", indent: " ")); + if (scanResult.DanglingSymlinks.Count > 0) Console.WriteLine(ConsoleUi.FormatSummaryLine("Dangling symlinks", $"{scanResult.DanglingSymlinks.Count:N0} skipped", indent: " ")); if (options.Verbose && scanResult.UnknownExtensionFiles.Count > 0) { Console.WriteLine($" Unknown extension files: {scanResult.UnknownExtensionFiles.Count:N0}"); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs index 2d526f816b..b19a2f245d 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs @@ -14,7 +14,7 @@ public static partial class IndexCommandRunner [ "--db", "--data-dir", "--rebuild", "--verbose", "--json", "--dry-run", "--force", "--yes", "--watch", "--debounce", "--duration-format", "--max-file-bytes", - "--parallelism", + "--parallelism", "--follow-symlinks", "--commits", "--changed-between", "--files", "--solution", "--project", "--include-symbol-kind", "--exclude-symbol-kind", "--optimize", "--help", "--read-only", "--immutable", @@ -41,6 +41,7 @@ public static IndexCommandOptions ParseArgs(string[] args) var durationFormat = DurationOutputFormat.Auto; long? maxFileSizeBytes = ReadMaxFileSizeBytesFromEnvironment(); var parallelism = ReadIndexParallelismFromEnvironment(); + var symlinkPolicy = FileIndexer.SymlinkPolicy.None; string? easterEgg = null; int spinnerFlagCount = 0; bool randomSpinner = false; @@ -144,6 +145,12 @@ public static IndexCommandOptions ParseArgs(string[] args) case var option when option.StartsWith("--parallelism=", StringComparison.Ordinal): parallelism = ParseIndexParallelism(option["--parallelism=".Length..], parallelism, "--parallelism"); break; + case "--follow-symlinks" when i + 1 < args.Length: + symlinkPolicy = ParseSymlinkPolicy(args[++i], symlinkPolicy, ref parseError); + break; + case var option when option.StartsWith("--follow-symlinks=", StringComparison.Ordinal): + symlinkPolicy = ParseSymlinkPolicy(option["--follow-symlinks=".Length..], symlinkPolicy, ref parseError); + break; case "--commits": while (i + 1 < args.Length && !args[i + 1].StartsWith('-')) { @@ -289,10 +296,27 @@ public static IndexCommandOptions ParseArgs(string[] args) DurationFormat = durationFormat, MaxFileSizeBytes = maxFileSizeBytes, Parallelism = parallelism, + SymlinkPolicy = symlinkPolicy, SymbolKindFilter = SymbolKindFilter.Create(includeSymbolKinds, excludeSymbolKinds, symbolKindFilterError), }; } + private static FileIndexer.SymlinkPolicy ParseSymlinkPolicy(string value, FileIndexer.SymlinkPolicy fallback, ref string? parseError) + { + switch (value.Trim().ToLowerInvariant()) + { + case "none": + return FileIndexer.SymlinkPolicy.None; + case "internal": + return FileIndexer.SymlinkPolicy.Internal; + case "all": + return FileIndexer.SymlinkPolicy.All; + default: + parseError ??= $"invalid --follow-symlinks value '{value}': expected none, internal, or all"; + return fallback; + } + } + private static string BuildUnknownIndexOptionError(string token) { var name = TrimInlineValue(token); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index 395373bbd1..c68ca31d43 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -239,7 +239,7 @@ internal static int Run(string[] indexArgs, JsonSerializerOptions jsonOptions, C AddToGitExclude(options.ProjectPath, dbPath); var writer = new DbWriter(db); - var indexer = new FileIndexer(options.ProjectPath, ignoreCase, ignoreRuleRoot, options.MaxFileSizeBytes); + var indexer = new FileIndexer(options.ProjectPath, ignoreCase, ignoreRuleRoot, options.MaxFileSizeBytes, directoryIgnoreCaseProbe: null, symlinkPolicy: options.SymlinkPolicy); var currentHotspotFamilyMarkerFingerprints = GetHotspotFamilyMarkerFingerprints(indexer); var projectRoot = Path.GetFullPath(options.ProjectPath!); @@ -1172,6 +1172,7 @@ public sealed class IndexCommandOptions public DurationOutputFormat DurationFormat { get; init; } = DurationOutputFormat.Auto; public long? MaxFileSizeBytes { get; init; } public int Parallelism { get; init; } = IndexCommandRunner.DefaultIndexParallelism(); + public FileIndexer.SymlinkPolicy SymlinkPolicy { get; init; } = FileIndexer.SymlinkPolicy.None; public SymbolKindFilter SymbolKindFilter { get; init; } = SymbolKindFilter.Empty; } diff --git a/src/CodeIndex/Cli/IndexWatchRunner.cs b/src/CodeIndex/Cli/IndexWatchRunner.cs index 10748d0618..df04b773a3 100644 --- a/src/CodeIndex/Cli/IndexWatchRunner.cs +++ b/src/CodeIndex/Cli/IndexWatchRunner.cs @@ -201,6 +201,11 @@ private static List BuildSubRunArgs(IndexCommandOptions baseOptions) args.Add("--max-file-bytes"); args.Add(maxFileSizeBytes.ToString(CultureInfo.InvariantCulture)); } + if (baseOptions.SymlinkPolicy != FileIndexer.SymlinkPolicy.None) + { + args.Add("--follow-symlinks"); + args.Add(baseOptions.SymlinkPolicy.ToString().ToLowerInvariant()); + } return args; } diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 0159343e4c..d1f56bb050 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -159,6 +159,8 @@ internal sealed class IndexFullScanSummaryJsonResult public int FilesScanned { get; init; } public int FilesSkipped { get; init; } public int FilesPurged { get; init; } + [JsonPropertyName("dangling_symlinks_skipped")] + public int DanglingSymlinksSkipped { get; init; } public int Warnings { get; init; } public int Errors { get; init; } public int SymbolsDroppedByKindFilter { get; init; } diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index a3b737ee30..9c0834b978 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -15,6 +15,13 @@ namespace CodeIndex.Indexer; /// public class FileIndexer { + public enum SymlinkPolicy + { + None, + Internal, + All, + } + internal enum FileProbeStatus { Supported, @@ -49,7 +56,8 @@ public readonly record struct ScanFilesResult( IReadOnlySet CheckpointedDirectories, IReadOnlyList AncestorIgnoreDirectories, IReadOnlyList AttributePrunedDirectories, - IReadOnlyList NestedRepositories) + IReadOnlyList NestedRepositories, + IReadOnlyList DanglingSymlinks) { public bool HadErrors => Errors.Any(error => error.IsFatal); } @@ -413,6 +421,7 @@ internal static bool IsDefaultExcludedFileName(string fileName) private readonly Func> _enumerateFiles; private readonly Dictionary _directoryIgnoreCaseCache; private readonly long _maxFileSizeBytes; + private readonly SymlinkPolicy _symlinkPolicy; // Submodule working-tree paths declared in /.gitmodules, relative to // _projectRoot and slash-normalized. Used to override SkipDirs so that submodules // hosted under SkipDirs-named directories (e.g. vendor/foo) remain visible to the @@ -927,7 +936,8 @@ internal FileIndexer( string? ignoreRuleRoot, long? maxFileSizeBytes, Func? directoryIgnoreCaseProbe, - Func>? enumerateFiles = null) + Func>? enumerateFiles = null, + SymlinkPolicy symlinkPolicy = SymlinkPolicy.None) { _projectRoot = Path.GetFullPath(projectRoot); _ignoreRuleRoot = NormalizeIgnoreRuleRoot(ignoreRuleRoot); @@ -937,6 +947,7 @@ internal FileIndexer( _enumerateFiles = enumerateFiles ?? (dir => Directory.EnumerateFiles(LongPath.EnsureWindowsPrefix(dir))); _directoryIgnoreCaseCache = new Dictionary(StringComparer.Ordinal); _maxFileSizeBytes = ResolveMaxFileSizeBytes(maxFileSizeBytes); + _symlinkPolicy = symlinkPolicy; var pathComparer = _ignoreCase ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; (_submodulePaths, _submoduleAncestorPaths) = LoadGitSubmodulePaths(_ignoreRuleRoot, _projectRoot, pathComparer); } @@ -1288,6 +1299,72 @@ private static bool HasSkippedAttributes(string path) } } + private static bool IsReparsePoint(string path) + { + try + { + return (File.GetAttributes(LongPath.EnsureWindowsPrefix(path)) & FileAttributes.ReparsePoint) != 0; + } + catch (FileNotFoundException) + { + return false; + } + catch (DirectoryNotFoundException) + { + return false; + } + catch (UnauthorizedAccessException) + { + return false; + } + catch (IOException) + { + return false; + } + } + + private bool ShouldSkipDirectoryLink(string subDir, List errors, HashSet danglingSymlinks) + { + if (!IsReparsePoint(subDir)) + return HasSkippedAttributes(subDir); + + var relative = ToRelativePath(subDir); + DirectoryInfo info = new(LongPath.EnsureWindowsPrefix(subDir)); + FileSystemInfo? target; + try + { + target = info.ResolveLinkTarget(returnFinalTarget: true); + } + catch (FileNotFoundException) + { + target = null; + } + catch (DirectoryNotFoundException) + { + target = null; + } + catch (IOException) + { + target = null; + } + + if (target?.FullName is not { Length: > 0 } targetPath || !Directory.Exists(LongPath.EnsureWindowsPrefix(targetPath))) + { + danglingSymlinks.Add(relative); + errors.Add(new ScanError(relative, "Skipped dangling symlink because its target could not be resolved.", ScanIssueSeverity.Warning)); + return true; + } + + if (_symlinkPolicy == SymlinkPolicy.All) + return false; + + if (_symlinkPolicy == SymlinkPolicy.Internal && IsPathEqualOrParent(_projectRoot, targetPath)) + return false; + + errors.Add(new ScanError(relative, $"Skipped symlinked directory outside the active symlink policy: {targetPath}", ScanIssueSeverity.Warning)); + return true; + } + internal static FileProbeStatus GetFileIndexability(string filePath) { if (OperatingSystem.IsWindows() && IsWindowsDevicePath(filePath)) @@ -1699,12 +1776,14 @@ internal ScanFilesResult ScanFilesDetailed( : new HashSet(StringComparer.Ordinal); var attributePrunedDirectories = new HashSet(StringComparer.Ordinal); var nestedRepositories = new HashSet(StringComparer.Ordinal); + var danglingSymlinks = new HashSet(StringComparer.Ordinal); var visitedFileIdentities = new HashSet(); + var visitedDirectories = new HashSet(StringComparer.Ordinal) { NormalizePathForComparison(_projectRoot) }; var fullyScanned = true; var preloadResult = LoadAncestorIgnoreRules(errors, ref fullyScanned); if (preloadResult.IgnoreRulesAvailable) { - ScanDirectory(_projectRoot, files, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, activeCheckpointedDirectories, attributePrunedDirectories, nestedRepositories, visitedFileIdentities, preloadResult.Rules, isProjectRoot: true, continueOnError, cancellationToken); + ScanDirectory(_projectRoot, files, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, activeCheckpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, preloadResult.Rules, isProjectRoot: true, continueOnError, cancellationToken); } return new ScanFilesResult( files, @@ -1717,7 +1796,8 @@ internal ScanFilesResult ScanFilesDetailed( activeCheckpointedDirectories.Concat(fullyScannedDirectories).ToHashSet(StringComparer.Ordinal), _ancestorIgnoreDirectories.ToList(), attributePrunedDirectories.ToList(), - nestedRepositories.OrderBy(path => path, StringComparer.Ordinal).ToList()); + nestedRepositories.OrderBy(path => path, StringComparer.Ordinal).ToList(), + danglingSymlinks.OrderBy(path => path, StringComparer.Ordinal).ToList()); } private bool ScanDirectory( @@ -1732,7 +1812,9 @@ private bool ScanDirectory( HashSet checkpointedDirectories, HashSet attributePrunedDirectories, HashSet nestedRepositories, + HashSet danglingSymlinks, HashSet visitedFileIdentities, + HashSet visitedDirectories, IgnoreRuleSet activeIgnoreRules, bool isProjectRoot = false, bool continueOnError = true, @@ -1752,7 +1834,7 @@ private bool ScanDirectory( return true; } - return EnumerateDirectory(dir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, visitedFileIdentities, activeIgnoreRules, continueOnError, cancellationToken); + return EnumerateDirectory(dir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, activeIgnoreRules, continueOnError, cancellationToken); } private bool IsNestedGitRepository(string dir) @@ -1776,7 +1858,9 @@ private bool EnumerateDirectory( HashSet checkpointedDirectories, HashSet attributePrunedDirectories, HashSet nestedRepositories, + HashSet danglingSymlinks, HashSet visitedFileIdentities, + HashSet visitedDirectories, IgnoreRuleSet inheritedIgnoreRules, bool continueOnError, CancellationToken cancellationToken = default) @@ -1929,6 +2013,21 @@ private bool EnumerateDirectory( // 子サブツリー失敗が sibling file purge の authority を奪ってはいけない。 listedDirectories.Add(ToRelativePath(dir)); + foreach (var enumeratedEntry in Directory.EnumerateFileSystemEntries(LongPath.EnsureWindowsPrefix(dir))) + { + cancellationToken.ThrowIfCancellationRequested(); + var entry = LongPath.RemoveWindowsPrefix(enumeratedEntry); + if (!IsReparsePoint(entry) || Directory.Exists(LongPath.EnsureWindowsPrefix(entry))) + continue; + + var relativeEntry = ToRelativePath(entry); + danglingSymlinks.Add(relativeEntry); + errors.Add(new ScanError(relativeEntry, "Skipped dangling symlink because its target could not be resolved.", ScanIssueSeverity.Warning)); + listedDirectories.Add(relativeEntry); + fullyScannedDirectories.Add(relativeEntry); + attributePrunedDirectories.Add(relativeEntry); + } + foreach (var enumeratedSubDir in Directory.EnumerateDirectories(LongPath.EnsureWindowsPrefix(dir))) { cancellationToken.ThrowIfCancellationRequested(); @@ -1966,16 +2065,27 @@ private bool EnumerateDirectory( // drive root 走査で OS 管理 cache に降りないよう Hidden/System ディレクトリもスキップする。 // skip したディレクトリ自身を listed 扱い（immediate parent purge 用）かつ prune prefix として // 記録することで、以前の実行でできた深い子孫エントリも purge walker が確実に削除できる。 - if (HasSkippedAttributes(subDir)) + if (ShouldSkipDirectoryLink(subDir, errors, danglingSymlinks)) + { + var subRelative = ToRelativePath(subDir); + listedDirectories.Add(subRelative); + fullyScannedDirectories.Add(subRelative); + attributePrunedDirectories.Add(subRelative); + continue; + } + + var resolvedSubDir = NormalizePathForComparison(subDir); + if (!visitedDirectories.Add(resolvedSubDir)) { var subRelative = ToRelativePath(subDir); + errors.Add(new ScanError(subRelative, "Skipped symlinked directory because its resolved target was already scanned.", ScanIssueSeverity.Warning)); listedDirectories.Add(subRelative); fullyScannedDirectories.Add(subRelative); attributePrunedDirectories.Add(subRelative); continue; } - var childFullyScanned = ScanDirectory(subDir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, visitedFileIdentities, activeIgnoreRules, continueOnError: continueOnError, cancellationToken: cancellationToken); + var childFullyScanned = ScanDirectory(subDir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, activeIgnoreRules, continueOnError: continueOnError, cancellationToken: cancellationToken); fullyScanned &= childFullyScanned; if (!continueOnError && !childFullyScanned) break; @@ -2594,46 +2704,52 @@ public static string NormalizeIndexPath(string path) long sizeBytes; DateTime modifiedUtc; var ioPath = LongPath.EnsureWindowsPrefix(absolutePath); - using (var stream = new FileStream( - ioPath, - FileMode.Open, - FileAccess.Read, - FileShare.Read, - bufferSize: 4096, - useAsync: false)) - { - var initialLength = stream.Length; - if (initialLength > _maxFileSizeBytes) - throw new FileTooLargeSkippedException( - normalizedRelativePath, - initialLength, - _maxFileSizeBytes, - BuildFileTooLargeMessage(initialLength, grewDuringRead: false)); - - // Pre-size the accumulator to the observed length but cap initial capacity - // at the configured limit so a tampered Length cannot force a huge up-front allocation. - // 初期容量は観測した長さに合わせるが設定上限で打ち切り、Length を偽装 - // されても巨大な事前確保を起こさないようにする。 - var initialCapacity = (int)Math.Min(initialLength, _maxFileSizeBytes); - using var accumulator = new MemoryStream(initialCapacity); - var buffer = new byte[81920]; - long total = 0; - int read; - while ((read = stream.Read(buffer, 0, buffer.Length)) > 0) - { - cancellationToken.ThrowIfCancellationRequested(); - total += read; - if (total > _maxFileSizeBytes) + for (var attempt = 0; ; attempt++) + { + var modifiedBeforeRead = File.GetLastWriteTimeUtc(ioPath); + using (var stream = new FileStream( + ioPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 4096, + useAsync: false)) + { + var initialLength = stream.Length; + if (initialLength > _maxFileSizeBytes) throw new FileTooLargeSkippedException( normalizedRelativePath, - total, + initialLength, _maxFileSizeBytes, - BuildFileTooLargeMessage(total, grewDuringRead: true)); - accumulator.Write(buffer, 0, read); + BuildFileTooLargeMessage(initialLength, grewDuringRead: false)); + + // Pre-size the accumulator to the observed length but cap initial capacity + // at the configured limit so a tampered Length cannot force a huge up-front allocation. + // 初期容量は観測した長さに合わせるが設定上限で打ち切り、Length を偽装 + // されても巨大な事前確保を起こさないようにする。 + var initialCapacity = (int)Math.Min(initialLength, _maxFileSizeBytes); + using var accumulator = new MemoryStream(initialCapacity); + var buffer = new byte[81920]; + long total = 0; + int read; + while ((read = stream.Read(buffer, 0, buffer.Length)) > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + total += read; + if (total > _maxFileSizeBytes) + throw new FileTooLargeSkippedException( + normalizedRelativePath, + total, + _maxFileSizeBytes, + BuildFileTooLargeMessage(total, grewDuringRead: true)); + accumulator.Write(buffer, 0, read); + } + bytes = accumulator.ToArray(); + sizeBytes = total; } - bytes = accumulator.ToArray(); - sizeBytes = total; modifiedUtc = File.GetLastWriteTimeUtc(ioPath); + if (modifiedUtc == modifiedBeforeRead || attempt > 0) + break; } // Compute the checksum on the byte stream after collapsing CRLF / CR to LF so diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 867fe0b024..47058f71a9 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -3026,6 +3026,116 @@ static IEnumerable DeleteBeforeProbe(string path) } } + [Fact] + public void ScanFilesDetailed_DanglingDirectorySymlink_RecordsWarningAndCount() + { + if (OperatingSystem.IsWindows()) + return; + + var tempDir = Path.Combine(Path.GetTempPath(), $"cdidx-dangling-symlink-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + try + { + var linkPath = Path.Combine(tempDir, "missing-link"); + Directory.CreateSymbolicLink(linkPath, Path.Combine(tempDir, "missing-target")); + + var result = new FileIndexer(tempDir).ScanFilesDetailed(); + + Assert.Contains("missing-link", result.DanglingSymlinks); + Assert.Contains( + result.Errors, + error => error.Path == "missing-link" + && error.Severity == FileIndexer.ScanIssueSeverity.Warning + && error.Message.Contains("dangling symlink", StringComparison.OrdinalIgnoreCase)); + Assert.False(result.HadErrors); + } + finally + { + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + } + + [Fact] + public void ScanFiles_FollowSymlinksInternal_SkipsOutOfTreeDirectorySymlink() + { + if (OperatingSystem.IsWindows()) + return; + + var tempDir = Path.Combine(Path.GetTempPath(), $"cdidx-symlink-policy-{Guid.NewGuid():N}"); + var externalDir = Path.Combine(Path.GetTempPath(), $"cdidx-symlink-external-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + Directory.CreateDirectory(externalDir); + try + { + File.WriteAllText(Path.Combine(externalDir, "external.py"), "print('external')\n"); + var linkPath = Path.Combine(tempDir, "external"); + Directory.CreateSymbolicLink(linkPath, externalDir); + + var indexer = new FileIndexer( + tempDir, + ignoreCase: false, + ignoreRuleRoot: null, + maxFileSizeBytes: null, + directoryIgnoreCaseProbe: null, + symlinkPolicy: FileIndexer.SymlinkPolicy.Internal); + + var result = indexer.ScanFilesDetailed(); + + Assert.Empty(result.Files); + Assert.Contains( + result.Errors, + error => error.Path == "external" + && error.Severity == FileIndexer.ScanIssueSeverity.Warning + && error.Message.Contains("symlinked directory", StringComparison.OrdinalIgnoreCase)); + } + finally + { + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + if (Directory.Exists(externalDir)) + Directory.Delete(externalDir, true); + } + } + + [Fact] + public void ScanFiles_FollowSymlinksInternal_FollowsInTreeDirectorySymlinkOnce() + { + if (OperatingSystem.IsWindows()) + return; + + var tempDir = Path.Combine(Path.GetTempPath(), $"cdidx-symlink-internal-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + try + { + var targetDir = Path.Combine(tempDir, "src"); + Directory.CreateDirectory(targetDir); + File.WriteAllText(Path.Combine(targetDir, "app.py"), "print('app')\n"); + Directory.CreateSymbolicLink(Path.Combine(tempDir, "src-link"), targetDir); + + var indexer = new FileIndexer( + tempDir, + ignoreCase: false, + ignoreRuleRoot: null, + maxFileSizeBytes: null, + directoryIgnoreCaseProbe: null, + symlinkPolicy: FileIndexer.SymlinkPolicy.Internal); + + var files = indexer.ScanFiles() + .Select(path => Path.GetRelativePath(tempDir, path).Replace('\\', '/')) + .OrderBy(path => path, StringComparer.Ordinal) + .ToList(); + + Assert.Single(files); + Assert.Contains(files[0], new[] { "src/app.py", "src-link/app.py" }); + } + finally + { + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + } + [Fact] public void ScanFiles_DescendsIntoSubmoduleHostedUnderSkipDir() { From 3016f907d57cc4e1daa64b59766a9c310eede0b3 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 13:33:06 +0900 Subject: [PATCH 2/2] Fix symlink traversal cycle guard (#1711) --- src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 27 ++++++++++++- tests/CodeIndex.Tests/FileIndexerTests.cs | 38 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index 9c0834b978..ffb22677be 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -1365,6 +1365,31 @@ private bool ShouldSkipDirectoryLink(string subDir, List errors, Hash return true; } + private static string GetDirectoryTraversalIdentity(string directory) + { + if (!IsReparsePoint(directory)) + return directory; + + try + { + DirectoryInfo info = new(LongPath.EnsureWindowsPrefix(directory)); + var target = info.ResolveLinkTarget(returnFinalTarget: true); + if (target?.FullName is { Length: > 0 } targetPath) + return targetPath; + } + catch (FileNotFoundException) + { + } + catch (DirectoryNotFoundException) + { + } + catch (IOException) + { + } + + return directory; + } + internal static FileProbeStatus GetFileIndexability(string filePath) { if (OperatingSystem.IsWindows() && IsWindowsDevicePath(filePath)) @@ -2074,7 +2099,7 @@ private bool EnumerateDirectory( continue; } - var resolvedSubDir = NormalizePathForComparison(subDir); + var resolvedSubDir = NormalizePathForComparison(GetDirectoryTraversalIdentity(subDir)); if (!visitedDirectories.Add(resolvedSubDir)) { var subRelative = ToRelativePath(subDir); diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 47058f71a9..ffb969f3e2 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -3136,6 +3136,44 @@ public void ScanFiles_FollowSymlinksInternal_FollowsInTreeDirectorySymlinkOnce() } } + [Fact] + public void ScanFiles_FollowSymlinksInternal_SkipsCycleToProjectRoot() + { + if (OperatingSystem.IsWindows()) + return; + + var tempDir = Path.Combine(Path.GetTempPath(), $"cdidx-symlink-cycle-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + try + { + File.WriteAllText(Path.Combine(tempDir, "app.py"), "print('app')\n"); + Directory.CreateSymbolicLink(Path.Combine(tempDir, "self"), tempDir); + + var indexer = new FileIndexer( + tempDir, + ignoreCase: false, + ignoreRuleRoot: null, + maxFileSizeBytes: null, + directoryIgnoreCaseProbe: null, + symlinkPolicy: FileIndexer.SymlinkPolicy.Internal); + + var result = indexer.ScanFilesDetailed(); + + Assert.Single(result.Files); + Assert.Contains( + result.Errors, + error => error.Path == "self" + && error.Severity == FileIndexer.ScanIssueSeverity.Warning + && error.Message.Contains("already scanned", StringComparison.OrdinalIgnoreCase)); + Assert.False(result.HadErrors); + } + finally + { + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + } + [Fact] public void ScanFiles_DescendsIntoSubmoduleHostedUnderSkipDir() {