diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index ac673479ba..2e25dbca08 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -188,7 +188,7 @@ case-sensitive equality so format drift is visible instead of silently accepted. -- File metadata files ( id INTEGER PRIMARY KEY AUTOINCREMENT, - path TEXT NOT NULL UNIQUE, -- relative path from project root + path TEXT NOT NULL UNIQUE, -- relative path from project root, slash-normalized and Unicode NFC lang TEXT, -- detected language (e.g. "python") size INTEGER, -- file size in bytes lines INTEGER, -- line count diff --git a/USER_GUIDE.md b/USER_GUIDE.md index cbbcdbd110..98b7af0aa9 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -576,7 +576,7 @@ By default, `cdidx index` stores the database in `/.cdidx/codeindex `--watch` keeps the process alive after the initial scan and rebuilds the index incrementally as files are created, edited, renamed, or deleted. It uses `FileSystemWatcher` (FSEvents on macOS, inotify on Linux, ReadDirectoryChangesW on Windows), debounces bursts of events (`--debounce `, default 500 ms) into a single `--files` update, releases the per-DB index lock between batches so other `cdidx` commands can still query, and falls back to a full incremental rescan if the watcher buffer overflows. With `--json` it streams `status: "watching" / "updated" / "rescanned" / "overflow" / "stopped"` lifecycle events to stdout; otherwise it writes `[watch] …` summaries to stderr. Stop the loop with Ctrl+C (or SIGTERM); the final exit code is `0` for a clean stop. `--watch` cannot be combined with `--commits`, `--files`, or `--dry-run` — the loop already drives continuous incremental updates. -Indexing keeps the built-in skip lists (`node_modules`, `bin`, `obj`, lockfiles, etc.) and also honors user `.gitignore` plus optional `.cdidxignore` rules across full scans, `--files`, and `--commits` updates. Ignore files are read as UTF-8, so non-ASCII patterns behave the same across platforms. On Windows, paths marked with the Hidden or System attribute are skipped before language detection so broad scans do not enter OS-owned caches such as `System Volume Information` or `$Recycle.Bin`; clear those attributes before indexing project-owned source files because ignore rules only exclude additional paths. When the project is inside Git, ignore matching follows the repository's `core.ignorecase` setting, even when the indexed project path is a subdirectory inside that repo; repo-root and other ancestor `.gitignore` files above that subdirectory still apply, and `--commits` resolves changed paths from the repository root before narrowing them back to the indexed project root. `**` only gets Git-style special handling in the documented path forms rather than as an unrestricted cross-directory wildcard. If an update refresh includes ignore-file changes, cdidx automatically falls back to a full scan so newly ignored files are purged safely. Invalid ignore lines are skipped with a warning instead of aborting the whole run, while unreadable ignore files fail closed for that directory scope so cdidx does not index with incomplete rules. +Indexing keeps the built-in skip lists (`node_modules`, `bin`, `obj`, lockfiles, etc.) and also honors user `.gitignore` plus optional `.cdidxignore` rules across full scans, `--files`, and `--commits` updates. A project-root `.codeindex/.cdidxignore` is also loaded as a workspace-scoped ignore file, which lets multi-workspace manifests keep local cdidx-only ignore rules out of the repository root. Ignore files are read as UTF-8, so non-ASCII patterns behave the same across platforms. On Windows, paths marked with the Hidden or System attribute are skipped before language detection so broad scans do not enter OS-owned caches such as `System Volume Information` or `$Recycle.Bin`; clear those attributes before indexing project-owned source files because ignore rules only exclude additional paths. When the project is inside Git, ignore matching follows the repository's `core.ignorecase` setting, even when the indexed project path is a subdirectory inside that repo; repo-root and other ancestor `.gitignore` files above that subdirectory still apply, and `--commits` resolves changed paths from the repository root before narrowing them back to the indexed project root. Nested directories that contain their own `.git` directory or gitfile are treated as repository boundaries and skipped by default. Indexed file paths are stored in Unicode NFC form so composed and decomposed path spellings match across platforms. `**` only gets Git-style special handling in the documented path forms rather than as an unrestricted cross-directory wildcard. If an update refresh includes ignore-file changes, cdidx automatically falls back to a full scan so newly ignored files are purged safely. Invalid ignore lines are skipped with a warning instead of aborting the whole run, while unreadable ignore files fail closed for that directory scope so cdidx does not index with incomplete rules. Default output: @@ -2510,7 +2510,7 @@ cdidx ./myproject --json インデックスの問題をデバッグしたり、どのファイルが実際に処理されたかを確認するのに便利です。 -既定では `cdidx index` は DB を `/.cdidx/codeindex.db` に置きます。組み込みのスキップ対象 (`node_modules`、`bin`、`obj`、lockfile など) は常に除外され、さらにユーザーの `.gitignore` と任意の `.cdidxignore` もフルスキャン、`--files`、`--commits` の更新経路すべてで尊重されます。ignore ファイルは UTF-8 として読み込むため、非 ASCII のパターンも platform 間で同じように動作します。同じディレクトリでは `.gitignore` を先に読み、その後で `.cdidxignore` を読みます。後のルールは加算的に適用されるため、`.cdidxignore` の `!` パターンで同じディレクトリスコープの `.gitignore` が先に除外した path を再包含できます。Windows では Hidden または System 属性が付いたパスを言語検出前にスキップするため、広い範囲を走査しても `System Volume Information` や `$Recycle.Bin` のような OS 管理 cache には入りません。プロジェクト所有のソースを索引したい場合は、ignore ルールでは再包含できないため先にそれらの属性を外してください。Git 管理下では ignore の大文字小文字判定は OS 名ではなくリポジトリの `core.ignorecase` に従い、repo 配下の subdirectory を project root にした場合でも同じ設定を引き継ぎます。さらに、その subdirectory より上位にある repo-root などの `.gitignore` も有効で、`--commits` の changed path も一度リポジトリルート基準で解決してから project root 配下へ絞り込みます。`**` も無制限のクロスディレクトリ wildcard ではなく Git の path-form globstar でのみ特別扱いされます。`--commits` 実行中に ignore ファイル自体が変わっていた場合は、新しく無視対象になったファイルを安全にパージするため自動でフルスキャンへフォールバックします。不正な ignore 行は警告してスキップし、index 全体は中断しません。逆に ignore ファイル自体が読めない場合は、そのディレクトリ範囲を fail-closed で扱い、不完全なルールのまま index しません。 +既定では `cdidx index` は DB を `/.cdidx/codeindex.db` に置きます。組み込みのスキップ対象 (`node_modules`、`bin`、`obj`、lockfile など) は常に除外され、さらにユーザーの `.gitignore` と任意の `.cdidxignore` もフルスキャン、`--files`、`--commits` の更新経路すべてで尊重されます。project root の `.codeindex/.cdidxignore` も workspace scope の ignore ファイルとして読み込むため、multi-workspace manifest 用の cdidx 専用ルールを repository root に置かずに管理できます。ignore ファイルは UTF-8 として読み込むため、非 ASCII のパターンも platform 間で同じように動作します。同じディレクトリでは `.gitignore` を先に読み、その後で `.cdidxignore` を読みます。後のルールは加算的に適用されるため、`.cdidxignore` の `!` パターンで同じディレクトリスコープの `.gitignore` が先に除外した path を再包含できます。Windows では Hidden または System 属性が付いたパスを言語検出前にスキップするため、広い範囲を走査しても `System Volume Information` や `$Recycle.Bin` のような OS 管理 cache には入りません。プロジェクト所有のソースを索引したい場合は、ignore ルールでは再包含できないため先にそれらの属性を外してください。Git 管理下では ignore の大文字小文字判定は OS 名ではなくリポジトリの `core.ignorecase` に従い、repo 配下の subdirectory を project root にした場合でも同じ設定を引き継ぎます。さらに、その subdirectory より上位にある repo-root などの `.gitignore` も有効で、`--commits` の changed path も一度リポジトリルート基準で解決してから project root 配下へ絞り込みます。独自の `.git` ディレクトリまたは gitfile を持つ nested directory は repository boundary として扱い、既定でスキップします。index に保存する file path は Unicode NFC へ正規化するため、合成済み・分解済みの path 表記が platform をまたいでも一致します。`**` も無制限のクロスディレクトリ wildcard ではなく Git の path-form globstar でのみ特別扱いされます。`--commits` 実行中に ignore ファイル自体が変わっていた場合は、新しく無視対象になったファイルを安全にパージするため自動でフルスキャンへフォールバックします。不正な ignore 行は警告してスキップし、index 全体は中断しません。逆に ignore ファイル自体が読めない場合は、そのディレクトリ範囲を fail-closed で扱い、不完全なルールのまま index しません。 古い `.cdidx/codeindex.db` を Unicode-aware な `--exact` に上げたいだけなら、フル rebuild は不要です: diff --git a/changelog.d/unreleased/1832.fixed.md b/changelog.d/unreleased/1832.fixed.md new file mode 100644 index 0000000000..4f9465f27b --- /dev/null +++ b/changelog.d/unreleased/1832.fixed.md @@ -0,0 +1,15 @@ +--- +category: fixed +issues: + - 1832 +affected: + - tests/CodeIndex.Tests/FileIndexerTests.cs +--- + +## English + +- **Case-varied built-in skip directory names stay excluded on case-insensitive scans (#1832)** — regression coverage now locks in that directories such as `Node_Modules/` match the built-in `node_modules` skip rule. + +## 日本語 + +- **大文字小文字が異なる組み込み skip directory 名も case-insensitive scan では除外され続けます (#1832)** — `Node_Modules/` のような directory が組み込みの `node_modules` skip ルールに一致することを回帰テストで固定しました。 diff --git a/changelog.d/unreleased/2022.fixed.md b/changelog.d/unreleased/2022.fixed.md new file mode 100644 index 0000000000..a798f038f4 --- /dev/null +++ b/changelog.d/unreleased/2022.fixed.md @@ -0,0 +1,21 @@ +--- +category: fixed +issues: + - 2022 + - 2023 + - 1947 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - src/CodeIndex/Cli/DbPathResolver.cs + - src/CodeIndex/Cli/IndexCommandRunner.cs + - USER_GUIDE.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Workspace-local ignore files, nested repository boundaries, and Unicode path spelling are now handled consistently (#2022, #2023, #1947)** — indexing loads `.codeindex/.cdidxignore`, skips nested `.git` repositories by default, and stores/looks up indexed paths in Unicode NFC form. + +## 日本語 + +- **workspace local の ignore ファイル、nested repository 境界、Unicode path 表記を一貫して扱うようになりました (#2022, #2023, #1947)** — indexing は `.codeindex/.cdidxignore` を読み込み、nested `.git` repository を既定でスキップし、index path の保存・lookup を Unicode NFC 形式に統一します。 diff --git a/src/CodeIndex/Cli/DbPathResolver.cs b/src/CodeIndex/Cli/DbPathResolver.cs index 6419186748..c59310363b 100644 --- a/src/CodeIndex/Cli/DbPathResolver.cs +++ b/src/CodeIndex/Cli/DbPathResolver.cs @@ -109,7 +109,7 @@ public static string ResolveQueryFilePath(string dbPath, string filePath, bool d if (string.IsNullOrWhiteSpace(filePath)) return filePath; - var normalizedFilePath = FileIndexer.NormalizePathSeparators(filePath); + var normalizedFilePath = FileIndexer.NormalizeIndexPath(filePath); if (!Path.IsPathRooted(filePath)) return normalizedFilePath; @@ -122,7 +122,7 @@ public static string ResolveQueryFilePath(string dbPath, string filePath, bool d if (!IsUnderDirectory(fullProjectRoot, fullFilePath)) return normalizedFilePath; - return FileIndexer.NormalizePathSeparators(Path.GetRelativePath(fullProjectRoot, fullFilePath)); + return FileIndexer.NormalizeIndexPath(Path.GetRelativePath(fullProjectRoot, fullFilePath)); } /// diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index d411f5da32..c5a6cd54b7 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -1769,12 +1769,13 @@ void ThrowIfUpdateCancelled() StartUpdateSpinnerIfNeeded(); currentUpdatePath = relPath; var absPath = Path.Combine(projectRoot, relPath.Replace('/', Path.DirectorySeparatorChar)); + var dbPath = FileIndexer.NormalizeIndexPath(relPath); var fileBatchMarked = false; try { if (!File.Exists(LongPath.EnsureWindowsPrefix(absPath))) { - if (!writer.HasFileAtPath(relPath)) + if (!writer.HasFileAtPath(dbPath)) { skipped++; WriteUpdateVerboseStatus($" [SKIP] {relPath} (not in DB)"); @@ -1783,7 +1784,7 @@ void ThrowIfUpdateCancelled() DemoteReadinessOnce(); using var deleteTxn = writer.BeginTransaction(); - if (writer.DeleteFileByPath(relPath)) + if (writer.DeleteFileByPath(dbPath)) { WriteProjectRootOnce(); deleteTxn.Commit(); @@ -1815,7 +1816,7 @@ void ThrowIfUpdateCancelled() continue; } - if (!writer.HasFileAtPath(relPath)) + if (!writer.HasFileAtPath(dbPath)) { skipped++; if (options.Verbose && !options.Json && !options.Quiet) @@ -1829,7 +1830,7 @@ void ThrowIfUpdateCancelled() DemoteReadinessOnce(); using var deleteTxn = writer.BeginTransaction(); - if (writer.DeleteFileByPath(relPath)) + if (writer.DeleteFileByPath(dbPath)) { WriteProjectRootOnce(); deleteTxn.Commit(); @@ -1869,11 +1870,11 @@ void ThrowIfUpdateCancelled() ResumeUpdateSpinnerAfterConsoleWrite(); } - if (writer.HasFileAtPath(relPath)) + if (writer.HasFileAtPath(dbPath)) { DemoteReadinessOnce(); using var deleteTxn = writer.BeginTransaction(); - if (writer.DeleteFileByPath(relPath)) + if (writer.DeleteFileByPath(dbPath)) { WriteProjectRootOnce(); deleteTxn.Commit(); @@ -1908,11 +1909,11 @@ void ThrowIfUpdateCancelled() if (indexability != FileIndexer.FileProbeStatus.Supported || detection.Status != FileIndexer.FileProbeStatus.Supported) { - if (!writer.HasFileAtPath(relPath)) + if (!writer.HasFileAtPath(dbPath)) { using var purgeTxn = writer.BeginTransaction(); var purged = projectRootWritten - ? writer.PurgeStaleFilesSharingDirectoryAndStem(projectRoot, relPath) + ? writer.PurgeStaleFilesSharingDirectoryAndStem(projectRoot, dbPath) : 0; if (purged > 0) { @@ -1943,7 +1944,7 @@ void ThrowIfUpdateCancelled() DemoteReadinessOnce(); using var deleteTxn = writer.BeginTransaction(); - if (writer.DeleteFileByPath(relPath)) + if (writer.DeleteFileByPath(dbPath)) { WriteProjectRootOnce(); deleteTxn.Commit(); @@ -1981,7 +1982,7 @@ void ThrowIfUpdateCancelled() ResumeUpdateSpinnerAfterConsoleWrite(); } - if (!writer.HasFileAtPath(relPath)) + if (!writer.HasFileAtPath(dbPath)) { skipped++; continue; @@ -1989,7 +1990,7 @@ void ThrowIfUpdateCancelled() DemoteReadinessOnce(); using var deleteTxn = writer.BeginTransaction(); - if (writer.DeleteFileByPath(relPath)) + if (writer.DeleteFileByPath(dbPath)) { WriteProjectRootOnce(); deleteTxn.Commit(); @@ -2131,11 +2132,11 @@ void ThrowIfUpdateCancelled() ResumeUpdateSpinnerAfterConsoleWrite(); } - if (writer.HasFileAtPath(relPath)) + if (writer.HasFileAtPath(dbPath)) { DemoteReadinessOnce(); using var deleteTxn = writer.BeginTransaction(); - if (writer.DeleteFileByPath(relPath)) + if (writer.DeleteFileByPath(dbPath)) { WriteProjectRootOnce(); deleteTxn.Commit(); @@ -2200,11 +2201,11 @@ void ThrowIfUpdateCancelled() ResumeUpdateSpinnerAfterConsoleWrite(); } - if (writer.HasFileAtPath(relPath)) + if (writer.HasFileAtPath(dbPath)) { DemoteReadinessOnce(); using var deleteTxn = writer.BeginTransaction(); - if (writer.DeleteFileByPath(relPath)) + if (writer.DeleteFileByPath(dbPath)) { WriteProjectRootOnce(); deleteTxn.Commit(); @@ -3236,26 +3237,30 @@ void ThrowIfFullScanCancelled(int filesProcessed, int? filesTotal) purgeCts = ConsoleUi.StartSpinner("Cleaning up stale entries...", spinnerFrames); var purged = 0; var retainedPaths = files - .Select(path => FileIndexer.NormalizePathSeparators(Path.GetRelativePath(projectRoot, path))) + .Select(path => FileIndexer.NormalizeIndexPath(Path.GetRelativePath(projectRoot, path))) .ToHashSet(StringComparer.Ordinal); if (scanResult.HadErrors) { SaveScanCheckpoint(scanCheckpointPath, currentHeadForCheckpoint, scanResult.CheckpointedDirectories); - retainedPaths.UnionWith(scanResult.ProbeFailedFilePaths); + retainedPaths.UnionWith(scanResult.ProbeFailedFilePaths.Select(FileIndexer.NormalizeIndexPath)); foreach (var relPath in scanResult.NonIndexablePaths) { - if (!writer.HasFileAtPath(relPath)) + var dbPath = FileIndexer.NormalizeIndexPath(relPath); + if (!writer.HasFileAtPath(dbPath)) continue; - if (writer.DeleteFileByPath(relPath)) + if (writer.DeleteFileByPath(dbPath)) purged++; } var authoritativeDirectories = scanResult.ListedDirectories + .Select(FileIndexer.NormalizeIndexPath) .ToHashSet(StringComparer.Ordinal); var attributePrunedDirectories = scanResult.AttributePrunedDirectories + .Select(FileIndexer.NormalizeIndexPath) .ToHashSet(StringComparer.Ordinal); + attributePrunedDirectories.UnionWith(scanResult.NestedRepositories.Select(FileIndexer.NormalizeIndexPath)); purged += writer.PurgeFilesOutsideRetainedSetWithinListedDirectories(retainedPaths, authoritativeDirectories, attributePrunedDirectories); } else @@ -3263,9 +3268,12 @@ void ThrowIfFullScanCancelled(int filesProcessed, int? filesTotal) if (checkpointedDirectories.Count > 0) { var authoritativeDirectories = scanResult.ListedDirectories + .Select(FileIndexer.NormalizeIndexPath) .ToHashSet(StringComparer.Ordinal); var attributePrunedDirectories = scanResult.AttributePrunedDirectories + .Select(FileIndexer.NormalizeIndexPath) .ToHashSet(StringComparer.Ordinal); + attributePrunedDirectories.UnionWith(scanResult.NestedRepositories.Select(FileIndexer.NormalizeIndexPath)); purged = writer.PurgeFilesOutsideRetainedSetWithinListedDirectories(retainedPaths, authoritativeDirectories, attributePrunedDirectories); } else diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index afe8a4b4fc..f349838694 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -48,7 +48,8 @@ public readonly record struct ScanFilesResult( IReadOnlyList FullyScannedDirectories, IReadOnlySet CheckpointedDirectories, IReadOnlyList AncestorIgnoreDirectories, - IReadOnlyList AttributePrunedDirectories) + IReadOnlyList AttributePrunedDirectories, + IReadOnlyList NestedRepositories) { public bool HadErrors => Errors.Any(error => error.IsFatal); } @@ -1586,6 +1587,9 @@ internal PathFilterResult EvaluatePathFilter(string absolutePath, bool isDirecto var isSubmodule = _submodulePaths.Contains(cumulativeRelPath); var isSubmoduleAncestor = _submoduleAncestorPaths.Contains(cumulativeRelPath); + if (IsNestedGitRepository(childDirectory) && !isSubmodule && !isSubmoduleAncestor) + return new PathFilterResult(PathFilterKind.ExcludedByDefaultDirectory, errors); + if (SkipDirs.Contains(directoryName)) { if (!isSubmodule && !isSubmoduleAncestor) @@ -1646,12 +1650,13 @@ internal ScanFilesResult ScanFilesDetailed( ? new HashSet(checkpointedDirectories, StringComparer.Ordinal) : new HashSet(StringComparer.Ordinal); var attributePrunedDirectories = new HashSet(StringComparer.Ordinal); + var nestedRepositories = new HashSet(StringComparer.Ordinal); var visitedFileIdentities = new HashSet(); var fullyScanned = true; var preloadResult = LoadAncestorIgnoreRules(errors, ref fullyScanned); if (preloadResult.IgnoreRulesAvailable) { - ScanDirectory(_projectRoot, files, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, activeCheckpointedDirectories, attributePrunedDirectories, visitedFileIdentities, preloadResult.Rules, isProjectRoot: true, continueOnError); + ScanDirectory(_projectRoot, files, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, activeCheckpointedDirectories, attributePrunedDirectories, nestedRepositories, visitedFileIdentities, preloadResult.Rules, isProjectRoot: true, continueOnError); } return new ScanFilesResult( files, @@ -1663,7 +1668,8 @@ internal ScanFilesResult ScanFilesDetailed( fullyScannedDirectories.ToList(), activeCheckpointedDirectories.Concat(fullyScannedDirectories).ToHashSet(StringComparer.Ordinal), _ancestorIgnoreDirectories.ToList(), - attributePrunedDirectories.ToList()); + attributePrunedDirectories.ToList(), + nestedRepositories.OrderBy(path => path, StringComparer.Ordinal).ToList()); } private bool ScanDirectory( @@ -1677,6 +1683,7 @@ private bool ScanDirectory( HashSet fullyScannedDirectories, HashSet checkpointedDirectories, HashSet attributePrunedDirectories, + HashSet nestedRepositories, HashSet visitedFileIdentities, IgnoreRuleSet activeIgnoreRules, bool isProjectRoot = false, @@ -1695,7 +1702,16 @@ private bool ScanDirectory( return true; } - return EnumerateDirectory(dir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, visitedFileIdentities, activeIgnoreRules, continueOnError); + return EnumerateDirectory(dir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, visitedFileIdentities, activeIgnoreRules, continueOnError); + } + + private bool IsNestedGitRepository(string dir) + { + if (PathsEqual(dir, _projectRoot)) + return false; + + return Directory.Exists(LongPath.EnsureWindowsPrefix(Path.Combine(dir, ".git"))) || + File.Exists(LongPath.EnsureWindowsPrefix(Path.Combine(dir, ".git"))); } private bool EnumerateDirectory( @@ -1709,6 +1725,7 @@ private bool EnumerateDirectory( HashSet fullyScannedDirectories, HashSet checkpointedDirectories, HashSet attributePrunedDirectories, + HashSet nestedRepositories, HashSet visitedFileIdentities, IgnoreRuleSet inheritedIgnoreRules, bool continueOnError) @@ -1862,6 +1879,15 @@ private bool EnumerateDirectory( foreach (var enumeratedSubDir in Directory.EnumerateDirectories(LongPath.EnsureWindowsPrefix(dir))) { var subDir = LongPath.RemoveWindowsPrefix(enumeratedSubDir); + if (IsNestedGitRepository(subDir) && !IsSubmoduleOrAncestor(subDir)) + { + var subRelative = ToRelativePath(subDir); + listedDirectories.Add(subRelative); + fullyScannedDirectories.Add(subRelative); + nestedRepositories.Add(subRelative); + continue; + } + // In passthrough mode, only descend into subdirectories that are themselves // submodules or submodule ancestors. Treat siblings the same way SkipDirs // would have treated them at this point. @@ -1895,7 +1921,7 @@ private bool EnumerateDirectory( continue; } - var childFullyScanned = ScanDirectory(subDir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, visitedFileIdentities, activeIgnoreRules, continueOnError: continueOnError); + var childFullyScanned = ScanDirectory(subDir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, visitedFileIdentities, activeIgnoreRules, continueOnError: continueOnError); fullyScanned &= childFullyScanned; if (!continueOnError && !childFullyScanned) break; @@ -2184,6 +2210,24 @@ private IgnoreRuleLoadResult LoadIgnoreRulesForDirectory( : new IgnoreRuleLoadResult(inheritedIgnoreRules, IgnoreRulesAvailable: false); } + private IgnoreRuleLoadResult LoadWorkspaceConfigIgnoreRules( + IgnoreRuleSet inheritedIgnoreRules, + List errors, + ref bool fullyScanned) + { + var configIgnorePath = Path.Combine(_projectRoot, ".codeindex", ".cdidxignore"); + if (!File.Exists(LongPath.EnsureWindowsPrefix(configIgnorePath))) + return new IgnoreRuleLoadResult(inheritedIgnoreRules, IgnoreRulesAvailable: true); + + return LoadIgnoreRulesFile( + sourceDirectory: _projectRoot, + ignorePath: configIgnorePath, + ignoreFileName: ".codeindex/.cdidxignore", + inheritedIgnoreRules, + errors, + ref fullyScanned); + } + private IgnoreRuleLoadResult LoadAncestorIgnoreRules(List errors, ref bool fullyScanned) { var activeIgnoreRules = IgnoreRuleSet.Empty; @@ -2202,7 +2246,46 @@ private IgnoreRuleLoadResult LoadAncestorIgnoreRules(List errors, ref return new IgnoreRuleLoadResult(activeIgnoreRules, IgnoreRulesAvailable: false); } - return new IgnoreRuleLoadResult(activeIgnoreRules, IgnoreRulesAvailable: true); + return LoadWorkspaceConfigIgnoreRules(activeIgnoreRules, errors, ref fullyScanned); + } + + private IgnoreRuleLoadResult LoadIgnoreRulesFile( + string sourceDirectory, + string ignorePath, + string ignoreFileName, + IgnoreRuleSet inheritedIgnoreRules, + List errors, + ref bool fullyScanned) + { + var rules = new List(); + var prefixedIgnorePath = LongPath.EnsureWindowsPrefix(ignorePath); + + try + { + var lineNumber = 0; + foreach (var line in File.ReadLines(prefixedIgnorePath, Encoding.UTF8)) + { + lineNumber++; + if (IgnoreRule.TryParse(sourceDirectory, line, _ignoreCase, out var rule, out var errorMessage) && rule != null) + rules.Add(rule); + else if (errorMessage != null) + errors.Add(new ScanError($"{ToRelativePath(ignorePath)}:{lineNumber}", errorMessage, ScanIssueSeverity.Warning)); + } + } + catch (UnauthorizedAccessException) + { + errors.Add(new ScanError(ToRelativePath(ignorePath), $"Could not read {ignoreFileName}.")); + fullyScanned = false; + return new IgnoreRuleLoadResult(inheritedIgnoreRules, IgnoreRulesAvailable: false); + } + catch (IOException) + { + errors.Add(new ScanError(ToRelativePath(ignorePath), $"Could not read {ignoreFileName}.")); + fullyScanned = false; + return new IgnoreRuleLoadResult(inheritedIgnoreRules, IgnoreRulesAvailable: false); + } + + return new IgnoreRuleLoadResult(IgnoreRuleSet.CreateChild(inheritedIgnoreRules, rules), IgnoreRulesAvailable: true); } private string NormalizeIgnoreRuleRoot(string? ignoreRuleRoot) @@ -2400,6 +2483,13 @@ private static string NormalizeIgnorePath(string path) public static string NormalizePathSeparators(string path) => Path.DirectorySeparatorChar == '\\' ? path.Replace('\\', '/') : path; + /// + /// Normalize index paths to the DB invariant: platform separators plus Unicode NFC. + /// DB 保存・lookup 用 path は区切り文字正規化に加えて Unicode NFC に正規化する。 + /// + public static string NormalizeIndexPath(string path) + => NormalizePathSeparators(path).Normalize(NormalizationForm.FormC); + /// /// Build a FileRecord and return file content (avoids reading the file twice). /// FileRecordを構築しファイル内容も返す（二重読み込み防止）。 @@ -2426,6 +2516,7 @@ public static string NormalizePathSeparators(string path) throw new InvalidOperationException("Only regular files can be indexed"); var relativePath = Path.GetRelativePath(_projectRoot, absolutePath); + var normalizedRelativePath = NormalizeIndexPath(relativePath); // Read raw bytes through a single FileStream and cap the accumulated payload at // the configured max-file limit so a file that grew between the size probe and the read can no @@ -2459,7 +2550,7 @@ public static string NormalizePathSeparators(string path) var initialLength = stream.Length; if (initialLength > _maxFileSizeBytes) throw new FileTooLargeSkippedException( - NormalizePathSeparators(relativePath), + normalizedRelativePath, initialLength, _maxFileSizeBytes, BuildFileTooLargeMessage(initialLength, grewDuringRead: false)); @@ -2478,7 +2569,7 @@ public static string NormalizePathSeparators(string path) total += read; if (total > _maxFileSizeBytes) throw new FileTooLargeSkippedException( - NormalizePathSeparators(relativePath), + normalizedRelativePath, total, _maxFileSizeBytes, BuildFileTooLargeMessage(total, grewDuringRead: true)); @@ -2576,13 +2667,13 @@ public static string NormalizePathSeparators(string path) content = StripLineLeadingInvisibles(content); var record = new FileRecord { - Path = NormalizePathSeparators(relativePath), + Path = normalizedRelativePath, Lang = TryDetectLanguage(absolutePath, content).Language, Size = sizeBytes, Lines = lineCount, Checksum = checksum, Modified = modifiedUtc, - Generated = IsGeneratedCodeFile(NormalizePathSeparators(relativePath), content), + Generated = IsGeneratedCodeFile(normalizedRelativePath, content), }; return (record, content, bytes, warning); @@ -2594,7 +2685,7 @@ public FileRecord BuildSkippedFileRecord(string absolutePath) throw new InvalidOperationException("Cannot index a file path that contains NUL or control characters."); var relativePath = Path.GetRelativePath(_projectRoot, absolutePath); - var normalizedRelativePath = NormalizePathSeparators(relativePath); + var normalizedRelativePath = NormalizeIndexPath(relativePath); var ioPath = LongPath.EnsureWindowsPrefix(absolutePath); var info = new FileInfo(ioPath); return new FileRecord diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 06773c537c..952a49dd78 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Runtime.Versioning; using System.Text; +using System.Text.Json; using System.Threading.Tasks; using CodeIndex.Database; using CodeIndex.Cli; @@ -62,6 +63,31 @@ public void ScanFilesDetailed_CaseInsensitiveChildDirectory_SkipsCaseOnlyDuplica } } + [Fact] + public void ScanFiles_SkipsBuiltInDirectoriesWithCaseInsensitiveNames() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"cdidx-skipdir-case-{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(Path.Combine(tempDir, "Node_Modules")); + File.WriteAllText(Path.Combine(tempDir, "Node_Modules", "ignored.js"), "export const ignored = true;"); + File.WriteAllText(Path.Combine(tempDir, "app.js"), "export const app = true;"); + + var indexer = new FileIndexer(tempDir, ignoreCase: true); + var files = indexer.ScanFiles() + .Select(path => Path.GetRelativePath(tempDir, path).Replace('\\', '/')) + .OrderBy(path => path, StringComparer.Ordinal) + .ToList(); + + Assert.Equal(["app.js"], files); + } + finally + { + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + } + [Fact] public void ScanFilesDetailed_HardlinkedFiles_SkipsDuplicatePathWithWarning() { @@ -1439,6 +1465,108 @@ public void ScanFiles_ReadsGitignoreAndCdidxignoreAsUtf8() } } + [Fact] + public void ScanFiles_RespectsWorkspaceConfigCdidxignore() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(Path.Combine(tempDir, ".codeindex")); + Directory.CreateDirectory(Path.Combine(tempDir, "generated")); + File.WriteAllText(Path.Combine(tempDir, ".codeindex", ".cdidxignore"), "generated/\n*.cache.js\n"); + File.WriteAllText(Path.Combine(tempDir, "generated", "Ignored.cs"), "class Ignored { }"); + File.WriteAllText(Path.Combine(tempDir, "app.cache.js"), "export const ignored = true;"); + File.WriteAllText(Path.Combine(tempDir, "app.js"), "export const app = true;"); + + var indexer = new FileIndexer(tempDir); + var files = indexer.ScanFiles() + .Select(path => Path.GetRelativePath(tempDir, path).Replace('\\', '/')) + .OrderBy(path => path, StringComparer.Ordinal) + .ToList(); + + Assert.Equal(["app.js"], files); + } + finally + { + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + } + + [Fact] + public void ScanFilesDetailed_SkipsNestedGitRepositoryBoundary() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(Path.Combine(tempDir, "nested", ".git")); + File.WriteAllText(Path.Combine(tempDir, "Root.cs"), "class Root { }"); + File.WriteAllText(Path.Combine(tempDir, "nested", "Nested.cs"), "class Nested { }"); + + var indexer = new FileIndexer(tempDir); + var result = indexer.ScanFilesDetailed(); + var files = result.Files + .Select(path => Path.GetRelativePath(tempDir, path).Replace('\\', '/')) + .OrderBy(path => path, StringComparer.Ordinal) + .ToList(); + + Assert.Equal(["Root.cs"], files); + Assert.Equal(["nested"], result.NestedRepositories); + } + finally + { + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + } + + [Fact] + public void EvaluatePathFilter_SkipsNestedGitRepositoryBoundary() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(Path.Combine(tempDir, "nested", ".git")); + var nestedFile = Path.Combine(tempDir, "nested", "Nested.cs"); + File.WriteAllText(nestedFile, "class Nested { }"); + + var indexer = new FileIndexer(tempDir); + var filter = indexer.EvaluatePathFilter(nestedFile); + + Assert.Equal(FileIndexer.PathFilterKind.ExcludedByDefaultDirectory, filter.FilterKind); + Assert.True(filter.ShouldDeleteExisting); + } + finally + { + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + } + + [Fact] + public void BuildRecord_NormalizesRelativePathToNfc() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + var nfdName = "Cafe\u0301.cs"; + var filePath = Path.Combine(tempDir, nfdName); + File.WriteAllText(filePath, "class Cafe { }"); + + var indexer = new FileIndexer(tempDir); + var (record, _, _) = indexer.BuildRecord(filePath); + + Assert.Equal("Caf\u00e9.cs", record.Path); + Assert.True(record.Path.IsNormalized(NormalizationForm.FormC)); + } + finally + { + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + } + [Fact] public void ScanFiles_FailsClosedWhenRootIgnoreFileIsUnreadable() { @@ -2886,6 +3014,7 @@ public void ScanFiles_DescendsIntoSubmoduleHostedUnderSkipDir() var submoduleDir = Path.Combine(vendorDir, "foo"); Directory.CreateDirectory(submoduleDir); + File.WriteAllText(Path.Combine(submoduleDir, ".git"), "gitdir: ../../.git/modules/foo\n"); File.WriteAllText(Path.Combine(submoduleDir, "lib.py"), "def f(): pass"); Directory.CreateDirectory(Path.Combine(submoduleDir, "src")); File.WriteAllText(Path.Combine(submoduleDir, "src", "nested.py"), "def g(): pass"); @@ -2905,6 +3034,99 @@ public void ScanFiles_DescendsIntoSubmoduleHostedUnderSkipDir() } } + [Fact] + public void PurgeFilesOutsideRetainedSet_UsesNfcRetainedPaths() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + var dbPath = TestProjectHelper.CreateProjectDb(tempDir); + var nfcPath = "Caf\u00e9.cs"; + var nfdPath = "Cafe\u0301.cs"; + TestProjectHelper.InsertIndexedFile(dbPath, nfcPath, "csharp", "class CafeFixture { }\n"); + + using var db = new DbContext(dbPath); + db.InitializeSchema(); + var writer = new DbWriter(db.Connection); + var retainedPaths = new[] + { + Path.Combine(tempDir, nfdPath), + } + .Select(path => FileIndexer.NormalizeIndexPath(Path.GetRelativePath(tempDir, path))) + .ToHashSet(StringComparer.Ordinal); + + var purged = writer.PurgeFilesOutsideRetainedSet(retainedPaths); + + Assert.Equal(0, purged); + Assert.Equal(1, CountFiles(db.Connection)); + } + finally + { + TestProjectHelper.DeleteDirectory(tempDir); + } + } + + [Fact] + public void PurgeFilesOutsideRetainedSetWithinListedDirectories_UsesNfcPrunedDirectories() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + var dbPath = TestProjectHelper.CreateProjectDb(tempDir); + TestProjectHelper.InsertIndexedFile(dbPath, "Caf\u00e9/src/File.cs", "csharp", "class NestedCafe { }\n"); + + using var db = new DbContext(dbPath); + db.InitializeSchema(); + var writer = new DbWriter(db.Connection); + var prunedDirectories = new[] { "Cafe\u0301" } + .Select(FileIndexer.NormalizeIndexPath) + .ToHashSet(StringComparer.Ordinal); + + var purged = writer.PurgeFilesOutsideRetainedSetWithinListedDirectories( + new HashSet(StringComparer.Ordinal), + new HashSet(StringComparer.Ordinal), + prunedDirectories); + + Assert.Equal(1, purged); + Assert.Equal(0, CountFiles(db.Connection)); + } + finally + { + TestProjectHelper.DeleteDirectory(tempDir); + } + } + + [Fact] + public void IndexFilesUpdate_UsesOriginalUnicodePathForIoAndNfcPathForDb() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + var nfdPath = "Cafe\u0301.cs"; + File.WriteAllText(Path.Combine(tempDir, nfdPath), "class FirstCafe { }\n"); + + var jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web); + Assert.Equal(CommandExitCodes.Success, IndexCommandRunner.Run([tempDir, "--json", "--quiet"], jsonOptions)); + + File.WriteAllText(Path.Combine(tempDir, nfdPath), "class UpdatedCafe { }\n"); + Assert.Equal(CommandExitCodes.Success, IndexCommandRunner.Run([tempDir, "--files", nfdPath, "--json", "--quiet"], jsonOptions)); + + var dbPath = Path.Combine(tempDir, ".cdidx", "codeindex.db"); + Assert.Equal("class UpdatedCafe { }", ReadSingleChunkContent(dbPath, "Caf\u00e9.cs")); + + File.WriteAllBytes(Path.Combine(tempDir, nfdPath), [0, 1, 2, 3]); + Assert.Equal(CommandExitCodes.Success, IndexCommandRunner.Run([tempDir, "--files", nfdPath, "--json", "--quiet"], jsonOptions)); + Assert.False(HasIndexedFile(dbPath, "Caf\u00e9.cs")); + } + finally + { + TestProjectHelper.DeleteDirectory(tempDir); + } + } + [Fact] public void ScanFiles_RespectsSubmoduleGitignore() { @@ -4075,4 +4297,35 @@ public void IsGeneratedCodeFile_HandwrittenFile_ReturnsFalse() Assert.False(FileIndexer.IsGeneratedCodeFile("src/Foo.cs", "class Foo { }\n")); Assert.False(FileIndexer.IsGeneratedCodeFile("src/Foo.cs", "// This file is not auto-generated.\nclass Foo { }\n")); } + + private static int CountFiles(SqliteConnection connection) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM files"; + return Convert.ToInt32(cmd.ExecuteScalar()); + } + + private static string ReadSingleChunkContent(string dbPath, string filePath) + { + using var db = new DbContext(dbPath); + using var cmd = db.Connection.CreateCommand(); + cmd.CommandText = """ + SELECT c.content + FROM chunks c + JOIN files f ON f.id = c.file_id + WHERE f.path = @path + ORDER BY c.chunk_index + """; + cmd.Parameters.AddWithValue("@path", filePath); + return Assert.IsType(cmd.ExecuteScalar()); + } + + private static bool HasIndexedFile(string dbPath, string filePath) + { + using var db = new DbContext(dbPath); + using var cmd = db.Connection.CreateCommand(); + cmd.CommandText = "SELECT 1 FROM files WHERE path = @path"; + cmd.Parameters.AddWithValue("@path", filePath); + return cmd.ExecuteScalar() != null; + } }