Fix scan edge cases and canonical indexing

USER_GUIDE.md

@@ -341,9 +341,10 @@ cdidx validate --json --path legacy/
 
 `validate` reports indexed files that are likely to produce misleading snippets
 or symbol names: U+FFFD replacement characters, UTF-16 BOMs, null bytes, mixed or
-CR-only line endings, and likely non-UTF-8 content. Treat failures as source
-encoding or repository hygiene work; after fixing files, rerun `cdidx index .`
-and then `cdidx validate` again.
+CR-only line endings, likely non-UTF-8 content, and Git LFS pointer placeholders.
+LFS pointers are recorded as `lfs_pointer_skipped` and their placeholder body is
+not indexed; run `git lfs pull` and then `cdidx index .` to index the real file
+content.
 
 ### Find potentially unused symbols
 
@@ -2355,8 +2356,9 @@ cdidx validate --json --path legacy/
 
 `validate` は、snippet や symbol name を誤らせやすい indexed file を報告します。
 対象は U+FFFD replacement character、UTF-16 BOM、null byte、mixed / CR-only line
-ending、likely non-UTF-8 content などです。失敗は source encoding または repository
-hygiene の問題として扱い、修正後に `cdidx index .`、続いて `cdidx validate` を再実行してください。
+ending、likely non-UTF-8 content、Git LFS pointer placeholder などです。LFS pointer
+は `lfs_pointer_skipped` として記録され、placeholder 本文は index されません。
+実体を index するには `git lfs pull` の後に `cdidx index .` を再実行してください。
 
 ### 未使用の可能性がある symbols を探す
 

changelog.d/unreleased/1466.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+issues:
+  - 1466
+affected:
+  - src/CodeIndex/Indexer/Scanning/FileIndexer.cs
+  - tests/CodeIndex.Tests/FileIndexerTests.cs
+---
+
+## English
+
+- **Directory scans now cap recursion and skip revisited symlink targets (#1466)** — `cdidx index` now avoids repeated traversal through symlink loops when symlink following is enabled and reports a warning instead of risking unbounded recursion.
+
+## 日本語
+
+- **ディレクトリ走査が再帰深度と symlink 再訪問を抑止するようになりました (#1466)** — symlink follow が有効な場合でも `cdidx index` は symlink loop の再走査を避け、無制限再帰の代わりに警告を記録します。

changelog.d/unreleased/1467.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+issues:
+  - 1467
+affected:
+  - src/CodeIndex/Indexer/Scanning/FileIndexer.cs
+  - tests/CodeIndex.Tests/FileIndexerTests.cs
+---
+
+## English
+
+- **Checksums now use the same BOM-stripped content stored in chunks (#1467)** — line-leading UTF-8 BOM markers are normalized before checksum calculation so incremental freshness and excerpt line metadata stay aligned.
+
+## 日本語
+
+- **checksum が chunk に保存される BOM 除去後の内容と一致するようになりました (#1467)** — 行頭の UTF-8 BOM を checksum 計算前に正規化し、incremental freshness と excerpt の行メタデータのずれを防ぎます。

changelog.d/unreleased/1622.fixed.md

@@ -0,0 +1,17 @@
+---
+category: fixed
+issues:
+  - 1622
+affected:
+  - src/CodeIndex/Indexer/Scanning/FileIndexer.cs
+  - tests/CodeIndex.Tests/FileIndexerTests.cs
+  - USER_GUIDE.md
+---
+
+## English
+
+- **Git LFS pointer placeholders are no longer indexed as code (#1622)** — pointer files now emit `lfs_pointer_skipped`, keep their placeholder text out of search results, and document `git lfs pull` as the recovery path.
+
+## 日本語
+
+- **Git LFS pointer placeholder を code として index しないようになりました (#1622)** — pointer file は `lfs_pointer_skipped` を出し、placeholder 本文を検索結果から除外し、復旧手順として `git lfs pull` を案内します。

src/CodeIndex/Indexer/Scanning/FileIndexer.cs

@@ -86,6 +86,8 @@ PathFilterKind.ExcludedByDefaultFile or
 
     private static readonly string[] HotspotFamilyMarkerLanguages = ["csharp", "vb", "fsharp", "msbuild"];
     private const int ConflictMarkerScanLimitBytes = 50 * 1024;
+    private const int MaxDirectoryTraversalDepth = 128;
+    private const int GitLfsPointerMaxBytes = 1024;
     private static readonly string[] IgnoreFileNames = [".gitignore", ".cdidxignore"];
     private const int MaxIgnorePatternLength = 512;
     private static readonly TimeSpan IgnoreRegexMatchTimeout = TimeSpan.FromMilliseconds(100);
@@ -1848,7 +1850,7 @@ internal ScanFilesResult ScanFilesDetailed(
         var preloadResult = LoadAncestorIgnoreRules(errors, ref fullyScanned);
         if (preloadResult.IgnoreRulesAvailable)
         {
-            ScanDirectory(_projectRoot, files, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, activeCheckpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, preloadResult.Rules, isProjectRoot: true, continueOnError, cancellationToken);
+            ScanDirectory(_projectRoot, files, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, activeCheckpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, preloadResult.Rules, isProjectRoot: true, continueOnError, cancellationToken, depth: 0);
         }
         return new ScanFilesResult(
             files,
@@ -1883,11 +1885,21 @@ private bool ScanDirectory(
         IgnoreRuleSet activeIgnoreRules,
         bool isProjectRoot = false,
         bool continueOnError = true,
-        CancellationToken cancellationToken = default)
+        CancellationToken cancellationToken = default,
+        int depth = 0)
     {
         cancellationToken.ThrowIfCancellationRequested();
         var relativeDir = ToRelativePath(dir);
 
+        if (depth > MaxDirectoryTraversalDepth)
+        {
+            errors.Add(new ScanError(
+                relativeDir,
+                $"Skipped directory because traversal depth exceeded {MaxDirectoryTraversalDepth}. Check for symlink loops or unexpectedly deep generated trees.",
+                ScanIssueSeverity.Warning));
+            return true;
+        }
+
         if (checkpointedDirectories.Contains(relativeDir))
             return true;
 
@@ -1899,7 +1911,7 @@ private bool ScanDirectory(
             return true;
         }
 
-        return EnumerateDirectory(dir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, activeIgnoreRules, continueOnError, cancellationToken);
+        return EnumerateDirectory(dir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, activeIgnoreRules, continueOnError, cancellationToken, depth);
     }
 
     private bool IsNestedGitRepository(string dir)
@@ -1928,7 +1940,8 @@ private bool EnumerateDirectory(
         HashSet<string> visitedDirectories,
         IgnoreRuleSet inheritedIgnoreRules,
         bool continueOnError,
-        CancellationToken cancellationToken = default)
+        CancellationToken cancellationToken = default,
+        int depth = 0)
     {
         cancellationToken.ThrowIfCancellationRequested();
         var fullyScanned = true;
@@ -2150,7 +2163,7 @@ private bool EnumerateDirectory(
                     continue;
                 }
 
-                var childFullyScanned = ScanDirectory(subDir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, activeIgnoreRules, continueOnError: continueOnError, cancellationToken: cancellationToken);
+                var childFullyScanned = ScanDirectory(subDir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, activeIgnoreRules, continueOnError: continueOnError, cancellationToken: cancellationToken, depth: depth + 1);
                 fullyScanned &= childFullyScanned;
                 if (!continueOnError && !childFullyScanned)
                     break;
@@ -2829,23 +2842,6 @@ public static string NormalizeIndexPath(string path)
                 break;
         }
 
-        // Compute the checksum on the byte stream after collapsing CRLF / CR to LF so
-        // a Windows clone (core.autocrlf=true) and a Linux/macOS clone of the same logical
-        // file produce identical checksums. Hashing the unnormalized raw bytes used to
-        // mark every file as "changed" the first time a developer indexed a cross-OS clone
-        // or a shared NAS workspace. BOM bytes are preserved verbatim so BOM addition /
-        // removal still flips the checksum and triggers incremental re-index. The streaming
-        // helper avoids re-encoding the UTF-8 string (~10 MB saved for large files).
-        // Closes #1544.
-        // CRLF / CR を LF に潰した上で SHA256 を取り、Windows (core.autocrlf=true) clone と
-        // Linux/macOS clone の同じ論理内容で checksum を一致させる。生バイトをそのまま
-        // ハッシュしていた以前は、cross-OS の clone や共有 NAS で全ファイルが「変更あり」
-        // 扱いとなり毎回フル再索引が走っていた。BOM はそのままハッシュ対象に残すので
-        // BOM の追加 / 削除はインクリメンタル再索引で引き続き検知できる。streaming
-        // ヘルパは UTF-8 文字列を再エンコードせずに済み、大ファイルで約 10 MB 節約する。
-        // Closes #1544.
-        var checksum = ComputeChecksum(bytes);
-
         var isUtf16Encoded = TryDetectUtf16Encoding(bytes, allowHeuristic: true, out var utf16BigEndian, out var hasUtf16Bom);
 
         if (!isUtf16Encoded && ContainsIndexBlockingNullByte(bytes))
@@ -2880,8 +2876,6 @@ public static string NormalizeIndexPath(string path)
                 warning = $"{relativePath}: contains invalid UTF-8 bytes (replaced with U+FFFD)";
             }
         }
-        var lineCount = CountPhysicalLines(content);
-
         // Normalize line endings to LF in one pass / 改行を1パスでLFに正規化
         content = NormalizeLineEndings(content);
         // Strip every line-leading UTF-8 BOM (U+FEFF): the leading BOM at offset 0
@@ -2893,10 +2887,8 @@ public static string NormalizeIndexPath(string path)
         // output. Non-line-leading U+FEFF (Unicode 3.2+ ZWNBSP inside a string
         // literal, identifier, or comment — e.g. `const s = "A\uFEFFB"`) is kept
         // verbatim: stripping it would corrupt content fidelity for intentional
-        // mid-line ZWNBSP use. The checksum computed above keeps BOM bytes in the
-        // hash input (only CRLF / CR are collapsed to LF) so incremental re-index
-        // still detects BOM add / removal. Closes #183. Cross-OS CRLF / LF parity
-        // is handled by ComputeChecksum itself; see #1544.
+        // mid-line ZWNBSP use. The checksum is computed after this canonicalization
+        // so freshness decisions match stored chunk/symbol line metadata. Closes #183/#1467.
         // 行頭の UTF-8 BOM (U+FEFF) だけを剥がす — オフセット 0 の先頭 BOM と、
         // `\n` の直後にある BOM (ファイル連結やツール挿入で発生) が対象。先頭 BOM
         // だけでも行指向の `^\s*` 固定正規表現で BOM 付き Windows 作成ソースの
@@ -2905,11 +2897,13 @@ public static string NormalizeIndexPath(string path)
         // (Unicode 3.2+ の ZWNBSP を文字列リテラル・識別子・コメントで意図的に
         // 使用しているケース、例: `const s = "A\uFEFFB"`) はそのまま保持する:
         // これを剥がすと mid-line ZWNBSP の意図的利用に対して内容が壊れる。
-        // checksum は上で算出済みで、ハッシュ入力に BOM をそのまま含めたまま
-        // CRLF / CR のみを LF に潰すため、BOM の追加 / 削除はインクリメンタル
-        // 再索引で引き続き検知される。Closes #183。OS をまたいだ CRLF / LF の
-        // 同一性は ComputeChecksum 自体で担保する。#1544 参照。
+        // checksum はこの canonicalization 後に算出し、freshness 判定と保存される
+        // chunk / symbol 行メタデータを一致させる。Closes #183/#1467。
         content = StripLineLeadingInvisibles(content);
+        if (IsGitLfsPointer(bytes))
+            content = string.Empty;
+        var lineCount = CountPhysicalLines(content);
+        var checksum = ComputeChecksum(Encoding.UTF8.GetBytes(content));
         var record = new FileRecord
         {
             Path = normalizedRelativePath,
@@ -3204,6 +3198,69 @@ internal static string StripLineLeadingInvisibles(string content)
 
     private static bool IsLineLeadingInvisible(char c) => c is '\uFEFF' or '\u200B';
 
+    internal static bool IsGitLfsPointer(byte[] rawBytes)
+    {
+        if (rawBytes.Length == 0 || rawBytes.Length >= GitLfsPointerMaxBytes)
+            return false;
+
+        var pointerText = Encoding.UTF8.GetString(rawBytes).Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n');
+        var lines = pointerText.Split('\n');
+        if (lines.Length > 0 && lines[^1].Length == 0)
+            lines = lines[..^1];
+        if (lines.Length < 3)
+            return false;
+        if (!string.Equals(lines[0], "version https://git-lfs.github.com/spec/v1", StringComparison.Ordinal))
+            return false;
+
+        var lineIndex = 1;
+        while (lineIndex < lines.Length && lines[lineIndex].StartsWith("ext-", StringComparison.Ordinal))
+            lineIndex++;
+
+        if (lineIndex + 1 >= lines.Length)
+            return false;
+        if (!IsGitLfsSha256OidLine(lines[lineIndex]))
+            return false;
+        lineIndex++;
+        if (!IsGitLfsSizeLine(lines[lineIndex]))
+            return false;
+
+        return lineIndex == lines.Length - 1;
+    }
+
+    private static bool IsGitLfsSha256OidLine(string line)
+    {
+        const string prefix = "oid sha256:";
+        if (!line.StartsWith(prefix, StringComparison.Ordinal))
+            return false;
+
+        var hash = line.AsSpan(prefix.Length);
+        if (hash.Length != 64)
+            return false;
+        foreach (var c in hash)
+        {
+            if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')))
+                return false;
+        }
+        return true;
+    }
+
+    private static bool IsGitLfsSizeLine(string line)
+    {
+        const string prefix = "size ";
+        if (!line.StartsWith(prefix, StringComparison.Ordinal))
+            return false;
+
+        var size = line.AsSpan(prefix.Length);
+        if (size.Length == 0)
+            return false;
+        foreach (var c in size)
+        {
+            if (c < '0' || c > '9')
+                return false;
+        }
+        return true;
+    }
+
     /// <summary>
     /// Validate file content for encoding issues.
     /// ファイル内容のエンコーディング問題を検証する。
@@ -3212,6 +3269,17 @@ public static List<FileIssue> ValidateContent(string relativePath, byte[] rawByt
     {
         var issues = new List<FileIssue>();
 
+        if (IsGitLfsPointer(rawBytes))
+        {
+            issues.Add(new FileIssue
+            {
+                Path = relativePath,
+                Kind = "lfs_pointer_skipped",
+                Line = 1,
+                Message = "Git LFS pointer file skipped; fetch LFS objects to index real file content",
+            });
+        }
+
         // UTF-16 BOM-detected files are decoded as UTF-16 in BuildRecordWithRawBytes, so the
         // raw-byte heuristics for `bom` / `null_byte` / `mixed_line_endings` would all misfire
         // (every UTF-16 LE character ASCII point looks like a NUL byte; CRLF appears as 0D 00