diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 6156d72c5b..2e632e4e80 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -341,9 +341,10 @@ cdidx validate --json --path legacy/ `validate` reports indexed files that are likely to produce misleading snippets or symbol names: U+FFFD replacement characters, UTF-16 BOMs, null bytes, mixed or -CR-only line endings, and likely non-UTF-8 content. Treat failures as source -encoding or repository hygiene work; after fixing files, rerun `cdidx index .` -and then `cdidx validate` again. +CR-only line endings, likely non-UTF-8 content, and Git LFS pointer placeholders. +LFS pointers are recorded as `lfs_pointer_skipped` and their placeholder body is +not indexed; run `git lfs pull` and then `cdidx index .` to index the real file +content. ### Find potentially unused symbols @@ -2355,8 +2356,9 @@ cdidx validate --json --path legacy/ `validate` は、snippet や symbol name を誤らせやすい indexed file を報告します。 対象は U+FFFD replacement character、UTF-16 BOM、null byte、mixed / CR-only line -ending、likely non-UTF-8 content などです。失敗は source encoding または repository -hygiene の問題として扱い、修正後に `cdidx index .`、続いて `cdidx validate` を再実行してください。 +ending、likely non-UTF-8 content、Git LFS pointer placeholder などです。LFS pointer +は `lfs_pointer_skipped` として記録され、placeholder 本文は index されません。 +実体を index するには `git lfs pull` の後に `cdidx index .` を再実行してください。 ### 未使用の可能性がある symbols を探す diff --git a/changelog.d/unreleased/1466.fixed.md b/changelog.d/unreleased/1466.fixed.md new file mode 100644 index 0000000000..eb5d7e052e --- /dev/null +++ b/changelog.d/unreleased/1466.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1466 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs +--- + +## English + +- **Directory scans now cap recursion and skip revisited symlink targets (#1466)** — `cdidx index` now avoids repeated traversal through symlink loops when symlink following is enabled and reports a warning instead of risking unbounded recursion. + +## 日本語 + +- **ディレクトリ走査が再帰深度と symlink 再訪問を抑止するようになりました (#1466)** — symlink follow が有効な場合でも `cdidx index` は symlink loop の再走査を避け、無制限再帰の代わりに警告を記録します。 diff --git a/changelog.d/unreleased/1467.fixed.md b/changelog.d/unreleased/1467.fixed.md new file mode 100644 index 0000000000..e64ef18222 --- /dev/null +++ b/changelog.d/unreleased/1467.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1467 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs +--- + +## English + +- **Checksums now use the same BOM-stripped content stored in chunks (#1467)** — line-leading UTF-8 BOM markers are normalized before checksum calculation so incremental freshness and excerpt line metadata stay aligned. + +## 日本語 + +- **checksum が chunk に保存される BOM 除去後の内容と一致するようになりました (#1467)** — 行頭の UTF-8 BOM を checksum 計算前に正規化し、incremental freshness と excerpt の行メタデータのずれを防ぎます。 diff --git a/changelog.d/unreleased/1622.fixed.md b/changelog.d/unreleased/1622.fixed.md new file mode 100644 index 0000000000..9e4c3a24f9 --- /dev/null +++ b/changelog.d/unreleased/1622.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 1622 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs + - USER_GUIDE.md +--- + +## English + +- **Git LFS pointer placeholders are no longer indexed as code (#1622)** — pointer files now emit `lfs_pointer_skipped`, keep their placeholder text out of search results, and document `git lfs pull` as the recovery path. + +## 日本語 + +- **Git LFS pointer placeholder を code として index しないようになりました (#1622)** — pointer file は `lfs_pointer_skipped` を出し、placeholder 本文を検索結果から除外し、復旧手順として `git lfs pull` を案内します。 diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index 3473929b2a..5e0a5f7714 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -86,6 +86,8 @@ PathFilterKind.ExcludedByDefaultFile or private static readonly string[] HotspotFamilyMarkerLanguages = ["csharp", "vb", "fsharp", "msbuild"]; private const int ConflictMarkerScanLimitBytes = 50 * 1024; + private const int MaxDirectoryTraversalDepth = 128; + private const int GitLfsPointerMaxBytes = 1024; private static readonly string[] IgnoreFileNames = [".gitignore", ".cdidxignore"]; private const int MaxIgnorePatternLength = 512; private static readonly TimeSpan IgnoreRegexMatchTimeout = TimeSpan.FromMilliseconds(100); @@ -1848,7 +1850,7 @@ internal ScanFilesResult ScanFilesDetailed( var preloadResult = LoadAncestorIgnoreRules(errors, ref fullyScanned); if (preloadResult.IgnoreRulesAvailable) { - ScanDirectory(_projectRoot, files, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, activeCheckpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, preloadResult.Rules, isProjectRoot: true, continueOnError, cancellationToken); + ScanDirectory(_projectRoot, files, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, activeCheckpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, preloadResult.Rules, isProjectRoot: true, continueOnError, cancellationToken, depth: 0); } return new ScanFilesResult( files, @@ -1883,11 +1885,21 @@ private bool ScanDirectory( IgnoreRuleSet activeIgnoreRules, bool isProjectRoot = false, bool continueOnError = true, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + int depth = 0) { cancellationToken.ThrowIfCancellationRequested(); var relativeDir = ToRelativePath(dir); + if (depth > MaxDirectoryTraversalDepth) + { + errors.Add(new ScanError( + relativeDir, + $"Skipped directory because traversal depth exceeded {MaxDirectoryTraversalDepth}. Check for symlink loops or unexpectedly deep generated trees.", + ScanIssueSeverity.Warning)); + return true; + } + if (checkpointedDirectories.Contains(relativeDir)) return true; @@ -1899,7 +1911,7 @@ private bool ScanDirectory( return true; } - return EnumerateDirectory(dir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, activeIgnoreRules, continueOnError, cancellationToken); + return EnumerateDirectory(dir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, activeIgnoreRules, continueOnError, cancellationToken, depth); } private bool IsNestedGitRepository(string dir) @@ -1928,7 +1940,8 @@ private bool EnumerateDirectory( HashSet visitedDirectories, IgnoreRuleSet inheritedIgnoreRules, bool continueOnError, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + int depth = 0) { cancellationToken.ThrowIfCancellationRequested(); var fullyScanned = true; @@ -2150,7 +2163,7 @@ private bool EnumerateDirectory( continue; } - var childFullyScanned = ScanDirectory(subDir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, activeIgnoreRules, continueOnError: continueOnError, cancellationToken: cancellationToken); + var childFullyScanned = ScanDirectory(subDir, results, errors, nonIndexablePaths, unknownExtensionFiles, probeFailedFilePaths, listedDirectories, fullyScannedDirectories, checkpointedDirectories, attributePrunedDirectories, nestedRepositories, danglingSymlinks, visitedFileIdentities, visitedDirectories, activeIgnoreRules, continueOnError: continueOnError, cancellationToken: cancellationToken, depth: depth + 1); fullyScanned &= childFullyScanned; if (!continueOnError && !childFullyScanned) break; @@ -2829,23 +2842,6 @@ public static string NormalizeIndexPath(string path) break; } - // Compute the checksum on the byte stream after collapsing CRLF / CR to LF so - // a Windows clone (core.autocrlf=true) and a Linux/macOS clone of the same logical - // file produce identical checksums. Hashing the unnormalized raw bytes used to - // mark every file as "changed" the first time a developer indexed a cross-OS clone - // or a shared NAS workspace. BOM bytes are preserved verbatim so BOM addition / - // removal still flips the checksum and triggers incremental re-index. The streaming - // helper avoids re-encoding the UTF-8 string (~10 MB saved for large files). - // Closes #1544. - // CRLF / CR を LF に潰した上で SHA256 を取り、Windows (core.autocrlf=true) clone と - // Linux/macOS clone の同じ論理内容で checksum を一致させる。生バイトをそのまま - // ハッシュしていた以前は、cross-OS の clone や共有 NAS で全ファイルが「変更あり」 - // 扱いとなり毎回フル再索引が走っていた。BOM はそのままハッシュ対象に残すので - // BOM の追加 / 削除はインクリメンタル再索引で引き続き検知できる。streaming - // ヘルパは UTF-8 文字列を再エンコードせずに済み、大ファイルで約 10 MB 節約する。 - // Closes #1544. - var checksum = ComputeChecksum(bytes); - var isUtf16Encoded = TryDetectUtf16Encoding(bytes, allowHeuristic: true, out var utf16BigEndian, out var hasUtf16Bom); if (!isUtf16Encoded && ContainsIndexBlockingNullByte(bytes)) @@ -2880,8 +2876,6 @@ public static string NormalizeIndexPath(string path) warning = $"{relativePath}: contains invalid UTF-8 bytes (replaced with U+FFFD)"; } } - var lineCount = CountPhysicalLines(content); - // Normalize line endings to LF in one pass / 改行を1パスでLFに正規化 content = NormalizeLineEndings(content); // Strip every line-leading UTF-8 BOM (U+FEFF): the leading BOM at offset 0 @@ -2893,10 +2887,8 @@ public static string NormalizeIndexPath(string path) // output. Non-line-leading U+FEFF (Unicode 3.2+ ZWNBSP inside a string // literal, identifier, or comment — e.g. `const s = "A\uFEFFB"`) is kept // verbatim: stripping it would corrupt content fidelity for intentional - // mid-line ZWNBSP use. The checksum computed above keeps BOM bytes in the - // hash input (only CRLF / CR are collapsed to LF) so incremental re-index - // still detects BOM add / removal. Closes #183. Cross-OS CRLF / LF parity - // is handled by ComputeChecksum itself; see #1544. + // mid-line ZWNBSP use. The checksum is computed after this canonicalization + // so freshness decisions match stored chunk/symbol line metadata. Closes #183/#1467. // 行頭の UTF-8 BOM (U+FEFF) だけを剥がす — オフセット 0 の先頭 BOM と、 // `\n` の直後にある BOM (ファイル連結やツール挿入で発生) が対象。先頭 BOM // だけでも行指向の `^\s*` 固定正規表現で BOM 付き Windows 作成ソースの @@ -2905,11 +2897,13 @@ public static string NormalizeIndexPath(string path) // (Unicode 3.2+ の ZWNBSP を文字列リテラル・識別子・コメントで意図的に // 使用しているケース、例: `const s = "A\uFEFFB"`) はそのまま保持する: // これを剥がすと mid-line ZWNBSP の意図的利用に対して内容が壊れる。 - // checksum は上で算出済みで、ハッシュ入力に BOM をそのまま含めたまま - // CRLF / CR のみを LF に潰すため、BOM の追加 / 削除はインクリメンタル - // 再索引で引き続き検知される。Closes #183。OS をまたいだ CRLF / LF の - // 同一性は ComputeChecksum 自体で担保する。#1544 参照。 + // checksum はこの canonicalization 後に算出し、freshness 判定と保存される + // chunk / symbol 行メタデータを一致させる。Closes #183/#1467。 content = StripLineLeadingInvisibles(content); + if (IsGitLfsPointer(bytes)) + content = string.Empty; + var lineCount = CountPhysicalLines(content); + var checksum = ComputeChecksum(Encoding.UTF8.GetBytes(content)); var record = new FileRecord { Path = normalizedRelativePath, @@ -3204,6 +3198,69 @@ internal static string StripLineLeadingInvisibles(string content) private static bool IsLineLeadingInvisible(char c) => c is '\uFEFF' or '\u200B'; + internal static bool IsGitLfsPointer(byte[] rawBytes) + { + if (rawBytes.Length == 0 || rawBytes.Length >= GitLfsPointerMaxBytes) + return false; + + var pointerText = Encoding.UTF8.GetString(rawBytes).Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n'); + var lines = pointerText.Split('\n'); + if (lines.Length > 0 && lines[^1].Length == 0) + lines = lines[..^1]; + if (lines.Length < 3) + return false; + if (!string.Equals(lines[0], "version https://git-lfs.github.com/spec/v1", StringComparison.Ordinal)) + return false; + + var lineIndex = 1; + while (lineIndex < lines.Length && lines[lineIndex].StartsWith("ext-", StringComparison.Ordinal)) + lineIndex++; + + if (lineIndex + 1 >= lines.Length) + return false; + if (!IsGitLfsSha256OidLine(lines[lineIndex])) + return false; + lineIndex++; + if (!IsGitLfsSizeLine(lines[lineIndex])) + return false; + + return lineIndex == lines.Length - 1; + } + + private static bool IsGitLfsSha256OidLine(string line) + { + const string prefix = "oid sha256:"; + if (!line.StartsWith(prefix, StringComparison.Ordinal)) + return false; + + var hash = line.AsSpan(prefix.Length); + if (hash.Length != 64) + return false; + foreach (var c in hash) + { + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))) + return false; + } + return true; + } + + private static bool IsGitLfsSizeLine(string line) + { + const string prefix = "size "; + if (!line.StartsWith(prefix, StringComparison.Ordinal)) + return false; + + var size = line.AsSpan(prefix.Length); + if (size.Length == 0) + return false; + foreach (var c in size) + { + if (c < '0' || c > '9') + return false; + } + return true; + } + /// /// Validate file content for encoding issues. /// ファイル内容のエンコーディング問題を検証する。 @@ -3212,6 +3269,17 @@ public static List ValidateContent(string relativePath, byte[] rawByt { var issues = new List(); + if (IsGitLfsPointer(rawBytes)) + { + issues.Add(new FileIssue + { + Path = relativePath, + Kind = "lfs_pointer_skipped", + Line = 1, + Message = "Git LFS pointer file skipped; fetch LFS objects to index real file content", + }); + } + // UTF-16 BOM-detected files are decoded as UTF-16 in BuildRecordWithRawBytes, so the // raw-byte heuristics for `bom` / `null_byte` / `mixed_line_endings` would all misfire // (every UTF-16 LE character ASCII point looks like a NUL byte; CRLF appears as 0D 00 diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 058274e251..d654f061a3 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -1311,6 +1311,79 @@ public void ScanFiles_SkipsDirectorySymlinkPointingAtAncestor() } } + [Fact] + public void ScanFiles_FollowAllSymlinks_SkipsAlreadyVisitedDirectoryTarget() + { + if (OperatingSystem.IsWindows()) + return; // Creating symlinks on Windows requires admin/developer mode / Windows で symlink 作成には管理者権限が必要 + + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + var subDir = Path.Combine(tempDir, "sub"); + Directory.CreateDirectory(subDir); + File.WriteAllText(Path.Combine(subDir, "foo.py"), "def hello(): pass\n"); + Directory.CreateSymbolicLink(Path.Combine(subDir, "parent_loop"), ".."); + + var indexer = new FileIndexer( + tempDir, + ignoreCase: false, + ignoreRuleRoot: null, + maxFileSizeBytes: null, + directoryIgnoreCaseProbe: null, + symlinkPolicy: FileIndexer.SymlinkPolicy.All); + var result = indexer.ScanFilesDetailed(); + var files = result.Files + .Select(path => Path.GetRelativePath(tempDir, path).Replace('\\', '/')) + .OrderBy(path => path, StringComparer.Ordinal) + .ToList(); + + Assert.Equal(["sub/foo.py"], files); + Assert.Contains( + result.Errors, + error => error.Severity == FileIndexer.ScanIssueSeverity.Warning + && error.Path == "sub/parent_loop" + && error.Message.Contains("already scanned", StringComparison.OrdinalIgnoreCase)); + Assert.False(result.HadErrors); + } + finally + { + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + } + + [Fact] + public void ScanFiles_ExcessiveDirectoryDepth_SkipsSubtreeWithWarning() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + var current = tempDir; + for (var i = 0; i < 130; i++) + { + current = Path.Combine(current, $"d{i:D3}"); + Directory.CreateDirectory(current); + } + File.WriteAllText(Path.Combine(current, "too_deep.py"), "print('too deep')\n"); + + var result = new FileIndexer(tempDir).ScanFilesDetailed(); + + Assert.Empty(result.Files); + Assert.Contains( + result.Errors, + error => error.Severity == FileIndexer.ScanIssueSeverity.Warning + && error.Message.Contains("traversal depth exceeded", StringComparison.OrdinalIgnoreCase)); + Assert.False(result.HadErrors); + } + finally + { + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + } + [Fact] public void ScanFiles_SkipsFileSymlinkToRealFileInProject() { @@ -2787,8 +2860,8 @@ public void BuildRecord_HandlesUnicodeAndCjkContent() [InlineData("class A\r\n{\r\n}\r\n", 3)] [InlineData("class A\n{\n}\n", 3)] [InlineData("class A\r{\r}\r", 3)] - [InlineData("\uFEFF", 1)] - public void BuildRecord_CountsPhysicalLinesBeforeLineLeadingInvisibleStripping(string content, int expectedLines) + [InlineData("\uFEFF", 0)] + public void BuildRecord_CountsPhysicalLinesAfterLineLeadingInvisibleStripping(string content, int expectedLines) { var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); try @@ -3601,12 +3674,12 @@ public void BuildRecord_LeadingBomStrippedFromContent() { // Files whose on-disk bytes begin with UTF-8 BOM (EF BB BF) must have the BOM // stripped from the decoded content so downstream consumers never see a phantom - // U+FEFF glyph on line 1. The checksum must still reflect the BOM bytes so adding - // or removing the BOM keeps triggering incremental change detection. Closes #183. + // U+FEFF glyph on line 1. The checksum must reflect the same canonical content + // stored in chunks so BOM-only changes do not desynchronize line metadata. Closes #183/#1467. // オンディスク先頭に UTF-8 BOM (EF BB BF) を持つファイルは、デコード後の content - // から BOM を剥がし、下流に幽霊 U+FEFF を渡さないようにする。checksum は BOM の - // バイトを含めたまま算出し、BOM 追加/削除をインクリメンタル更新判定で引き続き - // 検知できるようにする。Closes #183. + // から BOM を剥がし、下流に幽霊 U+FEFF を渡さないようにする。checksum は chunk + // に保存される canonical content と同じ内容から算出し、行メタデータとのずれを防ぐ。 + // Closes #183/#1467. var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); try { @@ -3627,19 +3700,9 @@ public void BuildRecord_LeadingBomStrippedFromContent() // 文字列オーバーロードではなくコードポイントで確認する。 Assert.DoesNotContain('\uFEFF', content); Assert.Equal(2, record.Lines); - // Pin the BOM-detection contract: BOM bytes feed into the checksum hash input - // so adding or removing a BOM still flips the checksum and triggers incremental - // re-index. This test's payload has no CR bytes, so the line-ending normalization - // added for #1544 is a no-op and the expected value still matches raw-byte SHA256. - // Cross-OS CRLF / LF parity is covered by BuildRecord_Checksum_CrlfAndLfMatch. - // Closes #183. - // BOM 検知契約を固定: BOM のバイトは checksum のハッシュ入力にそのまま含まれ、 - // BOM の追加 / 削除でハッシュが変化することでインクリメンタル再索引が走る。 - // このテストの payload には CR が無いため #1544 の改行正規化は no-op となり、 - // 期待値は生バイトの SHA256 と一致する。OS をまたいだ CRLF / LF の同一性は - // BuildRecord_Checksum_CrlfAndLfMatch で担保する。Closes #183. var expectedChecksum = Convert.ToHexString( - System.Security.Cryptography.SHA256.HashData(rawBytes)).ToLowerInvariant(); + System.Security.Cryptography.SHA256.HashData( + System.Text.Encoding.UTF8.GetBytes(content))).ToLowerInvariant(); Assert.Equal(expectedChecksum, record.Checksum); } finally @@ -3694,14 +3757,14 @@ public void BuildRecord_Checksum_CrlfAndLfMatch() } [Fact] - public void BuildRecord_Checksum_BomAddRemoveStillDetected() + public void BuildRecord_Checksum_BomAddRemoveUsesSameCanonicalContent() { - // BOM bytes (EF BB BF) must still be part of the checksum hash input so a clone - // that gained or lost a leading BOM is detected as changed by incremental re-index. - // Only CRLF / CR are collapsed; BOM passes through unchanged. Closes #1544. - // BOM のバイト (EF BB BF) はハッシュ入力に残し、BOM の有無が変わった clone を - // インクリメンタル再索引で変更として検知できるようにする。畳むのは CRLF / CR のみで - // BOM はそのまま通す。Closes #1544. + // BOM-only edits should hash to the same canonical content that chunking and + // excerpts see. This prevents freshness checks from accepting line metadata + // produced from a different byte sequence. Closes #1467. + // BOM のみの差分は chunk / excerpt が見る canonical content と同じ内容として + // hash される。別のバイト列から作られた行メタデータを freshness が受け入れる + // ずれを防ぐ。Closes #1467. var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); try { @@ -3716,7 +3779,7 @@ public void BuildRecord_Checksum_BomAddRemoveStillDetected() var (bomRecord, _, _) = indexer.BuildRecord(bomPath); var (noBomRecord, _, _) = indexer.BuildRecord(noBomPath); - Assert.NotEqual(bomRecord.Checksum, noBomRecord.Checksum); + Assert.Equal(bomRecord.Checksum, noBomRecord.Checksum); } finally { @@ -3770,15 +3833,14 @@ public void ComputeChecksum_ReturnsLowercaseHex() } [Fact] - public void BuildRecord_BomOnlyFile_ReportsOnePhysicalLine() + public void BuildRecord_BomOnlyFile_ReportsNoCanonicalLines() { // A file whose on-disk bytes are exactly the UTF-8 BOM (EF BB BF) and - // nothing else still has one physical decoded source line, even though the - // normalized content handed to chunking/extraction becomes empty. This pins - // the line-number contract used for stale-line detection. Closes #1890. + // nothing else becomes empty canonical content, so the stored line count must + // match the chunking/extraction input. Closes #1467/#1890. // オンディスクバイト列が UTF-8 BOM (EF BB BF) のみのファイルも、正規化後に - // chunk/extraction へ渡す content は空になるが、デコード済み元ソースとしては - // 1 つの物理行を持つ。この stale line 検出用の行番号契約を固定する。Closes #1890. + // chunk/extraction へ渡す canonical content が空になるため、保存する行数も + // その入力と一致させる。Closes #1467/#1890. var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); try { @@ -3790,7 +3852,7 @@ public void BuildRecord_BomOnlyFile_ReportsOnePhysicalLine() var (record, content, _) = indexer.BuildRecord(filePath); Assert.Equal(string.Empty, content); - Assert.Equal(1, record.Lines); + Assert.Equal(0, record.Lines); } finally { @@ -4133,6 +4195,90 @@ public void StripLineLeadingInvisibles_ConsecutiveLineLeadingInvisibles_AllStrip Assert.Equal("hello\nworld\n", output); } + [Fact] + public void BuildRecord_ChecksumUsesCanonicalContentAfterLineLeadingBomStrip() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + var plainPath = Path.Combine(tempDir, "plain.cs"); + var bomPath = Path.Combine(tempDir, "bom.cs"); + File.WriteAllText(plainPath, "class Plain\n{\n}\n"); + File.WriteAllText(bomPath, "\uFEFFclass Plain\n\uFEFF{\n}\n"); + + var indexer = new FileIndexer(tempDir); + var (plainRecord, plainContent, _) = indexer.BuildRecord(plainPath); + var (bomRecord, bomContent, _) = indexer.BuildRecord(bomPath); + + Assert.Equal(plainContent, bomContent); + Assert.Equal(plainRecord.Checksum, bomRecord.Checksum); + Assert.Equal(plainRecord.Lines, bomRecord.Lines); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public void BuildRecord_GitLfsPointerIndexesEmptyBodyAndValidationIssue() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + var filePath = Path.Combine(tempDir, "asset.cs"); + var pointer = """ + version https://git-lfs.github.com/spec/v1 + oid sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + size 12345 + """; + File.WriteAllText(filePath, pointer); + + var indexer = new FileIndexer(tempDir); + var (record, content, rawBytes, _) = indexer.BuildRecordWithRawBytes(filePath); + var issues = FileIndexer.ValidateContent(record.Path, rawBytes, content); + + Assert.Equal(string.Empty, content); + Assert.Equal(0, record.Lines); + var issue = Assert.Single(issues, i => i.Kind == "lfs_pointer_skipped"); + Assert.Equal("asset.cs", issue.Path); + Assert.Equal(1, issue.Line); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public void BuildRecord_GitLfsVersionLineWithoutPointerShapePreservesContent() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + var filePath = Path.Combine(tempDir, "example.txt"); + var text = """ + version https://git-lfs.github.com/spec/v1 + This is documentation text, not a Git LFS pointer. + """; + File.WriteAllText(filePath, text); + + var indexer = new FileIndexer(tempDir); + var (record, content, rawBytes, _) = indexer.BuildRecordWithRawBytes(filePath); + var issues = FileIndexer.ValidateContent(record.Path, rawBytes, content); + + Assert.Equal(text.Replace("\r\n", "\n", StringComparison.Ordinal), content); + Assert.DoesNotContain(issues, issue => issue.Kind == "lfs_pointer_skipped"); + } + finally + { + Directory.Delete(tempDir, true); + } + } + [Fact] public void BuildRecord_ThrowsFileTooLargeSkippedExceptionForOversizedFile() {