From e67966a3730214f7c8aa242e23e84f675d2b18df Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 05:10:19 +0900 Subject: [PATCH 1/2] Fix Windows device path indexing (#2043) --- changelog.d/unreleased/2043.fixed.md | 16 +++++++ src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 46 +++++++++++++++++++ tests/CodeIndex.Tests/FileIndexerTests.cs | 22 +++++++++ 3 files changed, 84 insertions(+) create mode 100644 changelog.d/unreleased/2043.fixed.md diff --git a/changelog.d/unreleased/2043.fixed.md b/changelog.d/unreleased/2043.fixed.md new file mode 100644 index 0000000000..1fa787eaa2 --- /dev/null +++ b/changelog.d/unreleased/2043.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 2043 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs +--- + +## English + +- **Windows device paths are skipped during indexing (#2043)** - `cdidx` now rejects reserved device paths such as `\\.\COM1`, `NUL`, `CON`, and `LPT1` before probing file contents. + +## 日本語 + +- **Windows のデバイスパスを索引対象から除外しました (#2043)** - `\\.\COM1`、`NUL`、`CON`、`LPT1` などの予約デバイスパスを、ファイル内容の読み取り前に拒否します。 diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index afe8a4b4fc..ad78309131 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -1202,6 +1202,49 @@ private static bool LooksLikeCppHeaderLine(ReadOnlySpan line) internal static bool CanIndexFile(string filePath) => GetFileIndexability(filePath) == FileProbeStatus.Supported; + internal static bool IsWindowsDevicePath(string filePath) + { + if (string.IsNullOrWhiteSpace(filePath)) + return false; + + var normalized = filePath.Replace('\\', '/'); + if (normalized.StartsWith("//./", StringComparison.Ordinal) + || normalized.StartsWith("//?/GLOBALROOT/Device/", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + foreach (var segment in normalized.Split('/', StringSplitOptions.RemoveEmptyEntries)) + { + var name = segment; + var extensionIndex = name.IndexOf('.'); + if (extensionIndex >= 0) + name = name[..extensionIndex]; + + if (IsWindowsReservedDeviceName(name)) + return true; + } + + return false; + } + + private static bool IsWindowsReservedDeviceName(string name) + { + if (name.Equals("CON", StringComparison.OrdinalIgnoreCase) + || name.Equals("PRN", StringComparison.OrdinalIgnoreCase) + || name.Equals("AUX", StringComparison.OrdinalIgnoreCase) + || name.Equals("NUL", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return name.Length == 4 + && (name.StartsWith("COM", StringComparison.OrdinalIgnoreCase) + || name.StartsWith("LPT", StringComparison.OrdinalIgnoreCase)) + && name[3] >= '1' + && name[3] <= '9'; + } + internal static bool HasSkippedAttributes(FileAttributes attributes, bool isWindows) { if ((attributes & FileAttributes.ReparsePoint) != 0) @@ -1246,6 +1289,9 @@ private static bool HasSkippedAttributes(string path) internal static FileProbeStatus GetFileIndexability(string filePath) { + if (OperatingSystem.IsWindows() && IsWindowsDevicePath(filePath)) + return FileProbeStatus.Unsupported; + // Reject symlinks/reparse points here so every caller (full scan, --files / --commits update mode, // dry-run) gets the same skip behavior. On Windows, Hidden/System paths are also rejected to avoid // indexing OS-owned caches such as System Volume Information and $Recycle.Bin during broad scans. diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 06773c537c..8632a86d7d 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -1258,6 +1258,28 @@ public void GetFileIndexability_RejectsFileSymlinkSoUpdateModeSkipsIt() } } + [Theory] + [InlineData(@"\\.\COM1")] + [InlineData(@"\\.\NUL")] + [InlineData(@"\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopy1")] + [InlineData(@"C:\repo\AUX.cs")] + [InlineData(@"C:\repo\con.txt")] + [InlineData(@"C:\repo\COM9")] + [InlineData(@"C:\repo\LPT1.log")] + public void IsWindowsDevicePath_RejectsReservedDeviceNames(string path) + { + Assert.True(FileIndexer.IsWindowsDevicePath(path)); + } + + [Theory] + [InlineData(@"C:\repo\COM10.cs")] + [InlineData(@"C:\repo\company.cs")] + [InlineData(@"C:\repo\template1.cs")] + public void IsWindowsDevicePath_AllowsOrdinaryNames(string path) + { + Assert.False(FileIndexer.IsWindowsDevicePath(path)); + } + [Theory] [InlineData(FileAttributes.ReparsePoint, false, true)] [InlineData(FileAttributes.ReparsePoint, true, true)] From 20d35e20533cfb578500d118bef999625de5496c Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 05:12:54 +0900 Subject: [PATCH 2/2] Handle UTF-16 NUL-byte detection (#1829) --- changelog.d/unreleased/1829.fixed.md | 16 +++ src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 106 ++++++++++++++---- tests/CodeIndex.Tests/FileIndexerTests.cs | 71 ++++++++++++ 3 files changed, 174 insertions(+), 19 deletions(-) create mode 100644 changelog.d/unreleased/1829.fixed.md diff --git a/changelog.d/unreleased/1829.fixed.md b/changelog.d/unreleased/1829.fixed.md new file mode 100644 index 0000000000..c682a673c9 --- /dev/null +++ b/changelog.d/unreleased/1829.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1829 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs +--- + +## English + +- **UTF-16 text is no longer skipped as binary because of NUL bytes (#1829)** - `cdidx` now treats BOM-less UTF-16 LE/BE byte patterns as text and suppresses raw NUL-byte validation for those files. + +## 日本語 + +- **UTF-16 テキストを NUL バイト理由でバイナリ扱いしないようにしました (#1829)** - BOM なし UTF-16 LE/BE のバイトパターンをテキストとして扱い、それらのファイルでは生 NUL バイト検証を抑止します。 diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index ad78309131..3f249a430e 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -2552,7 +2552,9 @@ public static string NormalizePathSeparators(string path) // Closes #1544. var checksum = ComputeChecksum(bytes); - if (ContainsIndexBlockingNullByte(bytes)) + var isUtf16Encoded = TryDetectUtf16Encoding(bytes, allowHeuristic: true, out var utf16BigEndian, out var hasUtf16Bom); + + if (!isUtf16Encoded && ContainsIndexBlockingNullByte(bytes)) throw new BinaryFileSkippedException($"{relativePath}: binary file skipped because it contains NULL bytes"); string content; @@ -2566,15 +2568,9 @@ public static string NormalizePathSeparators(string path) // デコーダで毎バイト U+FFFD / NUL に変換され、ファイル内のシンボルが丸ごと // 消える。UTF-32 LE は UTF-16 LE と先頭 2 バイトを共有する (FF FE [00 00]) // ため、UTF-16 LE 経路から除外し UTF-8 fallback に流す。Closes #1540. - if (bytes.Length >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF) - { - content = new UnicodeEncoding(bigEndian: true, byteOrderMark: false, throwOnInvalidBytes: false) - .GetString(bytes); - } - else if (bytes.Length >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE - && !(bytes.Length >= 4 && bytes[2] == 0x00 && bytes[3] == 0x00)) + if (isUtf16Encoded) { - content = new UnicodeEncoding(bigEndian: false, byteOrderMark: false, throwOnInvalidBytes: false) + content = new UnicodeEncoding(utf16BigEndian, byteOrderMark: hasUtf16Bom, throwOnInvalidBytes: false) .GetString(bytes); } else @@ -2933,19 +2929,16 @@ public static List ValidateContent(string relativePath, byte[] rawByt // (UTF-16 LE では ASCII 部の片バイトが NUL、CRLF は 0D 00 0A 00)。代わりに // `utf16_bom` 1 件を出して `validate` が「UTF-16 として解釈した」ことを示し、 // 不正サロゲートペアに備え content 側 U+FFFD 走査は継続する。Closes #1540. - var hasUtf16BeBom = rawBytes.Length >= 2 && rawBytes[0] == 0xFE && rawBytes[1] == 0xFF; - var hasUtf16LeBom = rawBytes.Length >= 2 && rawBytes[0] == 0xFF && rawBytes[1] == 0xFE - && !(rawBytes.Length >= 4 && rawBytes[2] == 0x00 && rawBytes[3] == 0x00); - var isUtf16 = hasUtf16BeBom || hasUtf16LeBom; + var isUtf16 = TryDetectUtf16Encoding(rawBytes, allowHeuristic: true, out var utf16BigEndian, out var hasUtf16Bom); - if (isUtf16) + if (isUtf16 && hasUtf16Bom) { issues.Add(new FileIssue { Path = relativePath, Kind = "utf16_bom", Line = 1, - Message = hasUtf16BeBom + Message = utf16BigEndian ? "UTF-16 BE BOM detected (decoded as UTF-16)" : "UTF-16 LE BOM detected (decoded as UTF-16)", }); @@ -3148,12 +3141,87 @@ public static List ValidateContent(string relativePath, byte[] rawByt internal static bool ContainsIndexBlockingNullByte(byte[] rawBytes) { - var hasUtf16BeBom = rawBytes.Length >= 2 && rawBytes[0] == 0xFE && rawBytes[1] == 0xFF; - var hasUtf16LeBom = rawBytes.Length >= 2 && rawBytes[0] == 0xFF && rawBytes[1] == 0xFE - && !(rawBytes.Length >= 4 && rawBytes[2] == 0x00 && rawBytes[3] == 0x00); - return !hasUtf16BeBom && !hasUtf16LeBom && rawBytes.Any(b => b == 0); + return !TryDetectUtf16Encoding(rawBytes, allowHeuristic: true, out _, out _) && rawBytes.Any(b => b == 0); + } + + internal static bool TryDetectUtf16Encoding( + byte[] rawBytes, + bool allowHeuristic, + out bool bigEndian, + out bool hasBom) + { + bigEndian = false; + hasBom = false; + + if (rawBytes.Length >= 2 && rawBytes[0] == 0xFE && rawBytes[1] == 0xFF) + { + bigEndian = true; + hasBom = true; + return true; + } + + if (rawBytes.Length >= 2 && rawBytes[0] == 0xFF && rawBytes[1] == 0xFE + && !(rawBytes.Length >= 4 && rawBytes[2] == 0x00 && rawBytes[3] == 0x00)) + { + hasBom = true; + return true; + } + + if (!allowHeuristic || rawBytes.Length < 4) + return false; + + var sampleLength = Math.Min(rawBytes.Length, 4096); + sampleLength -= sampleLength % 2; + var pairs = sampleLength / 2; + if (pairs == 0) + return false; + + var evenNulls = 0; + var oddNulls = 0; + var oddTextBytes = 0; + var evenTextBytes = 0; + for (var i = 0; i < sampleLength; i += 2) + { + if (rawBytes[i] == 0) + evenNulls++; + if (rawBytes[i + 1] == 0) + oddNulls++; + if (IsLikelyTextByte(rawBytes[i + 1])) + oddTextBytes++; + if (IsLikelyTextByte(rawBytes[i])) + evenTextBytes++; + } + + const double NullParityThreshold = 0.30; + const double OppositeNullThreshold = 0.01; + const double TextByteThreshold = 0.80; + var beScore = (double)evenNulls / pairs; + var leScore = (double)oddNulls / pairs; + var beOppositeScore = (double)oddNulls / pairs; + var leOppositeScore = (double)evenNulls / pairs; + + if (beScore >= NullParityThreshold + && beOppositeScore <= OppositeNullThreshold + && (double)oddTextBytes / pairs >= TextByteThreshold) + { + bigEndian = true; + return true; + } + + if (leScore >= NullParityThreshold + && leOppositeScore <= OppositeNullThreshold + && (double)evenTextBytes / pairs >= TextByteThreshold) + { + bigEndian = false; + return true; + } + + return false; } + private static bool IsLikelyTextByte(byte value) + => value is 0x09 or 0x0A or 0x0D || value >= 0x20; + internal sealed class BinaryFileSkippedException(string message) : InvalidOperationException(message); internal sealed class FileTooLargeSkippedException( diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 8632a86d7d..39621cdf4f 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -3365,6 +3365,64 @@ public void BuildRecord_Utf16BeBomFile_DecodedAsUtf16() } } + [Fact] + public void BuildRecord_Utf16LeWithoutBomFile_DecodedAsUtf16() + { + // Legacy Windows tools can save source files as UTF-16 LE without a BOM. The + // regular every-other-byte NUL pattern should be treated as an encoding signal, + // not as binary content. Closes #1829. + // 古い Windows ツールは BOM なし UTF-16 LE でソースを保存することがある。 + // 1 バイトおきの NUL パターンはバイナリ混入ではなくエンコーディングのシグナルとして扱う。 + // Closes #1829. + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + var filePath = Path.Combine(tempDir, "utf16le-nobom.cs"); + var payload = "using System;\nnamespace Utf16LeNoBom;\n"; + File.WriteAllBytes(filePath, System.Text.Encoding.Unicode.GetBytes(payload)); + + var indexer = new FileIndexer(tempDir); + var (_, content, _, warning) = indexer.BuildRecordWithRawBytes(filePath); + + Assert.Null(warning); + Assert.Contains("namespace Utf16LeNoBom;", content); + Assert.False(FileIndexer.ContainsIndexBlockingNullByte(System.Text.Encoding.Unicode.GetBytes(payload))); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public void BuildRecord_Utf16BeWithoutBomFile_DecodedAsUtf16() + { + // The heuristic must also handle BOM-less UTF-16 BE text so the fix is not tied + // to little-endian Windows output only. Closes #1829. + // BOM なし UTF-16 BE テキストも扱い、little-endian Windows 出力だけに限定しない。 + // Closes #1829. + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + var filePath = Path.Combine(tempDir, "utf16be-nobom.cs"); + var payload = "using System;\nnamespace Utf16BeNoBom;\n"; + File.WriteAllBytes(filePath, System.Text.Encoding.BigEndianUnicode.GetBytes(payload)); + + var indexer = new FileIndexer(tempDir); + var (_, content, _, warning) = indexer.BuildRecordWithRawBytes(filePath); + + Assert.Null(warning); + Assert.Contains("namespace Utf16BeNoBom;", content); + Assert.False(FileIndexer.ContainsIndexBlockingNullByte(System.Text.Encoding.BigEndianUnicode.GetBytes(payload))); + } + finally + { + Directory.Delete(tempDir, true); + } + } + [Fact] public void ValidateContent_Utf16LeBomFile_EmitsUtf16BomNotRawByteIssues() { @@ -3392,6 +3450,19 @@ public void ValidateContent_Utf16LeBomFile_EmitsUtf16BomNotRawByteIssues() Assert.DoesNotContain(issues, i => i.Kind == "non_utf8_likely"); } + [Fact] + public void ValidateContent_Utf16WithoutBom_DoesNotEmitNullByteIssue() + { + var payload = "using System;\nclass C { }\n"; + var rawBytes = System.Text.Encoding.Unicode.GetBytes(payload); + + var issues = FileIndexer.ValidateContent("utf16le-nobom.cs", rawBytes, payload); + + Assert.DoesNotContain(issues, i => i.Kind == "utf16_bom"); + Assert.DoesNotContain(issues, i => i.Kind == "null_byte"); + Assert.DoesNotContain(issues, i => i.Kind == "mixed_line_endings"); + } + [Fact] public void ValidateContent_HighFffdRatio_EmitsAggregateNonUtf8Likely() {