Fix Windows device paths and UTF-16 NUL detection

changelog.d/unreleased/1829.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+issues:
+  - 1829
+affected:
+  - src/CodeIndex/Indexer/Scanning/FileIndexer.cs
+  - tests/CodeIndex.Tests/FileIndexerTests.cs
+---
+
+## English
+
+- **UTF-16 text is no longer skipped as binary because of NUL bytes (#1829)** - `cdidx` now treats BOM-less UTF-16 LE/BE byte patterns as text and suppresses raw NUL-byte validation for those files.
+
+## 日本語
+
+- **UTF-16 テキストを NUL バイト理由でバイナリ扱いしないようにしました (#1829)** - BOM なし UTF-16 LE/BE のバイトパターンをテキストとして扱い、それらのファイルでは生 NUL バイト検証を抑止します。

changelog.d/unreleased/2043.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+issues:
+  - 2043
+affected:
+  - src/CodeIndex/Indexer/Scanning/FileIndexer.cs
+  - tests/CodeIndex.Tests/FileIndexerTests.cs
+---
+
+## English
+
+- **Windows device paths are skipped during indexing (#2043)** - `cdidx` now rejects reserved device paths such as `\\.\COM1`, `NUL`, `CON`, and `LPT1` before probing file contents.
+
+## 日本語
+
+- **Windows のデバイスパスを索引対象から除外しました (#2043)** - `\\.\COM1`、`NUL`、`CON`、`LPT1` などの予約デバイスパスを、ファイル内容の読み取り前に拒否します。

src/CodeIndex/Indexer/Scanning/FileIndexer.cs

@@ -1202,6 +1202,49 @@ private static bool LooksLikeCppHeaderLine(ReadOnlySpan<char> line)
     internal static bool CanIndexFile(string filePath)
         => GetFileIndexability(filePath) == FileProbeStatus.Supported;
 
+    internal static bool IsWindowsDevicePath(string filePath)
+    {
+        if (string.IsNullOrWhiteSpace(filePath))
+            return false;
+
+        var normalized = filePath.Replace('\\', '/');
+        if (normalized.StartsWith("//./", StringComparison.Ordinal)
+            || normalized.StartsWith("//?/GLOBALROOT/Device/", StringComparison.OrdinalIgnoreCase))
+        {
+            return true;
+        }
+
+        foreach (var segment in normalized.Split('/', StringSplitOptions.RemoveEmptyEntries))
+        {
+            var name = segment;
+            var extensionIndex = name.IndexOf('.');
+            if (extensionIndex >= 0)
+                name = name[..extensionIndex];
+
+            if (IsWindowsReservedDeviceName(name))
+                return true;
+        }
+
+        return false;
+    }
+
+    private static bool IsWindowsReservedDeviceName(string name)
+    {
+        if (name.Equals("CON", StringComparison.OrdinalIgnoreCase)
+            || name.Equals("PRN", StringComparison.OrdinalIgnoreCase)
+            || name.Equals("AUX", StringComparison.OrdinalIgnoreCase)
+            || name.Equals("NUL", StringComparison.OrdinalIgnoreCase))
+        {
+            return true;
+        }
+
+        return name.Length == 4
+            && (name.StartsWith("COM", StringComparison.OrdinalIgnoreCase)
+                || name.StartsWith("LPT", StringComparison.OrdinalIgnoreCase))
+            && name[3] >= '1'
+            && name[3] <= '9';
+    }
+
     internal static bool HasSkippedAttributes(FileAttributes attributes, bool isWindows)
     {
         if ((attributes & FileAttributes.ReparsePoint) != 0)
@@ -1246,6 +1289,9 @@ private static bool HasSkippedAttributes(string path)
 
     internal static FileProbeStatus GetFileIndexability(string filePath)
     {
+        if (OperatingSystem.IsWindows() && IsWindowsDevicePath(filePath))
+            return FileProbeStatus.Unsupported;
+
         // Reject symlinks/reparse points here so every caller (full scan, --files / --commits update mode,
         // dry-run) gets the same skip behavior. On Windows, Hidden/System paths are also rejected to avoid
         // indexing OS-owned caches such as System Volume Information and $Recycle.Bin during broad scans.
@@ -2506,7 +2552,9 @@ public static string NormalizePathSeparators(string path)
         // Closes #1544.
         var checksum = ComputeChecksum(bytes);
 
-        if (ContainsIndexBlockingNullByte(bytes))
+        var isUtf16Encoded = TryDetectUtf16Encoding(bytes, allowHeuristic: true, out var utf16BigEndian, out var hasUtf16Bom);
+
+        if (!isUtf16Encoded && ContainsIndexBlockingNullByte(bytes))
             throw new BinaryFileSkippedException($"{relativePath}: binary file skipped because it contains NULL bytes");
 
         string content;
@@ -2520,15 +2568,9 @@ public static string NormalizePathSeparators(string path)
         // デコーダで毎バイト U+FFFD / NUL に変換され、ファイル内のシンボルが丸ごと
         // 消える。UTF-32 LE は UTF-16 LE と先頭 2 バイトを共有する (FF FE [00 00])
         // ため、UTF-16 LE 経路から除外し UTF-8 fallback に流す。Closes #1540.
-        if (bytes.Length >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF)
-        {
-            content = new UnicodeEncoding(bigEndian: true, byteOrderMark: false, throwOnInvalidBytes: false)
-                .GetString(bytes);
-        }
-        else if (bytes.Length >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE
-                 && !(bytes.Length >= 4 && bytes[2] == 0x00 && bytes[3] == 0x00))
+        if (isUtf16Encoded)
         {
-            content = new UnicodeEncoding(bigEndian: false, byteOrderMark: false, throwOnInvalidBytes: false)
+            content = new UnicodeEncoding(utf16BigEndian, byteOrderMark: hasUtf16Bom, throwOnInvalidBytes: false)
                 .GetString(bytes);
         }
         else
@@ -2887,19 +2929,16 @@ public static List<FileIssue> ValidateContent(string relativePath, byte[] rawByt
         // (UTF-16 LE では ASCII 部の片バイトが NUL、CRLF は 0D 00 0A 00)。代わりに
         // `utf16_bom` 1 件を出して `validate` が「UTF-16 として解釈した」ことを示し、
         // 不正サロゲートペアに備え content 側 U+FFFD 走査は継続する。Closes #1540.
-        var hasUtf16BeBom = rawBytes.Length >= 2 && rawBytes[0] == 0xFE && rawBytes[1] == 0xFF;
-        var hasUtf16LeBom = rawBytes.Length >= 2 && rawBytes[0] == 0xFF && rawBytes[1] == 0xFE
-            && !(rawBytes.Length >= 4 && rawBytes[2] == 0x00 && rawBytes[3] == 0x00);
-        var isUtf16 = hasUtf16BeBom || hasUtf16LeBom;
+        var isUtf16 = TryDetectUtf16Encoding(rawBytes, allowHeuristic: true, out var utf16BigEndian, out var hasUtf16Bom);
 
-        if (isUtf16)
+        if (isUtf16 && hasUtf16Bom)
         {
             issues.Add(new FileIssue
             {
                 Path = relativePath,
                 Kind = "utf16_bom",
                 Line = 1,
-                Message = hasUtf16BeBom
+                Message = utf16BigEndian
                     ? "UTF-16 BE BOM detected (decoded as UTF-16)"
                     : "UTF-16 LE BOM detected (decoded as UTF-16)",
             });
@@ -3102,12 +3141,87 @@ public static List<FileIssue> ValidateContent(string relativePath, byte[] rawByt
 
     internal static bool ContainsIndexBlockingNullByte(byte[] rawBytes)
     {
-        var hasUtf16BeBom = rawBytes.Length >= 2 && rawBytes[0] == 0xFE && rawBytes[1] == 0xFF;
-        var hasUtf16LeBom = rawBytes.Length >= 2 && rawBytes[0] == 0xFF && rawBytes[1] == 0xFE
-            && !(rawBytes.Length >= 4 && rawBytes[2] == 0x00 && rawBytes[3] == 0x00);
-        return !hasUtf16BeBom && !hasUtf16LeBom && rawBytes.Any(b => b == 0);
+        return !TryDetectUtf16Encoding(rawBytes, allowHeuristic: true, out _, out _) && rawBytes.Any(b => b == 0);
     }
 
+    internal static bool TryDetectUtf16Encoding(
+        byte[] rawBytes,
+        bool allowHeuristic,
+        out bool bigEndian,
+        out bool hasBom)
+    {
+        bigEndian = false;
+        hasBom = false;
+
+        if (rawBytes.Length >= 2 && rawBytes[0] == 0xFE && rawBytes[1] == 0xFF)
+        {
+            bigEndian = true;
+            hasBom = true;
+            return true;
+        }
+
+        if (rawBytes.Length >= 2 && rawBytes[0] == 0xFF && rawBytes[1] == 0xFE
+            && !(rawBytes.Length >= 4 && rawBytes[2] == 0x00 && rawBytes[3] == 0x00))
+        {
+            hasBom = true;
+            return true;
+        }
+
+        if (!allowHeuristic || rawBytes.Length < 4)
+            return false;
+
+        var sampleLength = Math.Min(rawBytes.Length, 4096);
+        sampleLength -= sampleLength % 2;
+        var pairs = sampleLength / 2;
+        if (pairs == 0)
+            return false;
+
+        var evenNulls = 0;
+        var oddNulls = 0;
+        var oddTextBytes = 0;
+        var evenTextBytes = 0;
+        for (var i = 0; i < sampleLength; i += 2)
+        {
+            if (rawBytes[i] == 0)
+                evenNulls++;
+            if (rawBytes[i + 1] == 0)
+                oddNulls++;
+            if (IsLikelyTextByte(rawBytes[i + 1]))
+                oddTextBytes++;
+            if (IsLikelyTextByte(rawBytes[i]))
+                evenTextBytes++;
+        }
+
+        const double NullParityThreshold = 0.30;
+        const double OppositeNullThreshold = 0.01;
+        const double TextByteThreshold = 0.80;
+        var beScore = (double)evenNulls / pairs;
+        var leScore = (double)oddNulls / pairs;
+        var beOppositeScore = (double)oddNulls / pairs;
+        var leOppositeScore = (double)evenNulls / pairs;
+
+        if (beScore >= NullParityThreshold
+            && beOppositeScore <= OppositeNullThreshold
+            && (double)oddTextBytes / pairs >= TextByteThreshold)
+        {
+            bigEndian = true;
+            return true;
+        }
+
+        if (leScore >= NullParityThreshold
+            && leOppositeScore <= OppositeNullThreshold
+            && (double)evenTextBytes / pairs >= TextByteThreshold)
+        {
+            bigEndian = false;
+            return true;
+        }
+
+        return false;
+    }
+
+    private static bool IsLikelyTextByte(byte value)
+        => value is 0x09 or 0x0A or 0x0D || value >= 0x20;
+
     internal sealed class BinaryFileSkippedException(string message) : InvalidOperationException(message);
 
     internal sealed class FileTooLargeSkippedException(

tests/CodeIndex.Tests/FileIndexerTests.cs

@@ -1258,6 +1258,28 @@ public void GetFileIndexability_RejectsFileSymlinkSoUpdateModeSkipsIt()
         }
     }
 
+    [Theory]
+    [InlineData(@"\\.\COM1")]
+    [InlineData(@"\\.\NUL")]
+    [InlineData(@"\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopy1")]
+    [InlineData(@"C:\repo\AUX.cs")]
+    [InlineData(@"C:\repo\con.txt")]
+    [InlineData(@"C:\repo\COM9")]
+    [InlineData(@"C:\repo\LPT1.log")]
+    public void IsWindowsDevicePath_RejectsReservedDeviceNames(string path)
+    {
+        Assert.True(FileIndexer.IsWindowsDevicePath(path));
+    }
+
+    [Theory]
+    [InlineData(@"C:\repo\COM10.cs")]
+    [InlineData(@"C:\repo\company.cs")]
+    [InlineData(@"C:\repo\template1.cs")]
+    public void IsWindowsDevicePath_AllowsOrdinaryNames(string path)
+    {
+        Assert.False(FileIndexer.IsWindowsDevicePath(path));
+    }
+
     [Theory]
     [InlineData(FileAttributes.ReparsePoint, false, true)]
     [InlineData(FileAttributes.ReparsePoint, true, true)]
@@ -3343,6 +3365,64 @@ public void BuildRecord_Utf16BeBomFile_DecodedAsUtf16()
         }
     }
 
+    [Fact]
+    public void BuildRecord_Utf16LeWithoutBomFile_DecodedAsUtf16()
+    {
+        // Legacy Windows tools can save source files as UTF-16 LE without a BOM. The
+        // regular every-other-byte NUL pattern should be treated as an encoding signal,
+        // not as binary content. Closes #1829.
+        // 古い Windows ツールは BOM なし UTF-16 LE でソースを保存することがある。
+        // 1 バイトおきの NUL パターンはバイナリ混入ではなくエンコーディングのシグナルとして扱う。
+        // Closes #1829.
+        var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}");
+        try
+        {
+            Directory.CreateDirectory(tempDir);
+            var filePath = Path.Combine(tempDir, "utf16le-nobom.cs");
+            var payload = "using System;\nnamespace Utf16LeNoBom;\n";
+            File.WriteAllBytes(filePath, System.Text.Encoding.Unicode.GetBytes(payload));
+
+            var indexer = new FileIndexer(tempDir);
+            var (_, content, _, warning) = indexer.BuildRecordWithRawBytes(filePath);
+
+            Assert.Null(warning);
+            Assert.Contains("namespace Utf16LeNoBom;", content);
+            Assert.False(FileIndexer.ContainsIndexBlockingNullByte(System.Text.Encoding.Unicode.GetBytes(payload)));
+        }
+        finally
+        {
+            Directory.Delete(tempDir, true);
+        }
+    }
+
+    [Fact]
+    public void BuildRecord_Utf16BeWithoutBomFile_DecodedAsUtf16()
+    {
+        // The heuristic must also handle BOM-less UTF-16 BE text so the fix is not tied
+        // to little-endian Windows output only. Closes #1829.
+        // BOM なし UTF-16 BE テキストも扱い、little-endian Windows 出力だけに限定しない。
+        // Closes #1829.
+        var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}");
+        try
+        {
+            Directory.CreateDirectory(tempDir);
+            var filePath = Path.Combine(tempDir, "utf16be-nobom.cs");
+            var payload = "using System;\nnamespace Utf16BeNoBom;\n";
+            File.WriteAllBytes(filePath, System.Text.Encoding.BigEndianUnicode.GetBytes(payload));
+
+            var indexer = new FileIndexer(tempDir);
+            var (_, content, _, warning) = indexer.BuildRecordWithRawBytes(filePath);
+
+            Assert.Null(warning);
+            Assert.Contains("namespace Utf16BeNoBom;", content);
+            Assert.False(FileIndexer.ContainsIndexBlockingNullByte(System.Text.Encoding.BigEndianUnicode.GetBytes(payload)));
+        }
+        finally
+        {
+            Directory.Delete(tempDir, true);
+        }
+    }
+
     [Fact]
     public void ValidateContent_Utf16LeBomFile_EmitsUtf16BomNotRawByteIssues()
     {
@@ -3370,6 +3450,19 @@ public void ValidateContent_Utf16LeBomFile_EmitsUtf16BomNotRawByteIssues()
         Assert.DoesNotContain(issues, i => i.Kind == "non_utf8_likely");
     }
 
+    [Fact]
+    public void ValidateContent_Utf16WithoutBom_DoesNotEmitNullByteIssue()
+    {
+        var payload = "using System;\nclass C { }\n";
+        var rawBytes = System.Text.Encoding.Unicode.GetBytes(payload);
+
+        var issues = FileIndexer.ValidateContent("utf16le-nobom.cs", rawBytes, payload);
+
+        Assert.DoesNotContain(issues, i => i.Kind == "utf16_bom");
+        Assert.DoesNotContain(issues, i => i.Kind == "null_byte");
+        Assert.DoesNotContain(issues, i => i.Kind == "mixed_line_endings");
+    }
+
     [Fact]
     public void ValidateContent_HighFffdRatio_EmitsAggregateNonUtf8Likely()
     {