Detect UTF-16 shebang BOMs

changelog.d/unreleased/1830.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+issues:
+  - 1830
+affected:
+  - src/CodeIndex/Indexer/Scanning/FileIndexer.cs
+  - tests/CodeIndex.Tests/FileIndexerTests.cs
+---
+
+## English
+
+- **Extensionless shebang detection now honors UTF-16 BOM scripts (#1830)** — `cdidx` detects UTF-8, UTF-8 BOM, UTF-16 LE, and UTF-16 BE shebang lines before treating NUL bytes as binary content.
+
+## 日本語
+
+- **拡張子なし shebang 検出が UTF-16 BOM 付きスクリプトに対応しました (#1830)** — `cdidx` は NUL バイトを binary content として扱う前に、UTF-8、UTF-8 BOM、UTF-16 LE、UTF-16 BE の shebang 行を検出します。

src/CodeIndex/Indexer/Scanning/FileIndexer.cs

@@ -3236,18 +3236,25 @@ private static LanguageDetectionResult TryDetectLanguageFromShebang(string fileP
                 return new LanguageDetectionResult(FileProbeStatus.Unsupported, null);
 
             var bytes = buffer[..bytesRead];
-            if (bytes.Contains((byte)0))
+            var shebangEncoding = DetectShebangEncoding(bytes);
+            if (shebangEncoding == ShebangEncoding.Unsupported)
                 return new LanguageDetectionResult(FileProbeStatus.Unsupported, null);
 
-            var lineEnd = bytes.IndexOfAny((byte)'\r', (byte)'\n');
+            if ((shebangEncoding == ShebangEncoding.Utf8 || shebangEncoding == ShebangEncoding.Utf8Bom)
+                && bytes.Contains((byte)0))
+                return new LanguageDetectionResult(FileProbeStatus.Unsupported, null);
+
+            var preambleLength = GetShebangPreambleLength(shebangEncoding);
+            var lineEnd = FindShebangLineEnd(bytes, shebangEncoding, preambleLength);
             if (lineEnd < 0)
             {
                 if (bytesRead == ShebangProbeByteLimit)
                     return new LanguageDetectionResult(FileProbeStatus.Unsupported, null);
                 lineEnd = bytesRead;
             }
 
-            var firstLine = new UTF8Encoding(false, throwOnInvalidBytes: true).GetString(bytes[..lineEnd]);
+            var firstLineBytes = bytes[preambleLength..lineEnd];
+            var firstLine = DecodeShebangLine(firstLineBytes, shebangEncoding);
 
             if (firstLine.StartsWith('\uFEFF'))
                 firstLine = firstLine[1..];
@@ -3295,6 +3302,70 @@ private static LanguageDetectionResult TryDetectLanguageFromShebang(string fileP
         }
     }
 
+    private enum ShebangEncoding
+    {
+        Utf8,
+        Utf8Bom,
+        Utf16LittleEndian,
+        Utf16BigEndian,
+        Unsupported,
+    }
+
+    private static ShebangEncoding DetectShebangEncoding(ReadOnlySpan<byte> bytes)
+    {
+        if (bytes.Length >= 4)
+        {
+            if (bytes[0] == 0x00 && bytes[1] == 0x00 && bytes[2] == 0xFE && bytes[3] == 0xFF)
+                return ShebangEncoding.Unsupported;
+            if (bytes[0] == 0xFF && bytes[1] == 0xFE && bytes[2] == 0x00 && bytes[3] == 0x00)
+                return ShebangEncoding.Unsupported;
+        }
+
+        if (bytes.Length >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF)
+            return ShebangEncoding.Utf8Bom;
+        if (bytes.Length >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE)
+            return ShebangEncoding.Utf16LittleEndian;
+        if (bytes.Length >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF)
+            return ShebangEncoding.Utf16BigEndian;
+
+        return ShebangEncoding.Utf8;
+    }
+
+    private static int GetShebangPreambleLength(ShebangEncoding encoding) => encoding switch
+    {
+        ShebangEncoding.Utf8Bom => 3,
+        ShebangEncoding.Utf16LittleEndian or ShebangEncoding.Utf16BigEndian => 2,
+        _ => 0,
+    };
+
+    private static int FindShebangLineEnd(ReadOnlySpan<byte> bytes, ShebangEncoding encoding, int start)
+    {
+        if (encoding is ShebangEncoding.Utf8 or ShebangEncoding.Utf8Bom)
+            return bytes[start..].IndexOfAny((byte)'\r', (byte)'\n') is var lineEnd && lineEnd >= 0
+                ? start + lineEnd
+                : -1;
+
+        for (var i = start; i + 1 < bytes.Length; i += 2)
+        {
+            var ch = encoding == ShebangEncoding.Utf16LittleEndian
+                ? (bytes[i] | (bytes[i + 1] << 8))
+                : ((bytes[i] << 8) | bytes[i + 1]);
+            if (ch is '\r' or '\n')
+                return i;
+        }
+
+        return -1;
+    }
+
+    private static string DecodeShebangLine(ReadOnlySpan<byte> bytes, ShebangEncoding encoding) => encoding switch
+    {
+        ShebangEncoding.Utf16LittleEndian => new UnicodeEncoding(bigEndian: false, byteOrderMark: false, throwOnInvalidBytes: true)
+            .GetString(bytes),
+        ShebangEncoding.Utf16BigEndian => new UnicodeEncoding(bigEndian: true, byteOrderMark: false, throwOnInvalidBytes: true)
+            .GetString(bytes),
+        _ => new UTF8Encoding(false, throwOnInvalidBytes: true).GetString(bytes),
+    };
+
     private static string? ResolveShebangInterpreter(IReadOnlyList<string> tokens)
     {
         var interpreter = Path.GetFileName(tokens[0]).ToLowerInvariant();

tests/CodeIndex.Tests/FileIndexerTests.cs

@@ -2303,6 +2303,39 @@ public void ScanFilesDetailed_SeparatesUnknownExtensionsFromOtherNonIndexableFil
         }
     }
 
+    [Fact]
+    public void DetectLanguage_ExtensionlessShebangs_HonorsUnicodeBomEncodings()
+    {
+        var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}");
+        try
+        {
+            Directory.CreateDirectory(tempDir);
+
+            var files = new Dictionary<string, Encoding>
+            {
+                ["utf8"] = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
+                ["utf8-bom"] = new UTF8Encoding(encoderShouldEmitUTF8Identifier: true),
+                ["utf16-le"] = new UnicodeEncoding(bigEndian: false, byteOrderMark: true),
+                ["utf16-be"] = new UnicodeEncoding(bigEndian: true, byteOrderMark: true),
+            };
+
+            foreach (var (name, encoding) in files)
+            {
+                var path = Path.Combine(tempDir, name);
+                File.WriteAllText(path, "#!/usr/bin/env bash\nprintf 'ok'\n", encoding);
+            }
+
+            var detected = files.Keys
+                .ToDictionary(name => name, name => FileIndexer.DetectLanguage(Path.Combine(tempDir, name)));
+
+            Assert.All(detected, pair => Assert.Equal("shell", pair.Value));
+        }
+        finally
+        {
+            Directory.Delete(tempDir, true);
+        }
+    }
+
     [Fact]
     public void ScanFiles_IncludesModernNodeModuleExtensions()
     {