diff --git a/changelog.d/unreleased/1830.fixed.md b/changelog.d/unreleased/1830.fixed.md new file mode 100644 index 0000000000..23842abbe5 --- /dev/null +++ b/changelog.d/unreleased/1830.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1830 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs +--- + +## English + +- **Extensionless shebang detection now honors UTF-16 BOM scripts (#1830)** — `cdidx` detects UTF-8, UTF-8 BOM, UTF-16 LE, and UTF-16 BE shebang lines before treating NUL bytes as binary content. + +## 日本語 + +- **拡張子なし shebang 検出が UTF-16 BOM 付きスクリプトに対応しました (#1830)** — `cdidx` は NUL バイトを binary content として扱う前に、UTF-8、UTF-8 BOM、UTF-16 LE、UTF-16 BE の shebang 行を検出します。 diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index aceacb2259..a5585d2b22 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -3236,10 +3236,16 @@ private static LanguageDetectionResult TryDetectLanguageFromShebang(string fileP return new LanguageDetectionResult(FileProbeStatus.Unsupported, null); var bytes = buffer[..bytesRead]; - if (bytes.Contains((byte)0)) + var shebangEncoding = DetectShebangEncoding(bytes); + if (shebangEncoding == ShebangEncoding.Unsupported) return new LanguageDetectionResult(FileProbeStatus.Unsupported, null); - var lineEnd = bytes.IndexOfAny((byte)'\r', (byte)'\n'); + if ((shebangEncoding == ShebangEncoding.Utf8 || shebangEncoding == ShebangEncoding.Utf8Bom) + && bytes.Contains((byte)0)) + return new LanguageDetectionResult(FileProbeStatus.Unsupported, null); + + var preambleLength = GetShebangPreambleLength(shebangEncoding); + var lineEnd = FindShebangLineEnd(bytes, shebangEncoding, preambleLength); if (lineEnd < 0) { if (bytesRead == ShebangProbeByteLimit) @@ -3247,7 +3253,8 @@ private static LanguageDetectionResult TryDetectLanguageFromShebang(string fileP lineEnd = bytesRead; } - var firstLine = new UTF8Encoding(false, throwOnInvalidBytes: true).GetString(bytes[..lineEnd]); + var firstLineBytes = bytes[preambleLength..lineEnd]; + var firstLine = DecodeShebangLine(firstLineBytes, shebangEncoding); if (firstLine.StartsWith('\uFEFF')) firstLine = firstLine[1..]; @@ -3295,6 +3302,70 @@ private static LanguageDetectionResult TryDetectLanguageFromShebang(string fileP } } + private enum ShebangEncoding + { + Utf8, + Utf8Bom, + Utf16LittleEndian, + Utf16BigEndian, + Unsupported, + } + + private static ShebangEncoding DetectShebangEncoding(ReadOnlySpan bytes) + { + if (bytes.Length >= 4) + { + if (bytes[0] == 0x00 && bytes[1] == 0x00 && bytes[2] == 0xFE && bytes[3] == 0xFF) + return ShebangEncoding.Unsupported; + if (bytes[0] == 0xFF && bytes[1] == 0xFE && bytes[2] == 0x00 && bytes[3] == 0x00) + return ShebangEncoding.Unsupported; + } + + if (bytes.Length >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF) + return ShebangEncoding.Utf8Bom; + if (bytes.Length >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE) + return ShebangEncoding.Utf16LittleEndian; + if (bytes.Length >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF) + return ShebangEncoding.Utf16BigEndian; + + return ShebangEncoding.Utf8; + } + + private static int GetShebangPreambleLength(ShebangEncoding encoding) => encoding switch + { + ShebangEncoding.Utf8Bom => 3, + ShebangEncoding.Utf16LittleEndian or ShebangEncoding.Utf16BigEndian => 2, + _ => 0, + }; + + private static int FindShebangLineEnd(ReadOnlySpan bytes, ShebangEncoding encoding, int start) + { + if (encoding is ShebangEncoding.Utf8 or ShebangEncoding.Utf8Bom) + return bytes[start..].IndexOfAny((byte)'\r', (byte)'\n') is var lineEnd && lineEnd >= 0 + ? start + lineEnd + : -1; + + for (var i = start; i + 1 < bytes.Length; i += 2) + { + var ch = encoding == ShebangEncoding.Utf16LittleEndian + ? (bytes[i] | (bytes[i + 1] << 8)) + : ((bytes[i] << 8) | bytes[i + 1]); + if (ch is '\r' or '\n') + return i; + } + + return -1; + } + + private static string DecodeShebangLine(ReadOnlySpan bytes, ShebangEncoding encoding) => encoding switch + { + ShebangEncoding.Utf16LittleEndian => new UnicodeEncoding(bigEndian: false, byteOrderMark: false, throwOnInvalidBytes: true) + .GetString(bytes), + ShebangEncoding.Utf16BigEndian => new UnicodeEncoding(bigEndian: true, byteOrderMark: false, throwOnInvalidBytes: true) + .GetString(bytes), + _ => new UTF8Encoding(false, throwOnInvalidBytes: true).GetString(bytes), + }; + private static string? ResolveShebangInterpreter(IReadOnlyList tokens) { var interpreter = Path.GetFileName(tokens[0]).ToLowerInvariant(); diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index b9bb2b0c34..ef33fda1c7 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -2303,6 +2303,39 @@ public void ScanFilesDetailed_SeparatesUnknownExtensionsFromOtherNonIndexableFil } } + [Fact] + public void DetectLanguage_ExtensionlessShebangs_HonorsUnicodeBomEncodings() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + + var files = new Dictionary + { + ["utf8"] = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), + ["utf8-bom"] = new UTF8Encoding(encoderShouldEmitUTF8Identifier: true), + ["utf16-le"] = new UnicodeEncoding(bigEndian: false, byteOrderMark: true), + ["utf16-be"] = new UnicodeEncoding(bigEndian: true, byteOrderMark: true), + }; + + foreach (var (name, encoding) in files) + { + var path = Path.Combine(tempDir, name); + File.WriteAllText(path, "#!/usr/bin/env bash\nprintf 'ok'\n", encoding); + } + + var detected = files.Keys + .ToDictionary(name => name, name => FileIndexer.DetectLanguage(Path.Combine(tempDir, name))); + + Assert.All(detected, pair => Assert.Equal("shell", pair.Value)); + } + finally + { + Directory.Delete(tempDir, true); + } + } + [Fact] public void ScanFiles_IncludesModernNodeModuleExtensions() {