Handle UTF-16 checksum files

Author: mamoreau-devolutions

crates/multi-pwsh/src/install.rs

@@ -96,7 +96,10 @@ fn download_text_with_retry(http: &Agent, url: &str, retries: usize) -> Result<S
     for attempt in 1..=retries {
         let result = (|| -> Result<String> {
             let response = http.get(url).set("User-Agent", "multi-pwsh").call()?;
-            Ok(response.into_string()?)
+            let mut reader = response.into_reader();
+            let mut bytes = Vec::new();
+            reader.read_to_end(&mut bytes)?;
+            decode_checksum_text(&bytes)
         })();
 
         match result {
@@ -114,6 +117,85 @@ fn download_text_with_retry(http: &Agent, url: &str, retries: usize) -> Result<S
     Err(last_error.unwrap_or_else(|| MultiPwshError::Archive("download failed without detailed error".to_string())))
 }
 
+fn decode_checksum_text(bytes: &[u8]) -> Result<String> {
+    match detect_text_encoding(bytes) {
+        TextEncoding::Utf8 => String::from_utf8(bytes.to_vec()).map_err(invalid_checksum_encoding),
+        TextEncoding::Utf8Bom => String::from_utf8(bytes[3..].to_vec()).map_err(invalid_checksum_encoding),
+        TextEncoding::Utf16Le => decode_utf16_text(&bytes[2..], true),
+        TextEncoding::Utf16Be => decode_utf16_text(&bytes[2..], false),
+        TextEncoding::Utf16LeNoBom => decode_utf16_text(bytes, true),
+        TextEncoding::Utf16BeNoBom => decode_utf16_text(bytes, false),
+    }
+}
+
+fn decode_utf16_text(bytes: &[u8], little_endian: bool) -> Result<String> {
+    if bytes.len() % 2 != 0 {
+        return Err(MultiPwshError::Archive(
+            "invalid checksum file encoding: odd-length utf-16 payload".to_string(),
+        ));
+    }
+
+    let units: Vec<u16> = bytes
+        .chunks_exact(2)
+        .map(|chunk| {
+            if little_endian {
+                u16::from_le_bytes([chunk[0], chunk[1]])
+            } else {
+                u16::from_be_bytes([chunk[0], chunk[1]])
+            }
+        })
+        .collect();
+
+    String::from_utf16(&units)
+        .map_err(|error| MultiPwshError::Archive(format!("invalid checksum file encoding: {}", error)))
+}
+
+fn invalid_checksum_encoding(error: std::string::FromUtf8Error) -> MultiPwshError {
+    MultiPwshError::Archive(format!("invalid checksum file encoding: {}", error))
+}
+
+fn detect_text_encoding(bytes: &[u8]) -> TextEncoding {
+    if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
+        return TextEncoding::Utf8Bom;
+    }
+
+    if bytes.starts_with(&[0xFF, 0xFE]) {
+        return TextEncoding::Utf16Le;
+    }
+
+    if bytes.starts_with(&[0xFE, 0xFF]) {
+        return TextEncoding::Utf16Be;
+    }
+
+    if looks_like_utf16_le(bytes) {
+        return TextEncoding::Utf16LeNoBom;
+    }
+
+    if looks_like_utf16_be(bytes) {
+        return TextEncoding::Utf16BeNoBom;
+    }
+
+    TextEncoding::Utf8
+}
+
+fn looks_like_utf16_le(bytes: &[u8]) -> bool {
+    bytes.len() >= 4 && bytes[1] == 0 && bytes[3] == 0
+}
+
+fn looks_like_utf16_be(bytes: &[u8]) -> bool {
+    bytes.len() >= 4 && bytes[0] == 0 && bytes[2] == 0
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum TextEncoding {
+    Utf8,
+    Utf8Bom,
+    Utf16Le,
+    Utf16Be,
+    Utf16LeNoBom,
+    Utf16BeNoBom,
+}
+
 fn validate_archive_checksum(http: &Agent, release: &ResolvedRelease, archive_path: &Path) -> Result<()> {
     let checksums = download_text_with_retry(http, &release.checksum_asset_url, 8)?;
     let expected = find_expected_checksum(&checksums, &release.asset_name)?;
@@ -132,33 +214,10 @@ fn validate_archive_checksum(http: &Agent, release: &ResolvedRelease, archive_pa
 
 fn find_expected_checksum(checksums: &str, asset_name: &str) -> Result<String> {
     for (index, line) in checksums.lines().enumerate() {
-        let trimmed = line.trim();
-        if trimmed.is_empty() {
-            continue;
-        }
-
-        let checksum = trimmed
-            .split_ascii_whitespace()
-            .next()
-            .ok_or_else(|| MultiPwshError::Archive(format!("malformed checksum line {}", index + 1)))?;
-
-        if !is_valid_sha256_hex(checksum) {
-            return Err(MultiPwshError::Archive(format!(
-                "invalid sha256 checksum on line {}",
-                index + 1
-            )));
-        }
-
-        let file_name = trimmed[checksum.len()..].trim_start().trim_start_matches('*');
-        if file_name.is_empty() {
-            return Err(MultiPwshError::Archive(format!(
-                "missing file name in checksum line {}",
-                index + 1
-            )));
-        }
-
-        if file_name == asset_name {
-            return Ok(checksum.to_ascii_lowercase());
+        if let Some((checksum, file_name)) = parse_checksum_line(line, index + 1, asset_name)? {
+            if file_name == asset_name {
+                return Ok(checksum);
+            }
         }
     }
 
@@ -172,6 +231,81 @@ fn is_valid_sha256_hex(value: &str) -> bool {
     value.len() == 64 && value.as_bytes().iter().all(|byte| byte.is_ascii_hexdigit())
 }
 
+fn parse_checksum_line<'a>(
+    line: &'a str,
+    line_number: usize,
+    target_asset_name: &str,
+) -> Result<Option<(String, &'a str)>> {
+    let trimmed = line.trim().trim_start_matches('\u{feff}');
+    if trimmed.is_empty() || trimmed.starts_with('#') {
+        return Ok(None);
+    }
+
+    if let Some(parsed) = parse_bsd_checksum_line(trimmed, line_number)? {
+        return Ok(Some(parsed));
+    }
+
+    parse_gnu_checksum_line(trimmed, line_number, target_asset_name)
+}
+
+fn parse_bsd_checksum_line<'a>(line: &'a str, line_number: usize) -> Result<Option<(String, &'a str)>> {
+    let Some((left, right)) = line.split_once('=') else {
+        return Ok(None);
+    };
+
+    let Some(file_name) = left
+        .trim()
+        .strip_prefix("SHA256 (")
+        .and_then(|value| value.strip_suffix(')'))
+    else {
+        return Ok(None);
+    };
+
+    let checksum = right.trim();
+    if !is_valid_sha256_hex(checksum) {
+        return Err(MultiPwshError::Archive(format!(
+            "invalid sha256 checksum on line {}",
+            line_number
+        )));
+    }
+
+    Ok(Some((checksum.to_ascii_lowercase(), file_name)))
+}
+
+fn parse_gnu_checksum_line<'a>(
+    line: &'a str,
+    line_number: usize,
+    target_asset_name: &str,
+) -> Result<Option<(String, &'a str)>> {
+    let mut parts = line.split_ascii_whitespace();
+    let Some(checksum) = parts.next() else {
+        return Ok(None);
+    };
+
+    let remainder = line[checksum.len()..].trim_start();
+    if remainder.is_empty() {
+        return Ok(None);
+    }
+
+    let file_name = remainder.trim_start_matches('*').trim();
+    if file_name.is_empty() {
+        return Ok(None);
+    }
+
+    if !is_valid_sha256_hex(checksum) {
+        if file_name == target_asset_name {
+            return Err(MultiPwshError::Archive(format!(
+                "invalid sha256 checksum on line {}",
+                line_number
+            )));
+        }
+
+        return Ok(None);
+    }
+
+    Ok(Some((checksum.to_ascii_lowercase(), file_name)))
+}
+
 fn sha256_file(path: &Path) -> Result<String> {
     let mut file = File::open(path)?;
     let mut hasher = Sha256::new();
@@ -360,6 +494,35 @@ bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb *other.zip",
         );
     }
 
+    #[test]
+    fn find_expected_checksum_accepts_bsd_format() {
+        let checksum = find_expected_checksum(
+            "SHA256 (PowerShell-7.4.13-win-x64.zip) = aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+            "PowerShell-7.4.13-win-x64.zip",
+        )
+        .unwrap();
+
+        assert_eq!(
+            checksum,
+            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+        );
+    }
+
+    #[test]
+    fn find_expected_checksum_ignores_unrelated_non_checksum_lines() {
+        let checksum = find_expected_checksum(
+            "Checksums for release assets\n\
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa  PowerShell-7.4.13-win-x64.zip",
+            "PowerShell-7.4.13-win-x64.zip",
+        )
+        .unwrap();
+
+        assert_eq!(
+            checksum,
+            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+        );
+    }
+
     #[test]
     fn find_expected_checksum_rejects_invalid_lines() {
         let error = find_expected_checksum(
@@ -384,6 +547,39 @@ bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb *other.zip",
             .contains("checksum entry for 'PowerShell-7.4.13-win-x64.zip' not found"));
     }
 
+    #[test]
+    fn decode_checksum_text_accepts_utf16le_bom() {
+        let content =
+            "73601859461b130ee1e6624f0683000a794cbe86db0f4ff9f2ce2a7d4f5f6a01 *powershell-7.4.13-1.cm.aarch64.rpm\n";
+        let mut bytes = vec![0xFF, 0xFE];
+        for unit in content.encode_utf16() {
+            bytes.extend_from_slice(&unit.to_le_bytes());
+        }
+
+        let decoded = decode_checksum_text(&bytes).unwrap();
+
+        assert_eq!(decoded, content);
+    }
+
+    #[test]
+    fn find_expected_checksum_accepts_real_powershell_utf16le_line() {
+        let content =
+            "0aa943342ddd5ff5cd5bbb964e6594b7af3e10758ff59874cd26420bebb3c755 *PowerShell-7.4.13-win-arm64.exe\n\
+1820febe6f9567c8bab21be601dacb902777c1185e1beb81843c3a6f902d6b9d *PowerShell-7.4.13-win-arm64.zip\n";
+        let mut bytes = vec![0xFF, 0xFE];
+        for unit in content.encode_utf16() {
+            bytes.extend_from_slice(&unit.to_le_bytes());
+        }
+
+        let decoded = decode_checksum_text(&bytes).unwrap();
+        let checksum = find_expected_checksum(&decoded, "PowerShell-7.4.13-win-arm64.zip").unwrap();
+
+        assert_eq!(
+            checksum,
+            "1820febe6f9567c8bab21be601dacb902777c1185e1beb81843c3a6f902d6b9d"
+        );
+    }
+
     #[test]
     fn sha256_file_hashes_file_contents() {
         let temp_dir = tempfile::tempdir().unwrap();