enable decoding of multi segment baseline images from C++ lepton

lib/src/jpeg/jpeg_header.rs

@@ -546,6 +546,15 @@ impl Default for JpegHeader {
 }
 
 impl JpegHeader {
+    /// true if this image is a single scan, which can be partitioned and decode
+    /// completely independently by separate threads. If this is not the case, then
+    /// we need to decode the entire image in memory and then encode the JPEG sequentially.
+    pub fn is_single_scan(&self) -> bool {
+        assert!(self.jpeg_type != JpegType::Unknown);
+
+        self.jpeg_type == JpegType::Sequential && self.cmpc == self.cs_cmpc
+    }
+
     #[inline(always)]
     pub(super) fn get_huff_dc_codes(&self, cmp: usize) -> &HuffCodes {
         &self.h_codes[0][usize::from(self.cmp_info[cmp].huff_dc)]

lib/src/jpeg/jpeg_read.rs

@@ -97,10 +97,10 @@ pub fn read_jpeg_file<R: BufRead + Seek, FN: FnMut(&JpegHeader, &[u8])>(
 
     on_header_callback(jpeg_header, &rinfo.raw_jpeg_header);
 
-    if !enabled_features.progressive && jpeg_header.jpeg_type == JpegType::Progressive {
+    if !enabled_features.progressive && !jpeg_header.is_single_scan() {
         return err_exit_code(
             ExitCode::ProgressiveUnsupported,
-            "file is progressive, but this is disabled",
+            "file is progressive or contains multiple scans, but this is disabled",
         )
         .context();
     }
@@ -151,7 +151,7 @@ pub fn read_jpeg_file<R: BufRead + Seek, FN: FnMut(&JpegHeader, &[u8])>(
         .context();
     }
 
-    if jpeg_header.jpeg_type == JpegType::Sequential {
+    if jpeg_header.is_single_scan() {
         if rinfo.early_eof_encountered {
             if enabled_features.stop_reading_at_eoi {
                 return err_exit_code(ExitCode::ShortRead, "early EOF encountered");
@@ -200,7 +200,7 @@ pub fn read_jpeg_file<R: BufRead + Seek, FN: FnMut(&JpegHeader, &[u8])>(
             reader.read_to_end(&mut rinfo.garbage_data).context()?;
         }
     } else {
-        assert!(jpeg_header.jpeg_type == JpegType::Progressive);
+        assert!(jpeg_header.jpeg_type != JpegType::Unknown);
 
         if rinfo.early_eof_encountered {
             return err_exit_code(

lib/src/jpeg/jpeg_write.rs

@@ -794,55 +794,49 @@ mod tests {
         let mut reconstructed = Vec::new();
         reconstructed.extend_from_slice(&SOI);
 
-        match jpeg_header.jpeg_type {
-            JpegType::Sequential => {
-                // sequential JPEG consists of a single header + scan
-                reconstructed.extend_from_slice(rinfo.raw_jpeg_header.as_slice());
-
-                let mut prev_offset = 0;
-                for (offset, coding_info) in partitions {
-                    let mut r = jpeg_write_baseline_row_range(
-                        (offset - prev_offset) as usize,
-                        &coding_info,
-                        &image_data,
-                        &jpeg_header,
-                        &rinfo,
-                    )
-                    .unwrap();
-
-                    reconstructed.append(&mut r);
+        if jpeg_header.is_single_scan() {
+            // sequential JPEG consists of a single header + scan
+            reconstructed.extend_from_slice(rinfo.raw_jpeg_header.as_slice());
 
-                    prev_offset = offset;
-                }
+            let mut prev_offset = 0;
+            for (offset, coding_info) in partitions {
+                let mut r = jpeg_write_baseline_row_range(
+                    (offset - prev_offset) as usize,
+                    &coding_info,
+                    &image_data,
+                    &jpeg_header,
+                    &rinfo,
+                )
+                .unwrap();
 
-                assert_eq!(reconstructed.len(), end_scan_position as usize);
+                reconstructed.append(&mut r);
 
-                reconstructed.extend_from_slice(&EOI);
+                prev_offset = offset;
             }
-            JpegType::Progressive => {
-                // progressive JPEG consists of header + scan, header + scan, etc
-                let mut scnc = 0;
 
-                for (jh, raw_header) in headers {
-                    // progressive JPEG consists of headers + scan
-                    reconstructed.extend_from_slice(&raw_header);
+            assert_eq!(reconstructed.len(), end_scan_position as usize);
 
-                    let scan = jpeg_write_entire_scan(&image_data, &jh, &rinfo, scnc).unwrap();
+            reconstructed.extend_from_slice(&EOI);
+        } else {
+            // progressive JPEG consists of header + scan, header + scan, etc
+            let mut scnc = 0;
 
-                    reconstructed.extend_from_slice(&scan);
+            for (jh, raw_header) in headers {
+                // progressive JPEG consists of headers + scan
+                reconstructed.extend_from_slice(&raw_header);
 
-                    // advance to next scan
-                    scnc += 1;
-                }
+                let scan = jpeg_write_entire_scan(&image_data, &jh, &rinfo, scnc).unwrap();
 
-                reconstructed.extend_from_slice(&EOI);
+                reconstructed.extend_from_slice(&scan);
 
-                // progressive includes EOI in the scan
-                assert_eq!(reconstructed.len(), end_scan_position as usize);
-            }
-            _ => {
-                panic!("unexpected JPEG type: {:?}", jpeg_header.jpeg_type);
+                // advance to next scan
+                scnc += 1;
             }
+
+            reconstructed.extend_from_slice(&EOI);
+
+            // progressive includes EOI in the scan
+            assert_eq!(reconstructed.len(), end_scan_position as usize);
         }
 
         reconstructed

lib/src/structs/lepton_file_reader.rs

@@ -531,7 +531,12 @@ impl<'a> LeptonFileReader<'a> {
         if v[..] != LEPTON_HEADER_COMPLETION_MARKER {
             return err_exit_code(ExitCode::BadLeptonFile, "CMP marker not found");
         }
-        Ok(if lh.jpeg_header.jpeg_type == JpegType::Progressive {
+
+        // use progressive logic, which reads the entire block into memory and then performs
+        // the jpeg decoding. This permits multiple scans that are each encoded in two cases:
+        //  - progressive images
+        //  - baseline multiscan images (rare but permitted)
+        Ok(if !lh.jpeg_header.is_single_scan() {
             let mux = Self::run_lepton_decoder_threads(
                 lh,
                 enabled_features,
@@ -1001,12 +1006,8 @@ mod tests {
         assert!(r.is_err() && r.err().unwrap().exit_code() == ExitCode::OsError);
     }
 
-    /// tests corner case where we have garbage data due to the trunction of the file,
-    /// but the garbage data is not actually valid JPEG data. So basically what happened
-    /// was that the file got truncated mid-byte and the remaining bits are just random.
-    #[test]
-    fn test_truncated_with_bad_garbage_data() {
-        let original = read_file("truncbad", ".lep");
+    fn verifydecode(filename: &str) {
+        let original = read_file(filename, ".lep");
 
         let mut output = Vec::new();
 
@@ -1018,9 +1019,17 @@ mod tests {
         )
         .unwrap();
 
-        let jpg = read_file("truncbad", ".jpg");
+        let jpg = read_file(filename, ".jpg");
 
         assert_eq!(jpg.len(), output.len());
         assert!(output == jpg);
     }
+
+    /// tests corner case where we have garbage data due to the trunction of the file,
+    /// but the garbage data is not actually valid JPEG data. So basically what happened
+    /// was that the file got truncated mid-byte and the remaining bits are just random.
+    #[test]
+    fn test_truncated_with_bad_garbage_data() {
+        verifydecode("truncbad");
+    }
 }

tests/end_to_end.rs

@@ -15,6 +15,40 @@ use lepton_jpeg::{
 use lepton_jpeg::{ExitCode, LeptonError};
 use rstest::rstest;
 
+/// handy function to compare two arrays, and print the first mismatch. Useful for debugging.
+#[track_caller]
+pub fn assert_eq_array<T: PartialEq + std::fmt::Debug>(a: &[T], b: &[T]) {
+    use core::panic;
+
+    if a.len() != b.len() {
+        for i in 0..std::cmp::min(a.len(), b.len()) {
+            assert_eq!(
+                a[i],
+                b[i],
+                "length mismatch {},{} and first mismatch at offset {}",
+                a.len(),
+                b.len(),
+                i
+            );
+        }
+        panic!(
+            "length mismatch {} and {}, but common prefix identical",
+            a.len(),
+            b.len()
+        );
+    } else {
+        for i in 0..a.len() {
+            assert_eq!(
+                a[i],
+                b[i],
+                "length identical {}, but first mismatch at offset {}",
+                a.len(),
+                i
+            );
+        }
+    }
+}
+
 /// reads a file from the images directory for testing or benchmarking purposes
 pub fn read_file(filename: &str, ext: &str) -> Vec<u8> {
     use std::io::Read;
@@ -42,6 +76,8 @@ fn verify_decode(
         "androidprogressive_garbage",
         "androidtrail",
         "colorswap",
+        "cathedral_db_non_int",
+        "cathedral_db_non_int_rustold",
         "gray2sf",
         "grayscale",
         "hq",
@@ -87,14 +123,7 @@ fn verify_decode(
     )
     .unwrap();
 
-    assert_eq!(
-        output.len(),
-        expected.len(),
-        "length mismatch {} {}",
-        output.len(),
-        expected.len()
-    );
-    assert!(output[..] == expected[..]);
+    assert_eq_array(&output, &expected);
 }
 
 /// verifies that the decode will accept existing Lepton files and generate
@@ -120,7 +149,7 @@ fn verify_decode_scalar_overflow() {
     )
     .unwrap();
 
-    assert!(output[..] == expected[..]);
+    assert_eq_array(&output, &expected);
 }
 
 /// encodes as LEP and codes back to JPG to mostly test the encoder. Can't check against
@@ -181,7 +210,7 @@ fn verify_encode(
     )
     .unwrap();
 
-    assert!(input[..] == output[..]);
+    assert_eq_array(&input, &output);
 }
 
 #[test]
@@ -203,7 +232,7 @@ fn verify_16bitmath() {
         )
         .unwrap();
 
-        assert!(output[..] == expected[..]);
+        assert_eq_array(&output, &expected);
     }
 
     // verify that we can decode the one generated by the Rust version
@@ -224,7 +253,7 @@ fn verify_16bitmath() {
         )
         .unwrap();
 
-        assert!(output[..] == expected[..]);
+        assert_eq_array(&output, &expected);
     }
 }