Fix issue with decoding truncated files that were compressed with Rust in 0.5.5

Cargo.toml

@@ -1,9 +1,12 @@
 [package]
 name = "lepton_jpeg_root"
-version = "0.5.6"
 edition = "2024"
 authors = ["Kristof Roomp <kristofr@microsoft.com>"]
 
+[workspace.package]
+version = "0.5.8"
+edition = "2024"
+
 [profile.release]
 debug = true
 lto = true

dll/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lepton_jpeg_dll"
-version = "0.5.7"
+version.workspace = true
 edition = "2024"
 authors = ["Kristof Roomp <kristofr@microsoft.com>"]
 

lib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lepton_jpeg"
-version = "0.5.7"
+version.workspace = true
 edition = "2024"
 authors = ["Kristof Roomp <kristofr@microsoft.com>"]
 

lib/src/structs/lepton_file_reader.rs

@@ -295,7 +295,9 @@ impl<'a> LeptonFileReader<'a> {
 
                         // we need to truncate our file to the size minus the garbage data so
                         // that when we hit the garbage data, we have room to write it all out
-                        self.jpeg_file_size_left -= self.lh.rinfo.garbage_data.len() as u64;
+                        if !self.lh.bad_truncation_version() {
+                            self.jpeg_file_size_left -= self.lh.rinfo.garbage_data.len() as u64;
+                        }
 
                         self.state = DecoderState::CMP();
                     }
@@ -345,9 +347,7 @@ impl<'a> LeptonFileReader<'a> {
                         write_tail(&mut self.lh, &mut limited_output)?;
 
                         // here we write out any garbage data verbatim without truncating it
-                        output
-                            .write_all(&mut self.lh.rinfo.garbage_data)
-                            .context()?;
+                        write_garbage_data(&self.lh, limited_output)?;
 
                         self.metrics.merge_from(state.take_metrics());
 
@@ -397,10 +397,8 @@ impl<'a> LeptonFileReader<'a> {
                         }
 
                         write_tail(&mut self.lh, &mut limited_output)?;
-                        // here we write out any garbage data verbatim without truncating it
-                        output
-                            .write_all(&mut self.lh.rinfo.garbage_data)
-                            .context()?;
+
+                        write_garbage_data(&self.lh, limited_output)?;
 
                         self.metrics.merge_from(state.take_metrics());
 
@@ -765,6 +763,26 @@ fn baseline_decoding_thread(
     Ok(())
 }
 
+fn write_garbage_data(
+    lh: &LeptonHeader,
+    mut limited_output: LimitedOutputWriter<'_, impl Write>,
+) -> Result<()> {
+    if !lh.bad_truncation_version() {
+        // here we write out any garbage data verbatim without truncating it
+        // (since we already shrunk the max file size accordingly)
+        limited_output
+            .inner
+            .write_all(&lh.rinfo.garbage_data)
+            .context()?;
+    } else {
+        // the bad encoder wrote the garbage data in such a way that it could
+        // be truncated (see DecoderState::CompressedHeader case above)
+        limited_output.write_all(&lh.rinfo.garbage_data).context()?;
+    }
+
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1025,6 +1043,20 @@ mod tests {
         assert!(output == jpg);
     }
 
+    /// test we can decode an invalid file generated by a regression in the 5.5 version,
+    /// which triggers the bad_truncation_version() check in the LeptonHeader.
+    #[test]
+    fn test_truncated_with_bad_truncation_version() {
+        verifydecode("half_scan_rust55");
+    }
+
+    /// test we can decode the same file as above, but encoded by a
+    /// correctly behaving decoder
+    #[test]
+    fn test_truncated_with_ok_truncation_version() {
+        verifydecode("half_scan");
+    }
+
     /// tests corner case where we have garbage data due to the trunction of the file,
     /// but the garbage data is not actually valid JPEG data. So basically what happened
     /// was that the file got truncated mid-byte and the remaining bits are just random.

lib/src/structs/lepton_header.rs

@@ -46,6 +46,15 @@ pub struct LeptonHeader {
 }
 
 impl LeptonHeader {
+    /// For certain versions of the rust encoder, we didn't handle truncation and corruption correctly.
+    /// The correct behavior is to truncate the JPEG generated data up to the file size minus the garbage data,
+    /// then write out the garbage data.
+    ///
+    /// The incorrect behavior was to write out the JPEG data, append the garbage data, and then truncate.
+    pub fn bad_truncation_version(&self) -> bool {
+        self.encoder_version == 55
+    }
+
     pub fn read_lepton_fixed_header(
         &mut self,
         header: &[u8; FIXED_HEADER_SIZE],

package/Lepton.Jpeg.Rust.nuspec

@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2011/08/nuspec.xsd">
   <metadata>
     <id>Lepton.Jpeg.Rust</id>
-    <version>0.5.7.0</version>
+    <version>0.5.5.8</version>
     <title>Lepton JPEG Compression Rust version binaries and libraries</title>
     <authors>kristofr</authors>
     <owners>kristofr</owners>

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lepton_jpeg_python"
-version = "0.5.7"
+version.workspace = true
 edition = "2024"
 
 [lib]

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "lepton_jpeg_python"
-version = "0.5.7"
+version = "0.5.8"
 description = "Rust port of the Lepton JPEG compression library"
 authors = [{ name = "Kristof Roomp  ", email = "kristofr@gmail.com" }]
 readme = "README.md"

tests/end_to_end.rs

@@ -81,6 +81,8 @@ fn verify_decode(
         "gray2sf",
         "grayscale",
         "hq",
+        "half_scan",
+        "half_scan_rust55",
         "iphone",
         "iphonecity",
         "iphonecity_with_16KGarbage",
@@ -167,6 +169,7 @@ fn verify_encode(
             "gray2sf",
             "grayscale",
             "hq",
+            //"half_scan",
             "iphone",
             "iphonecity",
             "iphonecity_with_16KGarbage",
@@ -213,6 +216,20 @@ fn verify_encode(
     assert_eq_array(&input, &output);
 }
 
+/// these files are expected to fail encoding due to unsupported features or roundtrip errors
+#[rstest]
+fn verify_fail_encode(#[values("half_scan", "narrowrst", "nofsync")] file: &str) {
+    let input = read_file(file, ".jpg");
+
+    let result = encode_lepton_verify(
+        &input,
+        &EnabledFeatures::compat_lepton_vector_write(),
+        &DEFAULT_THREAD_POOL,
+    );
+
+    assert!(result.is_err(), "encoding was expected to fail");
+}
+
 #[test]
 fn verify_16bitmath() {
     // verifies that we can decode 16 bit encoded images from the C++ version

util/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lepton_jpeg_util"
-version = "0.5.7"
+version.workspace = true
 edition = "2024"
 authors = ["Kristof Roomp <kristofr@microsoft.com>"]