fix(psd): harden reader against oversized/bomb headers and ZIP-layer overflow

src/psd.imageio/psdinput.cpp

@@ -451,7 +451,7 @@ class PSDInput final : public ImageInput {
                              uint32_t unpacked_length);
     bool decompress_zip(span<char> src, span<char> dest);
     bool decompress_zip_prediction(span<char> src, span<char> dest,
-                                   const uint32_t width, const uint32_t height);
+                                   const uint64_t width, const uint64_t height);
 
     // These are AdditionalInfo entries that, for PSBs, have an 8-byte length
     static const char* additional_info_psb[];
@@ -590,6 +590,23 @@ PSDInput::open(const std::string& name, ImageSpec& newspec)
         return false;
     }
 
+    // Set m_type_desc to the appropriate TypeDesc (depends only on the depth,
+    // which load_header has just validated).
+    set_type_desc();
+
+    // Do the resolution/etc sanity checks here, the earliest point at which
+    // the composite resolution, channel count, and data type are known --
+    // before any section allocates or seeks based on them.
+    {
+        ImageSpec headerspec((int)m_header.width, (int)m_header.height,
+                             (int)m_header.channel_count, m_type_desc);
+        if (!check_open(headerspec, { 0, 300000, 0, 300000, 0, 1, 0, 56 })
+            || !check_compression_ratio(headerspec, ioproxy()->size())) {
+            close();
+            return false;
+        }
+    }
+
     // Color Mode Data
     if (!load_color_data()) {
         errorfmt("failed to open \"{}\": failed load_color_data", name);
@@ -628,11 +645,8 @@ PSDInput::open(const std::string& name, ImageSpec& newspec)
 
     // Layer count + 1 for merged composite (Image Data Section)
     m_subimage_count = m_layers.size() + 1;
-    // Set m_type_desc to the appropriate TypeDesc
-    set_type_desc();
     // Setup ImageSpecs and m_channels
-    bool ok = true;
-    ok &= setup();
+    bool ok = setup();
 
     if (ok)
         ok &= seek_subimage(0, 0);
@@ -1783,7 +1797,7 @@ PSDInput::load_layer_channel(Layer& layer, ChannelInfo& channel_info)
         // randomly so we parse the data up-front and store it
         std::vector<char> compressed_data(channel_info.data_length);
         channel_info.decompressed_data = std::vector<char>(
-            width * height * (m_header.depth / 8));
+            size_t(width) * size_t(height) * (m_header.depth / 8));
 
         if (!ioseek(channel_info.data_pos))
             return false;
@@ -1800,7 +1814,7 @@ PSDInput::load_layer_channel(Layer& layer, ChannelInfo& channel_info)
         // randomly so we parse the data up-front and store it
         std::vector<char> compressed_data(channel_info.data_length);
         channel_info.decompressed_data = std::vector<char>(
-            width * height * (m_header.depth / 8));
+            size_t(width) * size_t(height) * (m_header.depth / 8));
 
         if (!ioseek(channel_info.data_pos))
             return false;
@@ -2381,8 +2395,8 @@ bool
 PSDInput::decompress_packbits(const char* src, char* dst,
                               uint32_t packed_length, uint32_t unpacked_length)
 {
-    int32_t src_remaining = packed_length;
-    int32_t dst_remaining = unpacked_length;
+    int64_t src_remaining = packed_length;
+    int64_t dst_remaining = unpacked_length;
     int16_t header;
     int length;
 
@@ -2482,7 +2496,7 @@ PSDInput::decompress_zip(span<char> src, span<char> dest)
 
 bool
 PSDInput::decompress_zip_prediction(span<char> src, span<char> dest,
-                                    const uint32_t width, const uint32_t height)
+                                    const uint64_t width, const uint64_t height)
 {
     OIIO_ASSERT(width * height * (m_header.depth / 8) == dest.size());
     bool ok = true;

testsuite/psd/ref/out-linuxarm.txt

@@ -2123,3 +2123,6 @@ Full command line was:
 > oiiotool --info -v -a --hash src/crash-rowbounds-f999.psd
 src/crash-rowbounds-f999.psd :   10 x  111, 3 channel, uint8 psd
     channel list: R, G, B
+oiiotool ERROR: read : "src/crash-bomb-30000.psd": psd header claims a 2574 MB image from a 40 byte file; probably a corrupt or malicious header
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-bomb-30000.psd

testsuite/psd/ref/out.txt

@@ -480,7 +480,7 @@ Reading ../oiio-images/psd/psd_bitmap.psd
     stRef:originalDocumentID: "E146B3E37A92795EE3EA6577040DE6D5"
 Reading ../oiio-images/psd/psd_indexed_trans.psd
 ../oiio-images/psd/psd_indexed_trans.psd :  320 x  240, 4 channel, uint8 psd
-    SHA-1: 5B548EAE1F69EC4B65D762F52D8C542CF18515D5
+    SHA-1: 3DEA3F342B61B6B2064310C06B2AE23692079EE7
     channel list: R, G, B, A
     Artist: "Daniel Wyatt"
     DateTime: "2007-01-18T15:49:21"
@@ -2122,4 +2122,7 @@ oiiotool ERROR: -info : SHA-1: Corrupt PSD channel data for row 0
 Full command line was:
 > oiiotool --info -v -a --hash src/crash-rowbounds-f999.psd
 src/crash-rowbounds-f999.psd :   10 x  111, 3 channel, uint8 psd
-   channel list: R, G, B
+    channel list: R, G, B
+oiiotool ERROR: read : "src/crash-bomb-30000.psd": psd header claims a 2574 MB image from a 40 byte file; probably a corrupt or malicious header
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-bomb-30000.psd

testsuite/psd/run.py

@@ -40,3 +40,5 @@
 command += info_command ("src/crash-eofstring.psd", failureok=True)
 # Corruption where short per-channel rows could overrun interleaving
 command += info_command ("src/crash-rowbounds-f999.psd", failureok=True)
+# Tiny file declaring a huge composite (decompression bomb) is rejected
+command += info_command ("src/crash-bomb-30000.psd", failureok=True)