perf(decode): cut per-frame fixed overhead on the in-memory path

zstd/benches/compare_ffi.rs

@@ -753,21 +753,10 @@ fn bench_dictionary(c: &mut Criterion) {
             configure_group(&mut group, scenario, BenchOp::Compress);
             group.throughput(Throughput::Bytes(scenario.throughput_bytes()));
 
-            // Construct the FFI compressor INSIDE `b.iter` to match the
-            // per-iter setup shape used by both *_with_dict arms below.
-            // Reusing one compressor outside the loop here biased the
-            // baseline: `c_ffi_with_dict` paid CCtx-create + CDict attach
-            // every iter, but `c_ffi_without_dict` paid only the compress
-            // call, making the dict-vs-nodict delta look larger than the
-            // real attach cost.
-            group.bench_function("c_ffi_without_dict", |b| {
-                b.iter(|| {
-                    let mut compressor = zstd::bulk::Compressor::new(level.ffi_level).unwrap();
-                    configure_ffi_bulk_compressor(&mut compressor, &level);
-                    black_box(compressor.compress(&scenario.bytes).unwrap())
-                })
-            });
-
+            // No-dict baseline for this level/scenario is already measured by the
+            // plain `compress/{level}/{scenario}/matrix/c_ffi` group, so this
+            // dict group only times the two dictionary arms.
+            //
             // c_ffi_with_dict + pure_rust_with_dict: STEADY-STATE measurement.
             // Both build the compressor + attach the dictionary ONCE before
             // `b.iter`, then compress in the loop reusing that context — the

zstd/src/decoding/frame.rs

@@ -106,6 +106,94 @@ pub fn read_frame_header_with_format(
     Ok((frame_header, bytes_read as u8))
 }
 
+/// Slice-direct equivalent of [`read_frame_header_with_format`]: parses the
+/// header straight out of `input` with byte indexing + `from_le_bytes`, instead
+/// of per-field `read_exact` calls through the `Read` trait, and advances
+/// `*input` past the consumed bytes EXACTLY as the `Read` version does — so the
+/// skippable-frame and truncation contracts are byte-identical. The decode path
+/// already holds a `&[u8]`, so this avoids the `io::impls` `Read`-trait
+/// dispatch the generic version pays per field, mirroring upstream zstd, which
+/// parses the header from a raw pointer (`MEM_readLE32` / byte reads).
+pub(crate) fn read_frame_header_from_slice(
+    input: &mut &[u8],
+    magicless: bool,
+) -> Result<(FrameHeader, u8), ReadFrameHeaderError> {
+    use ReadFrameHeaderError as err;
+    fn eof() -> crate::io::Error {
+        crate::io::Error::from(crate::io::ErrorKind::UnexpectedEof)
+    }
+    fn take<'a>(input: &mut &'a [u8], n: usize) -> Option<&'a [u8]> {
+        if input.len() < n {
+            return None;
+        }
+        let (head, tail) = input.split_at(n);
+        *input = tail;
+        Some(head)
+    }
+
+    let mut bytes_read: u8 = 0;
+    if !magicless {
+        let m = take(input, 4).ok_or_else(|| err::MagicNumberReadError(eof()))?;
+        let magic_num = u32::from_le_bytes([m[0], m[1], m[2], m[3]]);
+        bytes_read = 4;
+        if (0x184D2A50..=0x184D2A5F).contains(&magic_num) {
+            let s = take(input, 4).ok_or_else(|| err::FrameDescriptorReadError(eof()))?;
+            let skip_size = u32::from_le_bytes([s[0], s[1], s[2], s[3]]);
+            return Err(ReadFrameHeaderError::SkipFrame {
+                magic_number: magic_num,
+                length: skip_size,
+            });
+        }
+        if magic_num != MAGIC_NUM {
+            return Err(ReadFrameHeaderError::BadMagicNumber(magic_num));
+        }
+    }
+
+    let d = take(input, 1).ok_or_else(|| err::FrameDescriptorReadError(eof()))?;
+    let desc = FrameDescriptor(d[0]);
+    bytes_read += 1;
+
+    let mut frame_header = FrameHeader {
+        descriptor: FrameDescriptor(desc.0),
+        dict_id: None,
+        frame_content_size: 0,
+        window_descriptor: 0,
+    };
+
+    if !desc.single_segment_flag() {
+        let w = take(input, 1).ok_or_else(|| err::WindowDescriptorReadError(eof()))?;
+        frame_header.window_descriptor = w[0];
+        bytes_read += 1;
+    }
+
+    let dict_id_len = desc.dictionary_id_bytes()? as usize;
+    if dict_id_len != 0 {
+        let b = take(input, dict_id_len).ok_or_else(|| err::DictionaryIdReadError(eof()))?;
+        bytes_read += dict_id_len as u8;
+        let mut buf4 = [0u8; 4];
+        buf4[..dict_id_len].copy_from_slice(b);
+        let dict_id = u32::from_le_bytes(buf4);
+        if dict_id != 0 {
+            frame_header.dict_id = Some(dict_id);
+        }
+    }
+
+    let fcs_len = desc.frame_content_size_bytes()? as usize;
+    if fcs_len != 0 {
+        let b = take(input, fcs_len).ok_or_else(|| err::FrameContentSizeReadError(eof()))?;
+        bytes_read += fcs_len as u8;
+        let mut buf8 = [0u8; 8];
+        buf8[..fcs_len].copy_from_slice(b);
+        let mut fcs = u64::from_le_bytes(buf8);
+        if fcs_len == 2 {
+            fcs += 256;
+        }
+        frame_header.frame_content_size = fcs;
+    }
+
+    Ok((frame_header, bytes_read))
+}
+
 /// A frame header has a variable size, with a minimum of 2 bytes, and a maximum of 14 bytes.
 pub struct FrameHeader {
     pub descriptor: FrameDescriptor,

zstd/src/decoding/frame_decoder.rs

@@ -860,6 +860,16 @@ impl FrameDecoderState {
         magicless: bool,
     ) -> Result<FrameDecoderState, FrameDecoderError> {
         let (frame, header_size) = frame::read_frame_header_with_format(source, magicless)?;
+        Self::new_with_parsed_header(frame, header_size)
+    }
+
+    /// Build a fresh state from an already-parsed frame header (the non-parsing
+    /// tail of [`new_with_format`]). Shared by the `Read` path and the
+    /// slice-direct path ([`FrameDecoder::reset_from_slice`]).
+    pub(crate) fn new_with_parsed_header(
+        frame: frame::FrameHeader,
+        header_size: u8,
+    ) -> Result<FrameDecoderState, FrameDecoderError> {
         let window_size = frame.window_size()?;
 
         if window_size > MAXIMUM_ALLOWED_WINDOW_SIZE {
@@ -906,6 +916,18 @@ impl FrameDecoderState {
         magicless: bool,
     ) -> Result<(), FrameDecoderError> {
         let (frame_header, header_size) = frame::read_frame_header_with_format(source, magicless)?;
+        self.reset_with_parsed_header(frame_header, header_size)
+    }
+
+    /// Apply an already-parsed frame header to this state (the non-parsing tail
+    /// of [`reset_with_format`]). Shared by the `Read` path and the slice-direct
+    /// path ([`FrameDecoder::reset_from_slice`]).
+    #[inline]
+    pub(crate) fn reset_with_parsed_header(
+        &mut self,
+        frame_header: frame::FrameHeader,
+        header_size: u8,
+    ) -> Result<(), FrameDecoderError> {
         let window_size = frame_header.window_size()?;
 
         if window_size > MAXIMUM_ALLOWED_WINDOW_SIZE {
@@ -1281,6 +1303,64 @@ impl FrameDecoder {
         Ok(())
     }
 
+    /// Slice-direct equivalent of [`reset`](Self::reset) for the in-memory
+    /// decode path: parses the frame header straight out of `*input` via
+    /// [`frame::read_frame_header_from_slice`] (no `Read`-trait `read_exact`
+    /// per field) and advances `*input` past it, then applies it through the
+    /// shared parsed-header path. Behaviour — including skippable-frame and
+    /// truncation errors, dictionary-id resolution, and pinned-expectation
+    /// validation — is identical to `reset`; only the header read avoids the
+    /// `io::impls` dispatch.
+    pub(crate) fn reset_from_slice(&mut self, input: &mut &[u8]) -> Result<(), FrameDecoderError> {
+        use FrameDecoderError as err;
+        #[cfg(all(feature = "lsm", feature = "hash"))]
+        self.computed_block_checksums.clear();
+        let magicless = self.magicless;
+        let (frame_header, header_size) = frame::read_frame_header_from_slice(input, magicless)?;
+        let dict_id = match &mut self.state {
+            Some(s) => {
+                s.reset_with_parsed_header(frame_header, header_size)?;
+                s.frame_header.dictionary_id()
+            }
+            None => {
+                self.state = Some(FrameDecoderState::new_with_parsed_header(
+                    frame_header,
+                    header_size,
+                )?);
+                self.state
+                    .as_ref()
+                    .and_then(|state| state.frame_header.dictionary_id())
+            }
+        };
+        #[cfg(feature = "lsm")]
+        if let Some(state) = self.state.as_ref() {
+            self.validate_expectations(&state.frame_header)?;
+        }
+        if let Some(dict_id) = dict_id {
+            let state = self.state.as_mut().expect("state initialized");
+            let owned_dicts = &self.owned_dicts;
+            #[cfg(target_has_atomic = "ptr")]
+            let shared_dicts = &self.shared_dicts;
+            let dict = owned_dicts
+                .get(&dict_id)
+                .or_else(|| {
+                    #[cfg(target_has_atomic = "ptr")]
+                    {
+                        shared_dicts.get(&dict_id)
+                    }
+                    #[cfg(not(target_has_atomic = "ptr"))]
+                    {
+                        None
+                    }
+                })
+                .ok_or(err::DictNotProvided { dict_id })?;
+            state.decoder_scratch.init_from_dict(dict);
+            state.set_active_dict(dict);
+            state.using_dict = Some(dict_id);
+        }
+        Ok(())
+    }
+
     /// Reset this decoder for a new frame using a pre-parsed dictionary handle.
     ///
     /// If the frame header has a dictionary ID, this validates it against
@@ -2286,11 +2366,11 @@ impl FrameDecoder {
     ) -> Result<usize, FrameDecoderError> {
         #[cfg(not(feature = "lsm"))]
         {
-            self.decode_all_impl(input, output, |this, src| this.init(src))
+            self.decode_all_impl(input, output, |this, src| this.reset_from_slice(src))
         }
         #[cfg(feature = "lsm")]
         {
-            self.decode_all_impl(input, output, |this, src| this.init(src), None)
+            self.decode_all_impl(input, output, |this, src| this.reset_from_slice(src), None)
         }
     }
 
@@ -2341,7 +2421,7 @@ impl FrameDecoder {
         self.decode_all_impl(
             input,
             output,
-            |this, src| this.init(src),
+            |this, src| this.reset_from_slice(src),
             Some(&mut visitor),
         )
     }
@@ -2946,6 +3026,69 @@ impl FrameDecoder {
             .as_mut()
             .expect("caller ensures init populated state");
 
+        // Fast path: a frame that is a single RAW block spanning the whole
+        // declared content. Upstream zstd handles this as one `ZSTD_copyRawBlock`
+        // (a `memmove`) inside `ZSTD_decompressFrame`; do the same here — a direct
+        // `copy_from_slice` into the caller's output slice — skipping the
+        // `DirectScratch` / `DecodeBuffer` / `UserSliceBackend` wrapper
+        // construction and the general per-block loop. Incompressible payloads
+        // (random / already-compressed data) emit exactly this shape, so the win
+        // lands on small high-entropy frames where the per-frame machinery, not
+        // the 1-block copy, dominates.
+        {
+            let mut probe = *input;
+            let mut header_dec = block_decoder::new();
+            if let Ok((bh, hsize)) = header_dec.read_block_header(&mut probe) {
+                let n = bh.decompressed_size as usize;
+                if bh.last_block
+                    && matches!(bh.block_type, crate::blocks::block::BlockType::Raw)
+                    && n as u64 == content_size
+                    && probe.len() >= n
+                    && output.len() >= n
+                {
+                    output[..n].copy_from_slice(&probe[..n]);
+                    *input = &probe[n..];
+                    state.bytes_read_counter += u64::from(hsize) + n as u64;
+                    state.block_counter += 1;
+                    // Consume the trailing 4-byte content checksum UNCONDITIONALLY
+                    // when the frame declares one — exactly like the general
+                    // direct loop and `decode_blocks`. Only the hash SEEDING is
+                    // `hash`-gated; the byte consumption / counter / `check_sum`
+                    // must not be, or a no-`hash` build leaves the 4 bytes in
+                    // `*input` (misparsed as the next frame) and never sets
+                    // `check_sum` (so `is_finished` stays false).
+                    if state.frame_header.descriptor.content_checksum_flag() {
+                        let mut chksum = [0u8; 4];
+                        Read::read_exact(input, &mut chksum).map_err(err::FailedToReadChecksum)?;
+                        state.bytes_read_counter += 4;
+                        state.check_sum = Some(u32::from_le_bytes(chksum));
+                        // Mirror the general path: seed the scratch hash so
+                        // `verify_content_checksum` / `get_calculated_checksum`
+                        // read the digest. Skipped under `ContentChecksum::None`.
+                        #[cfg(feature = "hash")]
+                        if self.content_checksum != ContentChecksum::None {
+                            use core::hash::Hasher;
+                            let mut h = twox_hash::XxHash64::with_seed(0);
+                            h.write(&output[..n]);
+                            match &mut state.decoder_scratch {
+                                DecoderScratchKind::Flat(s) => s.buffer.hash = h,
+                                DecoderScratchKind::Ring(s) => s.buffer.hash = h,
+                            }
+                        }
+                    }
+                    #[cfg(all(feature = "lsm", feature = "hash"))]
+                    if self.per_block_checksums_enabled {
+                        use core::hash::Hasher;
+                        let mut h = twox_hash::XxHash64::with_seed(0);
+                        h.write(&output[..n]);
+                        self.computed_block_checksums.push(h.finish() as u32);
+                    }
+                    state.frame_finished = true;
+                    return Ok(n);
+                }
+            }
+        }
+
         // Borrow persistent fields out of whichever scratch variant
         // `init` produced (Flat for single_segment, Ring for
         // multi-segment) — both expose the same HUF/FSE/Vec

zstd/src/decoding/scratch.rs

@@ -216,9 +216,22 @@ impl<B: BufferBackend> DecoderScratch<B> {
 
         self.buffer.reset(window_size);
 
-        self.fse.literal_lengths.reset();
-        self.fse.match_lengths.reset();
-        self.fse.offsets.reset();
+        // Mirror upstream zstd's `ZSTD_decompressBegin`: it never clears the
+        // entropy tables per frame (marks them invalid by flag, rebuilds lazily).
+        // Resetting an already-empty table is a no-op on the observable state
+        // (`accuracy_log`/`max_num_bits` stay 0, the buffers stay empty), so a
+        // RAW / RLE / zero-sequence frame — which never builds a table — skips
+        // the clears (and the Huffman `[0; 13]` rank-count memset) entirely.
+        // Byte-identical: the table is cleared exactly when it was built.
+        if self.fse.literal_lengths.is_populated() {
+            self.fse.literal_lengths.reset();
+        }
+        if self.fse.match_lengths.is_populated() {
+            self.fse.match_lengths.reset();
+        }
+        if self.fse.offsets.is_populated() {
+            self.fse.offsets.reset();
+        }
         // Reset the cached pipeline-gate signal alongside the FSE
         // table reset — otherwise scratch reuse across frames could
         // engage the long pipeline on a new frame's Repeat-mode
@@ -239,7 +252,14 @@ impl<B: BufferBackend> DecoderScratch<B> {
         // the documented "no behaviour change" property.
         self.fse.ddict_is_cold = false;
 
-        self.huf.table.reset();
+        // Same lazy-entropy skip as the FSE tables above: a Huffman table that
+        // was never built this round (`max_num_bits == 0`) is already in its
+        // reset state, so re-running `reset` (clearing 6 buffers + the
+        // `[0; 13]` rank-count memset) is wasted no-op work on RAW / RLE / raw-
+        // literal frames. Byte-identical.
+        if self.huf.table.max_num_bits != 0 {
+            self.huf.table.reset();
+        }
         // Mirror the FSE detach: a reused workspace must not read a
         // previous frame's dictionary Huffman table.
         self.huf.detach_dict();

zstd/src/fse/fse_decoder.rs

@@ -391,6 +391,20 @@ impl<E: FseEntry, const CAP: usize> FSETableImpl<E, CAP> {
         self.accuracy_log = 0;
     }
 
+    /// Whether this table holds a built decode table. A just-reset / never-built
+    /// table reads as `false`; resetting it again is a no-op, so the per-frame
+    /// scratch reset can skip it (mirrors upstream zstd, which never clears
+    /// entropy tables per frame — it marks them invalid by flag and rebuilds
+    /// lazily). The signal is `decode_len != 0`, NOT `accuracy_log != 0`: a valid
+    /// RLE DTable (`build_rle`) has `decode_len == 1` but `accuracy_log == 0`, so
+    /// an `accuracy_log` check would miss it and leave a used RLE table uncleared
+    /// for a later Repeat-mode frame. `init_state` uses the same `decode_len`
+    /// signal for "uninitialized".
+    #[inline]
+    pub(crate) fn is_populated(&self) -> bool {
+        self.decode_len != 0
+    }
+
     /// Build the equivalent encoder-side table from a parsed decoder table.
     pub(crate) fn to_encoder_table(&self) -> Option<crate::fse::fse_encoder::FSETable> {
         if self.accuracy_log == 0 || self.symbol_probabilities.is_empty() {

zstd/src/fse/fse_decoder/tests.rs

@@ -123,3 +123,21 @@ fn to_encoder_table_rejects_accuracy_log_above_nine() {
         "accuracy_log > 9 must not be reusable as an encoder table"
     );
 }
+
+/// Regression: `is_populated` must report a built table for RLE tables, not just
+/// FSE-mode ones. `build_rle` sets `decode_len = 1` but leaves `accuracy_log` at
+/// 0 (a valid RLE DTable's tableLog), so an `accuracy_log != 0` check would miss
+/// it — and the per-frame scratch reset that gates on `is_populated` would then
+/// SKIP clearing a used RLE sequence table, letting a later Repeat-mode frame
+/// read stale RLE state. The "built" signal is `decode_len != 0`, matching the
+/// uninitialized check `init_state` uses.
+#[test]
+fn is_populated_detects_built_rle_table() {
+    let mut t = SeqFSETable::new(9);
+    assert!(!t.is_populated(), "a fresh table must report unpopulated");
+    t.build_rle(7);
+    assert!(
+        t.is_populated(),
+        "an RLE table (decode_len = 1, accuracy_log = 0) must report populated"
+    );
+}