perf(codec): dict-band hot-path — slice header parse, dms-walk floor, SIMD prefix leading word

zstd/src/decoding/frame_decoder.rs

@@ -1434,6 +1434,60 @@ impl FrameDecoder {
         Ok(())
     }
 
+    /// Slice-direct equivalent of [`reset_with_dict_handle`](Self::reset_with_dict_handle)
+    /// for the in-memory decode path: parses the frame header straight out of
+    /// `*input` via [`frame::read_frame_header_from_slice`] (no `Read`-trait
+    /// `read_exact` per field) and applies it through the shared parsed-header
+    /// path, then attaches `dict`. Behaviour — dictionary-id mismatch, pinned
+    /// expectations, scratch init — is identical to `reset_with_dict_handle`;
+    /// only the header read avoids the `io::impls` dispatch.
+    pub(crate) fn reset_from_slice_with_dict_handle(
+        &mut self,
+        input: &mut &[u8],
+        dict: &DictionaryHandle,
+    ) -> Result<(), FrameDecoderError> {
+        use FrameDecoderError as err;
+        #[cfg(all(feature = "lsm", feature = "hash"))]
+        self.computed_block_checksums.clear();
+        Self::validate_registered_dictionary(dict.as_dict())?;
+        let magicless = self.magicless;
+        let (frame_header, header_size) = frame::read_frame_header_from_slice(input, magicless)?;
+        match &mut self.state {
+            Some(s) => s.reset_with_parsed_header(frame_header, header_size)?,
+            None => {
+                self.state = Some(FrameDecoderState::new_with_parsed_header(
+                    frame_header,
+                    header_size,
+                )?);
+            }
+        }
+        #[cfg(feature = "lsm")]
+        {
+            let header = &self
+                .state
+                .as_ref()
+                .expect("state populated by reset_with_parsed_header/new_with_parsed_header")
+                .frame_header;
+            self.validate_expectations(header)?;
+        }
+        let state = self
+            .state
+            .as_mut()
+            .expect("state populated by reset_with_parsed_header/new_with_parsed_header");
+        if let Some(dict_id) = state.frame_header.dictionary_id()
+            && dict_id != dict.id()
+        {
+            return Err(err::DictIdMismatch {
+                expected: dict_id,
+                provided: dict.id(),
+            });
+        }
+        state.decoder_scratch.init_from_dict(dict);
+        state.set_active_dict(dict);
+        state.using_dict = Some(dict.id());
+        Ok(())
+    }
+
     /// Add a dictionary that can be selected dynamically by frame dictionary ID.
     ///
     /// Returns [`FrameDecoderError::DictAlreadyRegistered`] if the ID is already
@@ -2452,15 +2506,15 @@ impl FrameDecoder {
         #[cfg(not(feature = "lsm"))]
         {
             self.decode_all_impl(input, output, |this, src| {
-                this.init_with_dict_handle(src, dict)
+                this.reset_from_slice_with_dict_handle(src, dict)
             })
         }
         #[cfg(feature = "lsm")]
         {
             self.decode_all_impl(
                 input,
                 output,
-                |this, src| this.init_with_dict_handle(src, dict),
+                |this, src| this.reset_from_slice_with_dict_handle(src, dict),
                 None,
             )
         }

zstd/src/encoding/fastpath/avx2_bmi2.rs

@@ -73,6 +73,23 @@ pub(crate) unsafe fn prefix_len_simd(lhs: *const u8, rhs: *const u8, max: usize)
 #[target_feature(enable = "avx2,bmi2")]
 #[inline]
 pub(crate) unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize {
+    // Leading scalar word probe (mirrors upstream C `ZSTD_count`'s first
+    // `MEM_readST` check). A prefix extension that diverges inside the first 8
+    // bytes — the common case on high-entropy / dictionary-seeded BT-tree node
+    // compares, where the candidate and the cursor share only a short run —
+    // returns here on a single 8-byte read + `xor` + `tzcnt`, skipping the
+    // 32-byte vector load + AVX2 compare entirely. A longer match (first word
+    // equal) falls through to the vector loop, which re-covers those 8 bytes;
+    // the one redundant word read is negligible against the wide-match win.
+    let chunk = core::mem::size_of::<usize>();
+    if chunk <= max {
+        let lhs_word = unsafe { core::ptr::read_unaligned(lhs.cast::<usize>()) };
+        let rhs_word = unsafe { core::ptr::read_unaligned(rhs.cast::<usize>()) };
+        let diff = lhs_word ^ rhs_word;
+        if diff != 0 {
+            return scalar::mismatch_byte_index(diff);
+        }
+    }
     let off = unsafe { prefix_len_simd(lhs, rhs, max) };
     unsafe { scalar::common_prefix_len_scalar_ptr(lhs, rhs, off, max) }
 }

zstd/src/encoding/fastpath/neon.rs

@@ -79,6 +79,20 @@ pub(crate) unsafe fn prefix_len_simd(lhs: *const u8, rhs: *const u8, max: usize)
 #[target_feature(enable = "neon")]
 #[inline]
 pub(crate) unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize {
+    // Leading scalar word probe (mirrors upstream C `ZSTD_count`'s first
+    // `MEM_readST` check): a prefix that diverges within the first 8 bytes — the
+    // common case on BT-tree node compares, where each node extends the seed by
+    // only a short run — returns on one 8-byte read + count-trailing-zeros,
+    // skipping the vector load. Longer matches fall through to the vector loop.
+    let chunk = core::mem::size_of::<usize>();
+    if chunk <= max {
+        let lhs_word = unsafe { core::ptr::read_unaligned(lhs.cast::<usize>()) };
+        let rhs_word = unsafe { core::ptr::read_unaligned(rhs.cast::<usize>()) };
+        let diff = lhs_word ^ rhs_word;
+        if diff != 0 {
+            return scalar::mismatch_byte_index(diff);
+        }
+    }
     let off = unsafe { prefix_len_simd(lhs, rhs, max) };
     unsafe { scalar::common_prefix_len_scalar_ptr(lhs, rhs, off, max) }
 }

zstd/src/encoding/fastpath/simd128.rs

@@ -67,6 +67,20 @@ pub(crate) unsafe fn prefix_len_simd(lhs: *const u8, rhs: *const u8, max: usize)
 #[target_feature(enable = "simd128")]
 #[inline]
 pub(crate) unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize {
+    // Leading scalar word probe (mirrors upstream C `ZSTD_count`'s first
+    // `MEM_readST` check): a prefix that diverges within the first 8 bytes — the
+    // common case on BT-tree node compares, where each node extends the seed by
+    // only a short run — returns on one 8-byte read + count-trailing-zeros,
+    // skipping the vector load. Longer matches fall through to the vector loop.
+    let chunk = core::mem::size_of::<usize>();
+    if chunk <= max {
+        let lhs_word = unsafe { core::ptr::read_unaligned(lhs.cast::<usize>()) };
+        let rhs_word = unsafe { core::ptr::read_unaligned(rhs.cast::<usize>()) };
+        let diff = lhs_word ^ rhs_word;
+        if diff != 0 {
+            return scalar::mismatch_byte_index(diff);
+        }
+    }
     let off = unsafe { prefix_len_simd(lhs, rhs, max) };
     unsafe { scalar::common_prefix_len_scalar_ptr(lhs, rhs, off, max) }
 }

zstd/src/encoding/fastpath/sse42.rs

@@ -70,6 +70,20 @@ pub(crate) unsafe fn prefix_len_simd(lhs: *const u8, rhs: *const u8, max: usize)
 #[target_feature(enable = "sse4.2")]
 #[inline]
 pub(crate) unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize {
+    // Leading scalar word probe (mirrors upstream C `ZSTD_count`'s first
+    // `MEM_readST` check): a prefix that diverges within the first 8 bytes — the
+    // common case on BT-tree node compares, where each node extends the seed by
+    // only a short run — returns on one 8-byte read + `tzcnt`, skipping the
+    // vector load. Longer matches fall through to the vector loop.
+    let chunk = core::mem::size_of::<usize>();
+    if chunk <= max {
+        let lhs_word = unsafe { core::ptr::read_unaligned(lhs.cast::<usize>()) };
+        let rhs_word = unsafe { core::ptr::read_unaligned(rhs.cast::<usize>()) };
+        let diff = lhs_word ^ rhs_word;
+        if diff != 0 {
+            return scalar::mismatch_byte_index(diff);
+        }
+    }
     let off = unsafe { prefix_len_simd(lhs, rhs, max) };
     unsafe { scalar::common_prefix_len_scalar_ptr(lhs, rhs, off, max) }
 }

zstd/src/encoding/hc/mod.rs

@@ -547,6 +547,19 @@ impl HcMatcher {
         // Hoist the dms chain-table base pointer (same Vec-header-reload removal
         // as the live walk above).
         let dms_chain_ptr: *const u32 = dms.chain_table.as_ptr();
+        // Reachability floor, hoisted out of the loop. A dict position is
+        // decoder-reachable iff its offset (`current_idx - dict_idx`) does not
+        // exceed `max_window_size`, i.e. `dict_idx >= current_idx -
+        // max_window_size`. Comparing `dict_idx` against this precomputed floor
+        // drops the per-iteration `current_idx - dict_idx` subtract from the hot
+        // path (the offset is only needed when a candidate WINS, which is rare on
+        // the walk) and frees the `max_window_size` register inside the loop.
+        // `saturating_sub` is the intended business logic, not an overflow guard:
+        // when `current_idx <= max_window_size` the whole live history fits the
+        // window, so every dict position is reachable and the floor is 0. Mirrors
+        // C's `ZSTD_HcFindBestMatch` dms loop, which bounds reachability by chain
+        // construction rather than re-checking each candidate.
+        let dict_reachable_floor = current_idx.saturating_sub(table.max_window_size);
         while steps < dms_budget {
             if dcur == 0 {
                 break;
@@ -558,9 +571,8 @@ impl HcMatcher {
             // SAFETY: `dict_idx` is a stored dict position `< chain_table.len()`.
             let dnext = unsafe { *dms_chain_ptr.add(dict_idx) };
             steps += 1;
-            let new_offset = current_idx - dict_idx;
             // Out-of-window dict positions are unreachable to the decoder.
-            if new_offset <= table.max_window_size {
+            if dict_idx >= dict_reachable_floor {
                 // Same self-tightening gate as the live walk; the caller's guard
                 // (`current_idx + ml < history_tail`) + the iLimit break keep
                 // `current_idx + (ml - 3) + 4 <= history_tail`, and