From 24ddb8ab30860e4bd1a06f486bd8a80d0cc6614e Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Tue, 30 Jun 2026 00:18:25 +0300 Subject: [PATCH 1/3] perf(decode): slice-direct header parse on the dict-handle path decode_all_with_dict_handle still routed its per-frame header read through the Read-trait reset_with_dict_handle (per-field read_exact via the io::impls shim) while the non-dict decode_all already parsed the header straight from the input slice. Add reset_from_slice_with_dict_handle, mirroring the Read-based dict reset but reading the header via read_frame_header_from_slice and applying it through the shared parsed-header path, then attaching the dictionary. Wire decode_all_with_dict_handle through it. Byte-identical (dict cross-validation + fuzz_interop round-trips pass). --- zstd/src/decoding/frame_decoder.rs | 58 ++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/zstd/src/decoding/frame_decoder.rs b/zstd/src/decoding/frame_decoder.rs index 8b0808201..ed7bfb891 100644 --- a/zstd/src/decoding/frame_decoder.rs +++ b/zstd/src/decoding/frame_decoder.rs @@ -1434,6 +1434,60 @@ impl FrameDecoder { Ok(()) } + /// Slice-direct equivalent of [`reset_with_dict_handle`](Self::reset_with_dict_handle) + /// for the in-memory decode path: parses the frame header straight out of + /// `*input` via [`frame::read_frame_header_from_slice`] (no `Read`-trait + /// `read_exact` per field) and applies it through the shared parsed-header + /// path, then attaches `dict`. Behaviour — dictionary-id mismatch, pinned + /// expectations, scratch init — is identical to `reset_with_dict_handle`; + /// only the header read avoids the `io::impls` dispatch. + pub(crate) fn reset_from_slice_with_dict_handle( + &mut self, + input: &mut &[u8], + dict: &DictionaryHandle, + ) -> Result<(), FrameDecoderError> { + use FrameDecoderError as err; + #[cfg(all(feature = "lsm", feature = "hash"))] + self.computed_block_checksums.clear(); + Self::validate_registered_dictionary(dict.as_dict())?; + let magicless = self.magicless; + let (frame_header, header_size) = frame::read_frame_header_from_slice(input, magicless)?; + match &mut self.state { + Some(s) => s.reset_with_parsed_header(frame_header, header_size)?, + None => { + self.state = Some(FrameDecoderState::new_with_parsed_header( + frame_header, + header_size, + )?); + } + } + #[cfg(feature = "lsm")] + { + let header = &self + .state + .as_ref() + .expect("state populated by reset_with_parsed_header/new_with_parsed_header") + .frame_header; + self.validate_expectations(header)?; + } + let state = self + .state + .as_mut() + .expect("state populated by reset_with_parsed_header/new_with_parsed_header"); + if let Some(dict_id) = state.frame_header.dictionary_id() + && dict_id != dict.id() + { + return Err(err::DictIdMismatch { + expected: dict_id, + provided: dict.id(), + }); + } + state.decoder_scratch.init_from_dict(dict); + state.set_active_dict(dict); + state.using_dict = Some(dict.id()); + Ok(()) + } + /// Add a dictionary that can be selected dynamically by frame dictionary ID. /// /// Returns [`FrameDecoderError::DictAlreadyRegistered`] if the ID is already @@ -2452,7 +2506,7 @@ impl FrameDecoder { #[cfg(not(feature = "lsm"))] { self.decode_all_impl(input, output, |this, src| { - this.init_with_dict_handle(src, dict) + this.reset_from_slice_with_dict_handle(src, dict) }) } #[cfg(feature = "lsm")] @@ -2460,7 +2514,7 @@ impl FrameDecoder { self.decode_all_impl( input, output, - |this, src| this.init_with_dict_handle(src, dict), + |this, src| this.reset_from_slice_with_dict_handle(src, dict), None, ) } From 8e00972c8aaa14b054c40ca4501f09715af3d3d9 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Tue, 30 Jun 2026 00:40:10 +0300 Subject: [PATCH 2/3] perf(compress): hoist the dict-walk reachability floor out of the loop The lazy hash-chain dict-match walk recomputed the candidate offset (current_idx - dict_idx) every iteration solely to gate on decoder reachability (offset <= max_window_size). The offset is only consumed when a candidate WINS, which is rare on the walk. Hoist the equivalent floor (current_idx.saturating_sub(max_window_size)) once before the loop and gate on dict_idx >= floor; the per-iteration subtract moves to the (rare) win branch and the max_window_size register is freed inside the hot loop. Mirrors C's ZSTD_HcFindBestMatch dms loop, which bounds reachability by chain construction rather than re-checking each candidate. Byte-identical: dict_idx >= current_idx - max_window_size is algebraically the same gate as current_idx - dict_idx <= max_window_size given dict_idx < current_idx (dict positions precede the cursor). Dict cross-validation + fuzz_interop round-trips pass. --- zstd/src/encoding/hc/mod.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/zstd/src/encoding/hc/mod.rs b/zstd/src/encoding/hc/mod.rs index 43797be93..588b75477 100644 --- a/zstd/src/encoding/hc/mod.rs +++ b/zstd/src/encoding/hc/mod.rs @@ -547,6 +547,19 @@ impl HcMatcher { // Hoist the dms chain-table base pointer (same Vec-header-reload removal // as the live walk above). let dms_chain_ptr: *const u32 = dms.chain_table.as_ptr(); + // Reachability floor, hoisted out of the loop. A dict position is + // decoder-reachable iff its offset (`current_idx - dict_idx`) does not + // exceed `max_window_size`, i.e. `dict_idx >= current_idx - + // max_window_size`. Comparing `dict_idx` against this precomputed floor + // drops the per-iteration `current_idx - dict_idx` subtract from the hot + // path (the offset is only needed when a candidate WINS, which is rare on + // the walk) and frees the `max_window_size` register inside the loop. + // `saturating_sub` is the intended business logic, not an overflow guard: + // when `current_idx <= max_window_size` the whole live history fits the + // window, so every dict position is reachable and the floor is 0. Mirrors + // C's `ZSTD_HcFindBestMatch` dms loop, which bounds reachability by chain + // construction rather than re-checking each candidate. + let dict_reachable_floor = current_idx.saturating_sub(table.max_window_size); while steps < dms_budget { if dcur == 0 { break; @@ -558,9 +571,8 @@ impl HcMatcher { // SAFETY: `dict_idx` is a stored dict position `< chain_table.len()`. let dnext = unsafe { *dms_chain_ptr.add(dict_idx) }; steps += 1; - let new_offset = current_idx - dict_idx; // Out-of-window dict positions are unreachable to the decoder. - if new_offset <= table.max_window_size { + if dict_idx >= dict_reachable_floor { // Same self-tightening gate as the live walk; the caller's guard // (`current_idx + ml < history_tail`) + the iLimit break keep // `current_idx + (ml - 3) + 4 <= history_tail`, and From 1aa1aab80865db8f6ca196a2ca5e5ac9eee6a2fb Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Tue, 30 Jun 2026 10:53:34 +0300 Subject: [PATCH 3/3] perf(compress): leading scalar word probe in the SIMD common-prefix kernels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SIMD common_prefix_len_ptr (avx2 / sse42 / neon / simd128) ran the wide vector loop even when the prefix diverges within the first 8 bytes. On BT-tree node compares — the optimal parser calls the seeded prefix counter per visited node, and each node extends the running seed by only a short run before the smaller/larger split — that divergence is almost always inside the first word, so the vector load + compare is pure overhead. Upstream C ZSTD_count leads with one MEM_readST (8-byte) check and returns on a mismatch there; mirror it with a leading scalar word probe, falling through to the vector loop only when the first word matches (long matches). i686 (ours-vs-c_ffi, flat control): compress-dict L13 small-10k-random 782->725us (-7.2%); compress L19 btultra2 large-log-stream 64.5->59.5ms (-7.7%) — the long-match path benefits too, since per-node tree extensions are short regardless of total match length. Byte-identical (838 lib + 59 ffi incl cross-validation; scalar already word-at-a-time). The scalar fallback kernel is unchanged. --- zstd/src/encoding/fastpath/avx2_bmi2.rs | 17 +++++++++++++++++ zstd/src/encoding/fastpath/neon.rs | 14 ++++++++++++++ zstd/src/encoding/fastpath/simd128.rs | 14 ++++++++++++++ zstd/src/encoding/fastpath/sse42.rs | 14 ++++++++++++++ 4 files changed, 59 insertions(+) diff --git a/zstd/src/encoding/fastpath/avx2_bmi2.rs b/zstd/src/encoding/fastpath/avx2_bmi2.rs index 0f4bcec0f..555c3bdad 100644 --- a/zstd/src/encoding/fastpath/avx2_bmi2.rs +++ b/zstd/src/encoding/fastpath/avx2_bmi2.rs @@ -73,6 +73,23 @@ pub(crate) unsafe fn prefix_len_simd(lhs: *const u8, rhs: *const u8, max: usize) #[target_feature(enable = "avx2,bmi2")] #[inline] pub(crate) unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize { + // Leading scalar word probe (mirrors upstream C `ZSTD_count`'s first + // `MEM_readST` check). A prefix extension that diverges inside the first 8 + // bytes — the common case on high-entropy / dictionary-seeded BT-tree node + // compares, where the candidate and the cursor share only a short run — + // returns here on a single 8-byte read + `xor` + `tzcnt`, skipping the + // 32-byte vector load + AVX2 compare entirely. A longer match (first word + // equal) falls through to the vector loop, which re-covers those 8 bytes; + // the one redundant word read is negligible against the wide-match win. + let chunk = core::mem::size_of::(); + if chunk <= max { + let lhs_word = unsafe { core::ptr::read_unaligned(lhs.cast::()) }; + let rhs_word = unsafe { core::ptr::read_unaligned(rhs.cast::()) }; + let diff = lhs_word ^ rhs_word; + if diff != 0 { + return scalar::mismatch_byte_index(diff); + } + } let off = unsafe { prefix_len_simd(lhs, rhs, max) }; unsafe { scalar::common_prefix_len_scalar_ptr(lhs, rhs, off, max) } } diff --git a/zstd/src/encoding/fastpath/neon.rs b/zstd/src/encoding/fastpath/neon.rs index c9464b4e8..0bd7404ae 100644 --- a/zstd/src/encoding/fastpath/neon.rs +++ b/zstd/src/encoding/fastpath/neon.rs @@ -79,6 +79,20 @@ pub(crate) unsafe fn prefix_len_simd(lhs: *const u8, rhs: *const u8, max: usize) #[target_feature(enable = "neon")] #[inline] pub(crate) unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize { + // Leading scalar word probe (mirrors upstream C `ZSTD_count`'s first + // `MEM_readST` check): a prefix that diverges within the first 8 bytes — the + // common case on BT-tree node compares, where each node extends the seed by + // only a short run — returns on one 8-byte read + count-trailing-zeros, + // skipping the vector load. Longer matches fall through to the vector loop. + let chunk = core::mem::size_of::(); + if chunk <= max { + let lhs_word = unsafe { core::ptr::read_unaligned(lhs.cast::()) }; + let rhs_word = unsafe { core::ptr::read_unaligned(rhs.cast::()) }; + let diff = lhs_word ^ rhs_word; + if diff != 0 { + return scalar::mismatch_byte_index(diff); + } + } let off = unsafe { prefix_len_simd(lhs, rhs, max) }; unsafe { scalar::common_prefix_len_scalar_ptr(lhs, rhs, off, max) } } diff --git a/zstd/src/encoding/fastpath/simd128.rs b/zstd/src/encoding/fastpath/simd128.rs index 123d33e09..5a727762f 100644 --- a/zstd/src/encoding/fastpath/simd128.rs +++ b/zstd/src/encoding/fastpath/simd128.rs @@ -67,6 +67,20 @@ pub(crate) unsafe fn prefix_len_simd(lhs: *const u8, rhs: *const u8, max: usize) #[target_feature(enable = "simd128")] #[inline] pub(crate) unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize { + // Leading scalar word probe (mirrors upstream C `ZSTD_count`'s first + // `MEM_readST` check): a prefix that diverges within the first 8 bytes — the + // common case on BT-tree node compares, where each node extends the seed by + // only a short run — returns on one 8-byte read + count-trailing-zeros, + // skipping the vector load. Longer matches fall through to the vector loop. + let chunk = core::mem::size_of::(); + if chunk <= max { + let lhs_word = unsafe { core::ptr::read_unaligned(lhs.cast::()) }; + let rhs_word = unsafe { core::ptr::read_unaligned(rhs.cast::()) }; + let diff = lhs_word ^ rhs_word; + if diff != 0 { + return scalar::mismatch_byte_index(diff); + } + } let off = unsafe { prefix_len_simd(lhs, rhs, max) }; unsafe { scalar::common_prefix_len_scalar_ptr(lhs, rhs, off, max) } } diff --git a/zstd/src/encoding/fastpath/sse42.rs b/zstd/src/encoding/fastpath/sse42.rs index 86788ae43..1623391fd 100644 --- a/zstd/src/encoding/fastpath/sse42.rs +++ b/zstd/src/encoding/fastpath/sse42.rs @@ -70,6 +70,20 @@ pub(crate) unsafe fn prefix_len_simd(lhs: *const u8, rhs: *const u8, max: usize) #[target_feature(enable = "sse4.2")] #[inline] pub(crate) unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize { + // Leading scalar word probe (mirrors upstream C `ZSTD_count`'s first + // `MEM_readST` check): a prefix that diverges within the first 8 bytes — the + // common case on BT-tree node compares, where each node extends the seed by + // only a short run — returns on one 8-byte read + `tzcnt`, skipping the + // vector load. Longer matches fall through to the vector loop. + let chunk = core::mem::size_of::(); + if chunk <= max { + let lhs_word = unsafe { core::ptr::read_unaligned(lhs.cast::()) }; + let rhs_word = unsafe { core::ptr::read_unaligned(rhs.cast::()) }; + let diff = lhs_word ^ rhs_word; + if diff != 0 { + return scalar::mismatch_byte_index(diff); + } + } let off = unsafe { prefix_len_simd(lhs, rhs, max) }; unsafe { scalar::common_prefix_len_scalar_ptr(lhs, rhs, off, max) } }