diff --git a/zstd/src/decoding/frame_decoder.rs b/zstd/src/decoding/frame_decoder.rs index 8b0808201..ed7bfb891 100644 --- a/zstd/src/decoding/frame_decoder.rs +++ b/zstd/src/decoding/frame_decoder.rs @@ -1434,6 +1434,60 @@ impl FrameDecoder { Ok(()) } + /// Slice-direct equivalent of [`reset_with_dict_handle`](Self::reset_with_dict_handle) + /// for the in-memory decode path: parses the frame header straight out of + /// `*input` via [`frame::read_frame_header_from_slice`] (no `Read`-trait + /// `read_exact` per field) and applies it through the shared parsed-header + /// path, then attaches `dict`. Behaviour — dictionary-id mismatch, pinned + /// expectations, scratch init — is identical to `reset_with_dict_handle`; + /// only the header read avoids the `io::impls` dispatch. + pub(crate) fn reset_from_slice_with_dict_handle( + &mut self, + input: &mut &[u8], + dict: &DictionaryHandle, + ) -> Result<(), FrameDecoderError> { + use FrameDecoderError as err; + #[cfg(all(feature = "lsm", feature = "hash"))] + self.computed_block_checksums.clear(); + Self::validate_registered_dictionary(dict.as_dict())?; + let magicless = self.magicless; + let (frame_header, header_size) = frame::read_frame_header_from_slice(input, magicless)?; + match &mut self.state { + Some(s) => s.reset_with_parsed_header(frame_header, header_size)?, + None => { + self.state = Some(FrameDecoderState::new_with_parsed_header( + frame_header, + header_size, + )?); + } + } + #[cfg(feature = "lsm")] + { + let header = &self + .state + .as_ref() + .expect("state populated by reset_with_parsed_header/new_with_parsed_header") + .frame_header; + self.validate_expectations(header)?; + } + let state = self + .state + .as_mut() + .expect("state populated by reset_with_parsed_header/new_with_parsed_header"); + if let Some(dict_id) = state.frame_header.dictionary_id() + && dict_id != dict.id() + { + return Err(err::DictIdMismatch { + expected: dict_id, + provided: dict.id(), + }); + } + state.decoder_scratch.init_from_dict(dict); + state.set_active_dict(dict); + state.using_dict = Some(dict.id()); + Ok(()) + } + /// Add a dictionary that can be selected dynamically by frame dictionary ID. /// /// Returns [`FrameDecoderError::DictAlreadyRegistered`] if the ID is already @@ -2452,7 +2506,7 @@ impl FrameDecoder { #[cfg(not(feature = "lsm"))] { self.decode_all_impl(input, output, |this, src| { - this.init_with_dict_handle(src, dict) + this.reset_from_slice_with_dict_handle(src, dict) }) } #[cfg(feature = "lsm")] @@ -2460,7 +2514,7 @@ impl FrameDecoder { self.decode_all_impl( input, output, - |this, src| this.init_with_dict_handle(src, dict), + |this, src| this.reset_from_slice_with_dict_handle(src, dict), None, ) } diff --git a/zstd/src/encoding/fastpath/avx2_bmi2.rs b/zstd/src/encoding/fastpath/avx2_bmi2.rs index 0f4bcec0f..555c3bdad 100644 --- a/zstd/src/encoding/fastpath/avx2_bmi2.rs +++ b/zstd/src/encoding/fastpath/avx2_bmi2.rs @@ -73,6 +73,23 @@ pub(crate) unsafe fn prefix_len_simd(lhs: *const u8, rhs: *const u8, max: usize) #[target_feature(enable = "avx2,bmi2")] #[inline] pub(crate) unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize { + // Leading scalar word probe (mirrors upstream C `ZSTD_count`'s first + // `MEM_readST` check). A prefix extension that diverges inside the first 8 + // bytes — the common case on high-entropy / dictionary-seeded BT-tree node + // compares, where the candidate and the cursor share only a short run — + // returns here on a single 8-byte read + `xor` + `tzcnt`, skipping the + // 32-byte vector load + AVX2 compare entirely. A longer match (first word + // equal) falls through to the vector loop, which re-covers those 8 bytes; + // the one redundant word read is negligible against the wide-match win. + let chunk = core::mem::size_of::(); + if chunk <= max { + let lhs_word = unsafe { core::ptr::read_unaligned(lhs.cast::()) }; + let rhs_word = unsafe { core::ptr::read_unaligned(rhs.cast::()) }; + let diff = lhs_word ^ rhs_word; + if diff != 0 { + return scalar::mismatch_byte_index(diff); + } + } let off = unsafe { prefix_len_simd(lhs, rhs, max) }; unsafe { scalar::common_prefix_len_scalar_ptr(lhs, rhs, off, max) } } diff --git a/zstd/src/encoding/fastpath/neon.rs b/zstd/src/encoding/fastpath/neon.rs index c9464b4e8..0bd7404ae 100644 --- a/zstd/src/encoding/fastpath/neon.rs +++ b/zstd/src/encoding/fastpath/neon.rs @@ -79,6 +79,20 @@ pub(crate) unsafe fn prefix_len_simd(lhs: *const u8, rhs: *const u8, max: usize) #[target_feature(enable = "neon")] #[inline] pub(crate) unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize { + // Leading scalar word probe (mirrors upstream C `ZSTD_count`'s first + // `MEM_readST` check): a prefix that diverges within the first 8 bytes — the + // common case on BT-tree node compares, where each node extends the seed by + // only a short run — returns on one 8-byte read + count-trailing-zeros, + // skipping the vector load. Longer matches fall through to the vector loop. + let chunk = core::mem::size_of::(); + if chunk <= max { + let lhs_word = unsafe { core::ptr::read_unaligned(lhs.cast::()) }; + let rhs_word = unsafe { core::ptr::read_unaligned(rhs.cast::()) }; + let diff = lhs_word ^ rhs_word; + if diff != 0 { + return scalar::mismatch_byte_index(diff); + } + } let off = unsafe { prefix_len_simd(lhs, rhs, max) }; unsafe { scalar::common_prefix_len_scalar_ptr(lhs, rhs, off, max) } } diff --git a/zstd/src/encoding/fastpath/simd128.rs b/zstd/src/encoding/fastpath/simd128.rs index 123d33e09..5a727762f 100644 --- a/zstd/src/encoding/fastpath/simd128.rs +++ b/zstd/src/encoding/fastpath/simd128.rs @@ -67,6 +67,20 @@ pub(crate) unsafe fn prefix_len_simd(lhs: *const u8, rhs: *const u8, max: usize) #[target_feature(enable = "simd128")] #[inline] pub(crate) unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize { + // Leading scalar word probe (mirrors upstream C `ZSTD_count`'s first + // `MEM_readST` check): a prefix that diverges within the first 8 bytes — the + // common case on BT-tree node compares, where each node extends the seed by + // only a short run — returns on one 8-byte read + count-trailing-zeros, + // skipping the vector load. Longer matches fall through to the vector loop. + let chunk = core::mem::size_of::(); + if chunk <= max { + let lhs_word = unsafe { core::ptr::read_unaligned(lhs.cast::()) }; + let rhs_word = unsafe { core::ptr::read_unaligned(rhs.cast::()) }; + let diff = lhs_word ^ rhs_word; + if diff != 0 { + return scalar::mismatch_byte_index(diff); + } + } let off = unsafe { prefix_len_simd(lhs, rhs, max) }; unsafe { scalar::common_prefix_len_scalar_ptr(lhs, rhs, off, max) } } diff --git a/zstd/src/encoding/fastpath/sse42.rs b/zstd/src/encoding/fastpath/sse42.rs index 86788ae43..1623391fd 100644 --- a/zstd/src/encoding/fastpath/sse42.rs +++ b/zstd/src/encoding/fastpath/sse42.rs @@ -70,6 +70,20 @@ pub(crate) unsafe fn prefix_len_simd(lhs: *const u8, rhs: *const u8, max: usize) #[target_feature(enable = "sse4.2")] #[inline] pub(crate) unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize { + // Leading scalar word probe (mirrors upstream C `ZSTD_count`'s first + // `MEM_readST` check): a prefix that diverges within the first 8 bytes — the + // common case on BT-tree node compares, where each node extends the seed by + // only a short run — returns on one 8-byte read + `tzcnt`, skipping the + // vector load. Longer matches fall through to the vector loop. + let chunk = core::mem::size_of::(); + if chunk <= max { + let lhs_word = unsafe { core::ptr::read_unaligned(lhs.cast::()) }; + let rhs_word = unsafe { core::ptr::read_unaligned(rhs.cast::()) }; + let diff = lhs_word ^ rhs_word; + if diff != 0 { + return scalar::mismatch_byte_index(diff); + } + } let off = unsafe { prefix_len_simd(lhs, rhs, max) }; unsafe { scalar::common_prefix_len_scalar_ptr(lhs, rhs, off, max) } } diff --git a/zstd/src/encoding/hc/mod.rs b/zstd/src/encoding/hc/mod.rs index 43797be93..588b75477 100644 --- a/zstd/src/encoding/hc/mod.rs +++ b/zstd/src/encoding/hc/mod.rs @@ -547,6 +547,19 @@ impl HcMatcher { // Hoist the dms chain-table base pointer (same Vec-header-reload removal // as the live walk above). let dms_chain_ptr: *const u32 = dms.chain_table.as_ptr(); + // Reachability floor, hoisted out of the loop. A dict position is + // decoder-reachable iff its offset (`current_idx - dict_idx`) does not + // exceed `max_window_size`, i.e. `dict_idx >= current_idx - + // max_window_size`. Comparing `dict_idx` against this precomputed floor + // drops the per-iteration `current_idx - dict_idx` subtract from the hot + // path (the offset is only needed when a candidate WINS, which is rare on + // the walk) and frees the `max_window_size` register inside the loop. + // `saturating_sub` is the intended business logic, not an overflow guard: + // when `current_idx <= max_window_size` the whole live history fits the + // window, so every dict position is reachable and the floor is 0. Mirrors + // C's `ZSTD_HcFindBestMatch` dms loop, which bounds reachability by chain + // construction rather than re-checking each candidate. + let dict_reachable_floor = current_idx.saturating_sub(table.max_window_size); while steps < dms_budget { if dcur == 0 { break; @@ -558,9 +571,8 @@ impl HcMatcher { // SAFETY: `dict_idx` is a stored dict position `< chain_table.len()`. let dnext = unsafe { *dms_chain_ptr.add(dict_idx) }; steps += 1; - let new_offset = current_idx - dict_idx; // Out-of-window dict positions are unreachable to the decoder. - if new_offset <= table.max_window_size { + if dict_idx >= dict_reachable_floor { // Same self-tightening gate as the live walk; the caller's guard // (`current_idx + ml < history_tail`) + the iLimit break keep // `current_idx + (ml - 3) + 4 <= history_tail`, and