diff --git a/Cargo.lock b/Cargo.lock index cd3b7da..3ecc698 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,7 @@ dependencies = [ "cipher", "cpubits", "cpufeatures", + "zeroize", ] [[package]] @@ -530,6 +531,7 @@ dependencies = [ "bzip2", "cap-fs-ext", "cap-std", + "cbc", "compression-codecs", "ctr", "flate2", @@ -852,12 +854,14 @@ dependencies = [ "bzip2", "cbc", "crc32fast", + "flate2", "getrandom", "js-sys", "lzma-rust2", "ppmd-rust", "sha2", "wasm-bindgen", + "zstd", ] [[package]] diff --git a/docs/adr/0012-codec-capability-contract.md b/docs/adr/0012-codec-capability-contract.md index 304fbf2..172db06 100644 --- a/docs/adr/0012-codec-capability-contract.md +++ b/docs/adr/0012-codec-capability-contract.md @@ -81,10 +81,14 @@ gap must not be mistaken for discharging the obligation to close it. |---|---|---|---|---| | Portable **streaming** zstd **encode** | ZIP member write method 93 is `native-codecs`-only; portable selection returns a structured `Unsupported` error | `ruzstd` ships only a one-shot whole-buffer encoder (`ruzstd::encoding::compress_to_vec`, used for outer-filter *frames* and `create --zstd`); it cannot emit a single ZIP member as a bounded stream without buffering the whole member, which would break the core bounded-memory guarantee. The `native-codecs` `compression-codecs` encoder is a true streaming encoder. | A streaming, single-stream, spec-robust pure-Rust zstd encoder — contributed upstream to `ruzstd` or provided as a dedicated crate the engine consumes. The engine core does not absorb the encoder. | RM-307 (this ADR) → follow-on codec initiative | | Deflate64 read and write | ZIP method 9 returns a structured `Unsupported` error and still enumerates | No pure-Rust Deflate64 encoder exists; decoders are scarce and the write direction has effectively no consumer demand. | Feasibility decision (own pure-Rust decoder for read-only, external decoder, or leave unsupported); no encoder is planned. | RM-306 feasibility ADR | +| 7z **PPMd** decode (method `03 04 01`) | A 7z folder coded with PPMd lists normally; extraction returns a structured `Unsupported` error and enumeration continues | 7z's PPMd7 (variant H) has no wired bounded-memory pure-Rust decoder, and the engine core will not absorb the model. This is a *new decoder deferred*, not a bent guarantee — the coder graph parses, the capability is typed. | Adopt or contribute a pure-Rust PPMd7 decoder consumed behind the codec-provider boundary; read-only, no encoder planned. | RM-303 → follow-on codec initiative | +| 7z **BCJ2** decode (method `03 03 01 1B`) | A 7z coder graph containing BCJ2 lists normally; extraction returns a structured `Unsupported` error and enumeration continues | BCJ2 is a four-input, multi-stream branch converter that does not fit the one-active-linear-folder decode model that keeps 7z decoding bounded. This is a *multi-stream decode deferred* — the graph resolver already validates its bind pairs; only the decode stage is absent. | Extend the folder decoder with a bounded four-stream BCJ2 junction stage; read-only, no encoder planned. | RM-303 → follow-on decoder slice | -Every other mainstream codec used by the engine (deflate, bzip2, xz/LZMA, lz4) -has complete pure-Rust read **and** write on the `portable-codecs` profile, so -these two are the entire ledger, not a pervasive condition. +Every mainstream compression codec used by the engine (deflate, bzip2, xz/LZMA, +lz4) has complete pure-Rust read **and** write on the `portable-codecs` profile. +The two 7z-only entries (PPMd, BCJ2) are read-only-deferred coders behind an +already-typed capability, not bent guarantees; this table is the entire ledger, +not a pervasive condition. ## Consequences diff --git a/docs/support-matrix.md b/docs/support-matrix.md index ce90285..a288d98 100644 --- a/docs/support-matrix.md +++ b/docs/support-matrix.md @@ -12,7 +12,7 @@ scheme, metadata field, or producer quirk is accepted. | cpio | sequential | binary little/big endian, odc, newc, crc | yes | none | known-size entry creation | | ar | sequential | GNU and BSD | yes | none | thin members are reported as external references and are never materialized automatically | | ZIP/ZIP64 | seek or streaming | see [ZIP compression methods](#zip-compression-methods) grid | see grid | optional WinZip AES-256 AE-2; ZipCrypto not enabled by default | descriptors, ZIP64, Unicode path/comment, and extended/NTFS/Info-ZIP-UX timestamp extras are interpreted as typed metadata; Info-ZIP Unix uid/gid are surfaced when recorded in the central directory (as arca writes them) and synthesized on write; unknown extras are preserved verbatim | -| 7z | seek | LZMA/LZMA2, encoded headers, solid single-folder archives | yes | none (AES unsupported) | optional `sevenz`; multiple folders and general coder graphs are unsupported | +| 7z | seek | LZMA, LZMA2, Delta, BCJ (x86/PPC/IA64/ARM/ARMT/SPARC/ARM64/RISC-V), Deflate, BZip2, Zstandard; multi-folder and general coder graphs; plain and encoded headers | yes | AES-256 (SHA-256 KDF) | optional `sevenz`; one active folder decoder (bounded); PPMd and BCJ2 are structured `Unsupported` — see [deficits](#codec-capability-deficits) | | ISO 9660 | seek | ISO 9660, Rock Ridge, Joliet | yes | none | UDF and continuation-area coverage are not complete | | CAB | seek | read-only (MSCF): Store and MSZIP folders | no (read-only) | none | QUANTUM/LZX folders and cross-cabinet spanning are structured `Unsupported`; the MSZIP window is carried across a folder's `CFDATA` blocks | | XAR | seek | read-only: stored and zlib (`x-gzip`) data | no (read-only) | none | zlib-XML TOC; `x-bzip2` and other data encodings are structured `Unsupported` | @@ -46,8 +46,16 @@ resting state. Deflate64 (method 9) is not yet implemented in either direction; boundary, landing in a follow-on slice) and write is a retired won't-do (no pure-Rust encoder exists). Traditional ZipCrypto is not enabled by default. -7z BCJ/Delta, Deflate, BZip2, Zstandard, PPMd, AES, multi-folder, and arbitrary -coder-graph coverage remain roadmap work. +7z now reads the general coder graph: multi-folder archives, chained BCJ/Delta +filters over LZMA/LZMA2, and Deflate/BZip2/Zstandard coders, plus AES-256/SHA-256 +encryption as its own [container column](#archive-containers). Decoding keeps one +active folder decoder at a time (bounded memory) and verifies each substream's +stored CRC-32. The two remaining coders — PPMd and BCJ2 — list normally but return +a structured `Unsupported` on extraction; they are tracked deficits with declared +resolution paths (see [Codec capability deficits](#codec-capability-deficits)). +Malformed coder graphs (cycles, stream overlaps, truncation) are fuzzed through the +structured `read_7z_graph` target, which asserts no panic, bounded work, and only +typed errors. ### Codec capability deficits @@ -63,6 +71,8 @@ read+write on portable. |---|---|---|---|---| | Portable **streaming** zstd encode | ZIP write method 93 on `portable-codecs` → structured `Unsupported`; `native-codecs` write works | `ruzstd` ships only a one-shot whole-buffer encoder (`ruzstd::encoding::compress_to_vec`, used for outer-filter frames and `create --zstd`). It cannot emit a single ZIP member as a bounded stream without buffering the whole member, which would break the core bounded-memory guarantee. The engine refuses the path rather than weaken the guarantee. | A streaming, single-stream pure-Rust zstd encoder — upstream to `ruzstd` or a dedicated crate the engine consumes. | RM-307 → follow-on codec initiative | | Deflate64 (method 9) | ZIP method 9 read/write → structured `Unsupported` (not yet implemented) | Read: a mature pure-Rust decoder (`deflate64`) exists but is not yet wired. Write: no pure-Rust encoder exists and demand is effectively nil (matches libarchive). | Read: adopt the external `deflate64` decoder behind the codec-provider boundary in a follow-on slice. Write: **won't-do**, retired per [ADR-0013](adr/0013-rar5-udf-deflate64-feasibility.md). | RM-306 / ADR-0013 | +| 7z **PPMd** (method `03 04 01`) | a folder whose coder is PPMd lists normally; reaching its payload returns a structured `Unsupported` | 7z uses the PPMd variant H (PPMd7) model; no bounded-memory pure-Rust decoder is wired, and the engine will not absorb one into its core. | Adopt or contribute a pure-Rust PPMd7 decoder behind the codec-provider boundary, consumed like the other coders; no encoder is planned (read-only, matching libarchive). | RM-303 → follow-on codec initiative | +| 7z **BCJ2** (method `03 03 01 1B`) | a folder whose coder graph contains BCJ2 lists normally; reaching its payload returns a structured `Unsupported` | BCJ2 is a four-input branch converter (main + two call/jump streams + a range-coder control stream); it is inherently multi-stream and does not fit the one-active-linear-folder decode model that keeps memory bounded. | Extend the folder decoder with a bounded BCJ2 four-stream junction stage (the graph parser already resolves the bind pairs); no encoder is planned. | RM-303 → follow-on decoder slice | CAB and XAR are implemented as read-only seek-native providers (see the archive containers table above). RAR5 and UDF read scope is resolved by diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 2c7a814..4506df8 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -70,6 +70,13 @@ test = false doc = false bench = false +[[bin]] +name = "read_7z_graph" +path = "fuzz_targets/read_7z_graph.rs" +test = false +doc = false +bench = false + [[bin]] name = "read_iso" path = "fuzz_targets/read_iso.rs" diff --git a/fuzz/corpus/read_7z_graph/arca_solid.7z b/fuzz/corpus/read_7z_graph/arca_solid.7z new file mode 100644 index 0000000..7a99693 Binary files /dev/null and b/fuzz/corpus/read_7z_graph/arca_solid.7z differ diff --git a/fuzz/corpus/read_7z_graph/bcj_lzma2.7z b/fuzz/corpus/read_7z_graph/bcj_lzma2.7z new file mode 100644 index 0000000..b4b4d26 Binary files /dev/null and b/fuzz/corpus/read_7z_graph/bcj_lzma2.7z differ diff --git a/fuzz/corpus/read_7z_graph/deflate_folder.7z b/fuzz/corpus/read_7z_graph/deflate_folder.7z new file mode 100644 index 0000000..b66decc Binary files /dev/null and b/fuzz/corpus/read_7z_graph/deflate_folder.7z differ diff --git a/fuzz/corpus/read_7z_graph/delta_lzma2.7z b/fuzz/corpus/read_7z_graph/delta_lzma2.7z new file mode 100644 index 0000000..879cd16 Binary files /dev/null and b/fuzz/corpus/read_7z_graph/delta_lzma2.7z differ diff --git a/fuzz/corpus/read_7z_graph/lzma_folder.7z b/fuzz/corpus/read_7z_graph/lzma_folder.7z new file mode 100644 index 0000000..c1feda1 Binary files /dev/null and b/fuzz/corpus/read_7z_graph/lzma_folder.7z differ diff --git a/fuzz/corpus/read_7z_graph/multi_folder.7z b/fuzz/corpus/read_7z_graph/multi_folder.7z new file mode 100644 index 0000000..28aa57b Binary files /dev/null and b/fuzz/corpus/read_7z_graph/multi_folder.7z differ diff --git a/fuzz/fuzz_lib/src/lib.rs b/fuzz/fuzz_lib/src/lib.rs index 3682800..de1607f 100644 --- a/fuzz/fuzz_lib/src/lib.rs +++ b/fuzz/fuzz_lib/src/lib.rs @@ -10,12 +10,15 @@ use std::collections::{BTreeMap, BTreeSet}; use std::io::{Cursor, Read, Write}; use arbitrary::Arbitrary; +use libarchive_oxide::filter::crc32; use libarchive_oxide::filter::gzip::{GzipDecoder, GzipEncoder}; use libarchive_oxide::{ ArchiveReader, ArchiveWriter, FilterReader, ReaderEvent, SeekArchiveReader, SeekArchiveWriter, + StreamError, }; use libarchive_oxide_core::{ - ArchivePath, Codec, CodecStatus, EndOfInput, EntryKind, EntryMetadata, FormatId, Limits, + ArchiveError, ArchivePath, Codec, CodecStatus, EndOfInput, EntryKind, EntryMetadata, ErrorKind, + FormatId, Limits, }; const MAX_ENTRIES: usize = 200_000; @@ -425,6 +428,397 @@ pub fn codec_lzma2(data: &[u8]) { } } +// ════════════════════════════════════════════════════════════════════════════════════════════════ +// Structured 7z coder-graph fuzz (read_7z_graph) +// ════════════════════════════════════════════════════════════════════════════════════════════════ +// +// Rather than mutate raw bytes, this target *synthesizes* a 7z `StreamsInfo` from an `arbitrary` +// spec — number of folders, per-coder method / arity / properties, bind pairs, packed indices, and +// coder/substream sizes — then frames it in a valid signature header (correct start- and +// next-header CRCs) so the reader's checksum gate is passed and the coder-graph parser is reached on +// every input. It then drives the seek reader and asserts the RM-303 invariant: no panic, bounded +// work, and every failure is a *typed* archive error (`Malformed` / `Unsupported` / `Integrity` / +// `Limit`) — never an untyped I/O error, a wrong answer, or an unbounded allocation. Cycles, stream +// overlaps, and truncation (via the `truncate` knob, recomputed under the header CRC) are the +// stressors the graph resolver and length arithmetic must survive. + +/// Upper bounds keeping a synthesized archive small regardless of input size. +const GRAPH_MAX_FOLDERS: usize = 6; +const GRAPH_MAX_CODERS: usize = 6; +const GRAPH_MAX_ARITY: usize = 6; +const GRAPH_MAX_PROPS: usize = 8; +const GRAPH_MAX_UNPACK: usize = 24; +const GRAPH_MAX_SUBSTREAMS: usize = 6; +const GRAPH_MAX_PACKDATA: usize = 4096; + +/// 7z signature magic (`'7' 'z' 0xBC 0xAF 0x27 0x1C`). +const GRAPH_SIGNATURE: [u8; 6] = [0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C]; + +// 7z next-header property ids used by the synthesizer. +const G_K_END: u8 = 0x00; +const G_K_HEADER: u8 = 0x01; +const G_K_MAIN_STREAMS_INFO: u8 = 0x04; +const G_K_FILES_INFO: u8 = 0x05; +const G_K_PACK_INFO: u8 = 0x06; +const G_K_UNPACK_INFO: u8 = 0x07; +const G_K_SUBSTREAMS_INFO: u8 = 0x08; +const G_K_SIZE: u8 = 0x09; +const G_K_CRC: u8 = 0x0A; +const G_K_FOLDER: u8 = 0x0B; +const G_K_CODERS_UNPACK_SIZE: u8 = 0x0C; +const G_K_NUM_UNPACK_STREAM: u8 = 0x0D; +const G_K_NAME: u8 = 0x11; + +/// Candidate coder method ids: decodable coders, filters, and the two deferred/unsupported ids +/// (PPMd, BCJ2) plus an unknown id — so the synthesizer builds graphs arca decodes *and* graphs it +/// must type as `Unsupported`. +const GRAPH_METHOD_IDS: &[&[u8]] = &[ + &[0x21], // LZMA2 + &[0x03, 0x01, 0x01], // LZMA + &[0x03], // Delta filter + &[0x03, 0x03, 0x01, 0x03], // BCJ x86 + &[0x0A], // BCJ ARM64 + &[0x04, 0x01, 0x08], // Deflate (raw) + &[0x04, 0x02, 0x02], // BZip2 + &[0x04, 0xF7, 0x11, 0x01], // Zstd + &[0x06, 0xF1, 0x07, 0x01], // AES-256 + &[0x03, 0x04, 0x01], // PPMd (deferred → Unsupported) + &[0x03, 0x03, 0x01, 0x1B], // BCJ2 (multi-stream, deferred → Unsupported) + &[0x00], // Copy / unknown +]; + +/// One coder in a synthesized folder. +#[derive(Debug, Arbitrary)] +struct GraphCoder { + method: u8, + complex: bool, + alt: u8, + num_in: u8, + num_out: u8, + props: Vec, +} + +/// One synthesized folder: its coders plus the bind-pair / packed-stream index pools the encoder +/// draws from (cycled to the arity the emitted coders imply). +#[derive(Debug, Arbitrary)] +struct GraphFolder { + coders: Vec, + bind_in: Vec, + bind_out: Vec, + packed: Vec, +} + +/// A full `StreamsInfo` skeleton. +#[derive(Debug, Arbitrary)] +struct GraphSpec { + pack_pos: u8, + pack_sizes: Vec, + folders: Vec, + unpack_sizes: Vec, + include_folder_crc: bool, + include_substreams: bool, + num_unpack_streams: Vec, + substream_sizes: Vec, + include_substream_crc: bool, + include_files_info: bool, + truncate: u16, + pack_data: Vec, +} + +/// Writes a 7z variable-length number (`WriteNumber`), mirroring the reader's `ReadNumber`. +#[allow(clippy::cast_possible_truncation)] +fn graph_number(out: &mut Vec, value: u64) { + let mut first = 0u8; + let mut mask = 0x80u8; + let mut i = 0u32; + while i < 8 { + if value < (1u64 << (7 * (i + 1))) { + first |= (value >> (8 * i)) as u8; + break; + } + first |= mask; + mask >>= 1; + i += 1; + } + out.push(first); + let mut v = value; + for _ in 0..i { + out.push(v as u8); + v >>= 8; + } +} + +impl GraphCoder { + /// Emits this coder and returns the `(num_in, num_out)` arca will parse from the emitted bytes. + fn emit(&self, out: &mut Vec) -> (usize, usize) { + let id = GRAPH_METHOD_IDS[self.method as usize % GRAPH_METHOD_IDS.len()]; + let id_size = id.len() as u8; // every candidate id is <= 4 bytes + let has_props = !self.props.is_empty(); + let mut flags = id_size & 0x0F; + if self.complex { + flags |= 0x10; + } + if has_props { + flags |= 0x20; + } + // Rarely set the "alternative methods" bit (~6%), which the reader must type as Unsupported. + if self.alt < 16 { + flags |= 0x80; + } + out.push(flags); + out.extend_from_slice(id); + let (num_in, num_out) = if self.complex { + let ni = (self.num_in as usize % GRAPH_MAX_ARITY) + 1; + let no = (self.num_out as usize % GRAPH_MAX_ARITY) + 1; + graph_number(out, ni as u64); + graph_number(out, no as u64); + (ni, no) + } else { + (1, 1) + }; + if has_props { + let props: Vec = self.props.iter().copied().take(GRAPH_MAX_PROPS).collect(); + graph_number(out, props.len() as u64); + out.extend_from_slice(&props); + } + (num_in, num_out) + } +} + +impl GraphSpec { + /// Serializes the spec into a framed 7z archive image. + #[allow(clippy::cast_possible_truncation)] + fn encode(&self) -> Vec { + let folders: Vec<&GraphFolder> = self.folders.iter().take(GRAPH_MAX_FOLDERS).collect(); + + let mut streams = Vec::new(); + // ── PackInfo ── + streams.push(G_K_PACK_INFO); + graph_number(&mut streams, u64::from(self.pack_pos)); + let pack_sizes: Vec = self + .pack_sizes + .iter() + .take(GRAPH_MAX_UNPACK) + .map(|&s| u64::from(s)) + .collect(); + graph_number(&mut streams, pack_sizes.len() as u64); + streams.push(G_K_SIZE); + for &size in &pack_sizes { + graph_number(&mut streams, size); + } + streams.push(G_K_END); + + // ── UnpackInfo ── + streams.push(G_K_UNPACK_INFO); + streams.push(G_K_FOLDER); + graph_number(&mut streams, folders.len() as u64); + streams.push(0x00); // folder definitions are inline (external == 0) + let mut total_outputs = 0usize; + for folder in &folders { + let coders: Vec<&GraphCoder> = folder.coders.iter().take(GRAPH_MAX_CODERS).collect(); + let num_coders = coders.len().max(1); + graph_number(&mut streams, num_coders as u64); + let mut total_in = 0usize; + let mut total_out = 0usize; + for i in 0..num_coders { + // At least one coder even if the pool is empty (an LZMA2 stand-in). + let (ni, no) = match coders.get(i) { + Some(coder) => coder.emit(&mut streams), + None => { + streams.push(0x01); // id_size 1, simple coder + streams.push(0x21); // LZMA2 + (1, 1) + }, + }; + total_in += ni; + total_out += no; + } + total_outputs += total_out; + // (numOut - 1) bind pairs, drawn from the folder's index pools (cycled), kept roughly + // in range so some resolve cleanly and some overlap / cycle. + let num_bind = total_out.saturating_sub(1); + for k in 0..num_bind { + let in_index = pool_index(&folder.bind_in, k, total_in); + let out_index = pool_index(&folder.bind_out, k, total_out); + graph_number(&mut streams, in_index as u64); + graph_number(&mut streams, out_index as u64); + } + // Packed inputs = total_in - num_bind; >1 are listed explicitly, 1 is implicit. + let num_packed = total_in.saturating_sub(num_bind); + if num_packed > 1 { + for k in 0..num_packed { + let idx = pool_index(&folder.packed, k, total_in.max(1)); + graph_number(&mut streams, idx as u64); + } + } + } + streams.push(G_K_CODERS_UNPACK_SIZE); + for i in 0..total_outputs { + let size = self + .unpack_sizes + .get(i % self.unpack_sizes.len().max(1)) + .map_or(0, |&s| u64::from(s)); + graph_number(&mut streams, size); + } + if self.include_folder_crc { + streams.push(G_K_CRC); + streams.push(0x00); // not all defined + let bytes = folders.len().div_ceil(8); + streams.extend(std::iter::repeat_n(0u8, bytes)); // none defined + } + streams.push(G_K_END); + + // ── SubStreamsInfo (optional) ── + if self.include_substreams && !folders.is_empty() { + streams.push(G_K_SUBSTREAMS_INFO); + let counts: Vec = (0..folders.len()) + .map(|f| { + self.num_unpack_streams + .get(f % self.num_unpack_streams.len().max(1)) + .map_or(1, |&n| usize::from(n) % GRAPH_MAX_SUBSTREAMS) + }) + .collect(); + streams.push(G_K_NUM_UNPACK_STREAM); + for &c in &counts { + graph_number(&mut streams, c as u64); + } + streams.push(G_K_SIZE); + let mut si = 0usize; + for &c in &counts { + for _ in 0..c.saturating_sub(1) { + let size = self + .substream_sizes + .get(si % self.substream_sizes.len().max(1)) + .map_or(0, |&s| u64::from(s)); + graph_number(&mut streams, size); + si += 1; + } + } + if self.include_substream_crc { + streams.push(G_K_CRC); + let total: usize = counts.iter().sum(); + streams.push(0x01); // all defined + for _ in 0..total { + // Deliberately (almost certainly) wrong CRCs: a reached decode must surface a + // typed `Integrity` error, never a silent wrong answer. + streams.extend_from_slice(&0xDEAD_BEEFu32.to_le_bytes()); + } + } + streams.push(G_K_END); + } + streams.push(G_K_END); // end StreamsInfo + + // ── Assemble the plain kHeader body ── + let mut next = vec![G_K_HEADER, G_K_MAIN_STREAMS_INFO]; + next.extend_from_slice(&streams); + // Optional minimal FilesInfo (empty names) so a consistent graph reaches the decode path. + if self.include_files_info && !folders.is_empty() { + next.push(G_K_FILES_INFO); + let num_files = folders.len(); + graph_number(&mut next, num_files as u64); + graph_number(&mut next, u64::from(G_K_NAME)); + let mut name_body = vec![0x00u8]; // names are inline (external == 0) + for _ in 0..num_files { + name_body.extend_from_slice(&[0x00, 0x00]); // empty UTF-16 name (terminator only) + } + graph_number(&mut next, name_body.len() as u64); + next.extend_from_slice(&name_body); + graph_number(&mut next, u64::from(G_K_END)); + } + next.push(G_K_END); // end kHeader + + // Truncation knob: cut the header short *before* computing its CRC, so the outer checksum + // gate still passes and the reader hits the truncation inside the graph parser. + if self.truncate != 0 && !next.is_empty() { + let keep = (self.truncate as usize) % next.len(); + next.truncate(keep); + } + + // ── Frame it: signature header + pack region + next header ── + let pack: Vec = self + .pack_data + .iter() + .copied() + .take(GRAPH_MAX_PACKDATA) + .collect(); + let mut file = vec![0u8; 32]; + file[..6].copy_from_slice(&GRAPH_SIGNATURE); + file[6] = 0; // format version major + file[7] = 4; // format version minor + file.extend_from_slice(&pack); + let header_offset = pack.len() as u64; + let header_size = next.len() as u64; + let header_crc = crc32(&next); + file.extend_from_slice(&next); + file[12..20].copy_from_slice(&header_offset.to_le_bytes()); + file[20..28].copy_from_slice(&header_size.to_le_bytes()); + file[28..32].copy_from_slice(&header_crc.to_le_bytes()); + let start_crc = crc32(&file[12..32]); + file[8..12].copy_from_slice(&start_crc.to_le_bytes()); + file + } +} + +/// Picks an index from a pool (cycled), kept within `[0, modulus)` so it lands near the real stream +/// range — some in-range (resolvable), some colliding (overlap/cycle the resolver must reject). +fn pool_index(pool: &[u8], k: usize, modulus: usize) -> usize { + let modulus = modulus.max(1); + match pool.get(k % pool.len().max(1)) { + Some(&raw) => usize::from(raw) % modulus, + None => k % modulus, + } +} + +/// Asserts a 7z read failure is a *typed* archive error of an expected kind — the RM-303 contract +/// that a hostile coder graph never panics, never leaks an untyped error, and never silently lies. +fn assert_graph_typed(error: &StreamError) { + match error.archive_error().map(ArchiveError::kind) { + Some( + ErrorKind::Malformed | ErrorKind::Unsupported | ErrorKind::Integrity | ErrorKind::Limit, + ) => {}, + other => panic!("7z coder-graph fuzz surfaced a non-typed / unexpected error: {other:?}"), + } +} + +/// Structured 7z coder-graph target: synthesize a `StreamsInfo`, frame it, and drive the reader. +pub fn read_7z_graph(data: &[u8]) { + let mut input = arbitrary::Unstructured::new(data); + let Ok(spec) = GraphSpec::arbitrary(&mut input) else { + return; + }; + let archive = spec.encode(); + let mut reader = match SeekArchiveReader::with_limits(Cursor::new(archive), fuzz_limits()) { + Ok(reader) => reader, + Err(error) => { + assert_graph_typed(&error); + return; + }, + }; + let mut events = 0usize; + let mut payload = 0u64; + loop { + match reader.next_event() { + Ok(ReaderEvent::Entry(metadata)) => { + let _ = metadata.path().as_bytes(); + events = events.saturating_add(1); + }, + Ok(ReaderEvent::Data(bytes)) => { + payload = payload.saturating_add(bytes.len() as u64); + }, + Ok(ReaderEvent::Done) => return, + Ok(ReaderEvent::ArchiveMetadata(_) | ReaderEvent::EndEntry) => {}, + Ok(_) => return, + Err(error) => { + assert_graph_typed(&error); + return; + }, + } + if events > MAX_ENTRIES || payload > MAX_TOTAL_BYTES { + return; + } + } +} + /// All portable fuzz targets. pub const TARGETS: &[&str] = &[ "read_tar", @@ -432,6 +826,7 @@ pub const TARGETS: &[&str] = &[ "read_ar", "read_zip", "read_7z", + "read_7z_graph", "read_iso", "roundtrip_tar", "roundtrip_cpio", @@ -454,6 +849,7 @@ pub fn run_target(name: &str, data: &[u8]) { "read_ar" => read_ar(data), "read_zip" => read_zip(data), "read_7z" => read_7z(data), + "read_7z_graph" => read_7z_graph(data), "read_iso" => read_iso(data), "roundtrip_tar" => roundtrip_tar(&entries_from_bytes(data)), "roundtrip_cpio" => roundtrip_cpio(&entries_from_bytes(data)), diff --git a/fuzz/fuzz_targets/read_7z_graph.rs b/fuzz/fuzz_targets/read_7z_graph.rs new file mode 100644 index 0000000..267511b --- /dev/null +++ b/fuzz/fuzz_targets/read_7z_graph.rs @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: 2026 libarchive_oxide contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#![no_main] + +//! Thin libFuzzer shim → the portable `libarchive_oxide_fuzz_cases::read_7z_graph` invariant, which +//! synthesizes a random 7z `StreamsInfo` coder graph and asserts the reader never panics, stays +//! bounded, and only ever returns a typed error. +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + libarchive_oxide_fuzz_cases::read_7z_graph(data); +}); diff --git a/libarchive_oxide-core/src/filter.rs b/libarchive_oxide-core/src/filter.rs index c9d87e8..2f43f1b 100644 --- a/libarchive_oxide-core/src/filter.rs +++ b/libarchive_oxide-core/src/filter.rs @@ -20,6 +20,11 @@ pub enum FilterId { Lz4, /// Bzip2 stream. Bzip2, + /// Raw DEFLATE (RFC 1951) with no framing — the 7z Deflate coder stores bare + /// deflate blocks with no gzip header, trailer, or checksum. Has no stream + /// signature, so it never participates in [`FilterId::probe`]; it is only ever + /// selected structurally from a 7z folder's coder graph. + Deflate, } impl FilterId { diff --git a/libarchive_oxide/Cargo.toml b/libarchive_oxide/Cargo.toml index ee7d7ac..a7ef1f8 100644 --- a/libarchive_oxide/Cargo.toml +++ b/libarchive_oxide/Cargo.toml @@ -56,8 +56,8 @@ bzip2 = [ zstd = ["dep:ruzstd"] xz = ["dep:lzma-rust2"] lz4 = ["dep:lz4_flex", "dep:twox-hash"] -# WinZip AES-256 AE-2. -aes = ["dep:aes", "dep:ctr", "dep:hmac", "dep:sha1", "dep:pbkdf2", "dep:subtle", "dep:getrandom"] +# WinZip AES-256 AE-2 (ZIP) and AES-256/SHA-256 (7z, via `cbc`). +aes = ["dep:aes", "dep:cbc", "dep:ctr", "dep:hmac", "dep:sha1", "dep:pbkdf2", "dep:subtle", "dep:getrandom"] # 7z single-folder LZMA2 and shared CRC-32. sevenz = ["dep:lzma-rust2", "gzip"] # Runtime-neutral and Tokio I/O adapters. Both drive the same sans-I/O @@ -83,8 +83,12 @@ flate2 = { version = "1.1.9", default-features = false, features = ["zlib"], opt xz-codec = { package = "xz2", version = "0.1.7", optional = true } # Shared by xz and 7z. lzma-rust2 = { version = "0.16.5", default-features = false, features = ["std", "encoder", "xz"], optional = true } -# WinZip AE-2 requires SHA-1. -aes = { version = "0.9", optional = true } +# WinZip AE-2 requires SHA-1. `zeroize` wipes the expanded AES key schedule on drop, +# so a folder/member key never lingers in freed memory after the decoder is dropped +# (defense in depth for both the 7z CBC and the ZIP AE-2 CTR paths). +aes = { version = "0.9", features = ["zeroize"], optional = true } +# 7z AES-256/SHA-256 uses CBC (ZIP AE-2 uses CTR via `ctr`). +cbc = { version = "0.2", optional = true } ctr = { version = "0.10", optional = true } hmac = { version = "0.13", optional = true } sha1 = { version = "0.11", optional = true } @@ -120,7 +124,9 @@ ruzstd = "0.8.2" zstd-codec = { package = "zstd", version = "0.13.3" } # Independent SHA-256 reference for OCI layer digest/diffID tests. sha2 = "0.11" -sevenz-rust2 = "0.21.3" +# `bzip2` is on by default; `deflate` and `zstd` are enabled so the 7z differential +# tests can produce Deflate/Zstd folders for arca's reuse-codec decode path. +sevenz-rust2 = { version = "0.21.3", features = ["deflate", "zstd"] } zip = { version = "8.6.0", default-features = false, features = ["deflate", "aes-crypto", "bzip2", "zstd", "lzma"] } # Independent tar/ar producers and consumers used only for RM-304 interop tests. tar = { version = "0.4", default-features = false } diff --git a/libarchive_oxide/src/codec_read.rs b/libarchive_oxide/src/codec_read.rs new file mode 100644 index 0000000..0c9ebed --- /dev/null +++ b/libarchive_oxide/src/codec_read.rs @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: 2026 libarchive_oxide contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Generic bounded `Read` adapter over any sans-I/O [`Codec`]. +//! +//! [`CodecReader`] drives a caller-supplied [`Codec`] against an inner byte +//! source, retaining neither the compressed input nor the decoded output beyond +//! one fixed-size staging buffer. It is the shared engine behind the outer +//! filter readers (zstd, LZ4, …) and any future coder that speaks the sans-I/O +//! [`Codec`] protocol. + +use std::io::{self, Read}; + +use libarchive_oxide_core::{ArchiveError, Codec, CodecStatus, EndOfInput, ErrorKind}; + +/// Staging-buffer size for the compressed side of a [`CodecReader`]. +pub(crate) const BUFFER: usize = 64 * 1024; + +/// A bounded streaming reader that decodes `input` through the sans-I/O `decoder`. +/// +/// Generic over the inner reader `Inner` and the codec `C`, so a single +/// implementation serves every codec that implements [`Codec`]. `name` labels +/// the codec in error messages. +pub(crate) struct CodecReader { + input: Inner, + decoder: C, + name: &'static str, + buffer: Vec, + start: usize, + end: usize, + eof: bool, + done: bool, + failed: bool, +} + +impl CodecReader { + /// Wraps `input`, decoding it through `decoder`. `name` labels the codec in errors. + pub(crate) fn new(input: Inner, decoder: C, name: &'static str) -> Self { + Self { + input, + decoder, + name, + buffer: vec![0; BUFFER], + start: 0, + end: 0, + eof: false, + done: false, + failed: false, + } + } + + /// Reclaims the inner reader at its current physical position. + pub(crate) fn into_inner(self) -> Inner { + self.input + } + + /// Borrows the inner reader (used to reach the raw source below a chain). + #[cfg(feature = "sevenz")] + pub(crate) fn get_ref(&self) -> &Inner { + &self.input + } + + fn fill(&mut self) -> io::Result<()> { + if self.start != 0 { + self.buffer.copy_within(self.start..self.end, 0); + self.end -= self.start; + self.start = 0; + } + if self.end == self.buffer.len() || self.eof { + return Ok(()); + } + let read = self.input.read(&mut self.buffer[self.end..])?; + if read == 0 { + self.eof = true; + } else { + self.end += read; + } + Ok(()) + } + + fn fail(&mut self, error: io::Error) -> io::Result { + self.failed = true; + Err(error) + } +} + +impl Read for CodecReader { + fn read(&mut self, output: &mut [u8]) -> io::Result { + if output.is_empty() || self.done { + return Ok(0); + } + if self.failed { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{} reader is in a failed state", self.name), + )); + } + loop { + if self.start == self.end && !self.eof { + self.fill()?; + } + let input_length = self.end - self.start; + let end = if self.eof { + EndOfInput::End + } else { + EndOfInput::More + }; + let step = match self + .decoder + .process(&self.buffer[self.start..self.end], output, end) + .and_then(|step| step.validate(input_length, output.len())) + { + Ok(step) => step, + Err(error) => return self.fail(codec_archive_io(error)), + }; + self.start += step.consumed; + if step.produced != 0 { + return Ok(step.produced); + } + match step.status { + CodecStatus::Done => { + self.done = true; + return Ok(0); + }, + CodecStatus::NeedInput if self.start == self.end && !self.eof => { + self.fill()?; + }, + CodecStatus::NeedOutput if step.consumed != 0 => {}, + CodecStatus::NeedInput if self.eof => { + return self.fail(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{} decoder requested input after the source ended", + self.name + ), + )); + }, + _ => { + return self.fail(io::Error::new( + io::ErrorKind::InvalidData, + format!("{} reader made no progress", self.name), + )); + }, + } + } + } +} + +/// Maps a codec [`ArchiveError`] onto an [`io::Error`], preserving the failure class. +pub(crate) fn codec_archive_io(error: ArchiveError) -> io::Error { + let kind = match error.kind() { + ErrorKind::Limit => io::ErrorKind::OutOfMemory, + ErrorKind::Unsupported => io::ErrorKind::Unsupported, + _ => io::ErrorKind::InvalidData, + }; + io::Error::new(kind, error) +} diff --git a/libarchive_oxide/src/filter/aes7z.rs b/libarchive_oxide/src/filter/aes7z.rs new file mode 100644 index 0000000..4caac09 --- /dev/null +++ b/libarchive_oxide/src/filter/aes7z.rs @@ -0,0 +1,368 @@ +// SPDX-FileCopyrightText: 2026 libarchive_oxide contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! 7z AES-256/SHA-256 decryption coder (method id `06 F1 07 01`) as a sans-I/O [`Codec`]. +//! +//! This is the 7-Zip encryption coder, distinct from the ZIP `WinZip` `AE-2` path +//! (`seek_stream::ZipAesDecoder`, AES-256-CTR + PBKDF2-HMAC-SHA1). The two differ +//! in three ways that matter for interoperability: +//! +//! * **Cipher mode.** 7z uses AES-256 in **CBC** with a stored IV; ZIP AE-2 uses +//! AES-256 in CTR. +//! * **Key derivation.** 7z runs a single SHA-256 context fed +//! `salt || password || counter` `2^numCyclesPower` times (7-Zip's own KDF); ZIP +//! uses PBKDF2-HMAC-SHA1 with 1000 iterations. +//! * **Password encoding.** 7z hashes the password as **UTF-16LE** code units; +//! ZIP AE-2 hashes the raw password bytes. The same user string therefore yields +//! different keys in the two formats. +//! +//! The decoder decrypts whole 16-byte blocks and buffers a partial input block +//! and a partial output block internally, so it composes with any input chunking +//! from the coder below it. The 7z encoder zero-pads the ciphertext up to a block +//! boundary; the coder's declared output size (`out_size`) caps how many decrypted +//! bytes are emitted, so that trailing padding is truncated rather than reported as +//! an error. The per-substream CRC-32 (verified by the folder reader) is what +//! ultimately distinguishes a correct password from a wrong one. + +use aes::cipher::{BlockModeDecrypt, KeyIvInit, array::Array}; +use libarchive_oxide_core::{ArchiveError, Codec, CodecStatus, CodecStep, EndOfInput, ErrorKind}; +use sha2::Digest; +use zeroize::Zeroize; + +/// AES block size in bytes. +const BLOCK: usize = 16; + +/// Largest accepted key-derivation work factor. The KDF runs `2^numCyclesPower` +/// SHA-256 rounds, so an attacker-supplied large power is a CPU-exhaustion `DoS` (and +/// `1 << power` also overflows for `power >= 64`). 7-Zip's own encoder never exceeds +/// this bound; a larger value only ever comes from a hostile archive. Keeping it below +/// 32 also makes the `1u64 << ncp` shift trivially safe. Mirrors `sevenz-rust2`. +const MAX_CYCLES_POWER: u8 = 24; + +type Aes256CbcDec = cbc::Decryptor; + +/// Parsed AES256SHA256 coder properties. Salt and IV are archive-public (they are +/// stored in the clear in the header), so this carries no secret material and is safe +/// to `Debug`. The derived key never lives here. +#[derive(Debug, Clone, Copy)] +pub(crate) struct AesParams { + ncp: u8, + salt: [u8; BLOCK], + salt_len: usize, + iv: [u8; BLOCK], +} + +impl AesParams { + /// Parses the coder property bytes for method `06 F1 07 01`. + /// + /// Returns `None` (→ the folder lists but is not decodable) for an external-key + /// marker (`numCyclesPower == 0x3F`), a work factor above [`MAX_CYCLES_POWER`], or + /// any malformed / out-of-range salt/IV framing. The property layout is: + /// `b0` = `numCyclesPower | (saltHi << 7) | (ivHi << 6)`, `b1` = + /// `(saltLow << 4) | ivLow`, then `saltSize` salt bytes and `ivSize` IV bytes. + pub(crate) fn parse(props: &[u8]) -> Option { + let b0 = *props.first()?; + let ncp = b0 & 0x3F; + // 0x3F is 7-Zip's "key supplied externally" marker; unsupported here. + if ncp == 0x3F || ncp > MAX_CYCLES_POWER { + return None; + } + // A one-byte property (some archives store the kEnd byte as a lone property) is + // treated as a zero second byte, matching mainstream decoders. + let b1 = props.get(1).copied().unwrap_or(0); + let iv_size = usize::from(((b0 >> 6) & 1) + (b1 & 0x0F)); + let salt_size = usize::from(((b0 >> 7) & 1) + (b1 >> 4)); + if salt_size > BLOCK || iv_size > BLOCK { + return None; + } + let end = 2usize.checked_add(salt_size)?.checked_add(iv_size)?; + if end > props.len() { + return None; + } + let mut salt = [0u8; BLOCK]; + salt[..salt_size].copy_from_slice(&props[2..2 + salt_size]); + let mut iv = [0u8; BLOCK]; + iv[..iv_size].copy_from_slice(&props[2 + salt_size..end]); + Some(Self { + ncp, + salt, + salt_len: salt_size, + iv, + }) + } +} + +/// Encodes a password (interpreted as a UTF-8 string) as UTF-16LE code units, the +/// form 7-Zip hashes. Lossy for invalid UTF-8 (one U+FFFD per maximal invalid +/// subsequence), matching `String::from_utf8_lossy`. +/// +/// Uses [`slice::utf8_chunks`] rather than `from_utf8_lossy` so the password is never +/// copied into an intermediate owned `String`: `chunk.valid()` borrows directly from +/// the caller's buffer, so no un-zeroized heap copy of password-derived bytes is left +/// behind on the invalid-UTF-8 path. The returned buffer is the caller's to zeroize. +fn password_utf16le(password: &[u8]) -> Vec { + let mut out = Vec::with_capacity(password.len().saturating_mul(2)); + for chunk in password.utf8_chunks() { + for unit in chunk.valid().encode_utf16() { + out.extend_from_slice(&unit.to_le_bytes()); + } + if !chunk.invalid().is_empty() { + out.extend_from_slice(&(char::REPLACEMENT_CHARACTER as u16).to_le_bytes()); + } + } + out +} + +/// The 7-Zip AES key schedule: a single SHA-256 context fed `salt || password || +/// counter` `2^ncp` times, where `counter` is a little-endian 64-bit round index. +/// `ncp == 0` degenerates to one round over `salt || password || 0u64`. +fn derive_key(ncp: u8, salt: &[u8], password_utf16le: &[u8]) -> [u8; 32] { + let mut sha = sha2::Sha256::default(); + let mut counter = [0u8; 8]; + for _ in 0..(1u64 << ncp) { + sha.update(salt); + sha.update(password_utf16le); + sha.update(counter); + for byte in &mut counter { + *byte = byte.wrapping_add(1); + if *byte != 0 { + break; + } + } + } + sha.finalize().into() +} + +/// Streaming AES-256-CBC decryption coder for a 7z folder. Holds one partial input +/// block and one partial output block (32 bytes of buffering total), so it never +/// retains more than a block regardless of the chunking below it. +pub(crate) struct AesDecoder { + dec: Aes256CbcDec, + in_block: [u8; BLOCK], + in_len: usize, + out_block: [u8; BLOCK], + out_pos: usize, + out_len: usize, + /// Remaining decrypted bytes to emit (the coder's declared output size). Trailing + /// ciphertext padding beyond this is dropped rather than surfaced. + remaining: u64, +} + +impl AesDecoder { + /// Builds a decoder from parsed properties and a raw (UTF-8) password. `out_size` + /// is the coder's declared uncompressed output size, used to truncate padding. + pub(crate) fn new(params: AesParams, out_size: u64, password: &[u8]) -> Self { + let mut pw = password_utf16le(password); + let mut key = derive_key(params.ncp, ¶ms.salt[..params.salt_len], &pw); + pw.zeroize(); + let dec = Aes256CbcDec::new(&Array::from(key), &Array::from(params.iv)); + key.zeroize(); + Self { + dec, + in_block: [0; BLOCK], + in_len: 0, + out_block: [0; BLOCK], + out_pos: 0, + out_len: 0, + remaining: out_size, + } + } +} + +impl Codec for AesDecoder { + fn process( + &mut self, + input: &[u8], + output: &mut [u8], + end: EndOfInput, + ) -> Result { + let mut consumed = 0usize; + let mut produced = 0usize; + loop { + if self.remaining == 0 { + // Output budget reached: drop any buffered plaintext and padding. + return Ok(CodecStep { + consumed, + produced, + status: CodecStatus::Done, + }); + } + // 1. Drain already-decrypted output first. + if self.out_pos < self.out_len { + let avail = self.out_len - self.out_pos; + let room = output.len() - produced; + let budget = usize::try_from(self.remaining).unwrap_or(usize::MAX); + let take = avail.min(room).min(budget); + if take == 0 { + break; // output is full + } + output[produced..produced + take] + .copy_from_slice(&self.out_block[self.out_pos..self.out_pos + take]); + self.out_pos += take; + produced += take; + self.remaining -= take as u64; + continue; + } + // 2. Fill the pending input block, decrypting it once complete. + while self.in_len < BLOCK && consumed < input.len() { + self.in_block[self.in_len] = input[consumed]; + self.in_len += 1; + consumed += 1; + } + if self.in_len == BLOCK { + let block: &mut Array = + self.in_block.as_mut_slice().try_into().map_err(|_| { + ArchiveError::new(ErrorKind::Malformed) + .with_context("7z AES: block framing") + })?; + self.dec.decrypt_block(block); + self.out_block = self.in_block; + self.out_pos = 0; + self.out_len = BLOCK; + self.in_len = 0; + continue; + } + break; // partial input block; need more bytes + } + if produced != 0 || consumed != 0 { + return Ok(CodecStep { + consumed, + produced, + status: CodecStatus::NeedInput, + }); + } + // No progress: a partial trailing block at end-of-input is padding and dropped. + let status = match end { + EndOfInput::End => CodecStatus::Done, + EndOfInput::More => CodecStatus::NeedInput, + }; + Ok(CodecStep { + consumed: 0, + produced: 0, + status, + }) + } +} + +#[cfg(test)] +#[allow( + clippy::unwrap_used, + clippy::cast_possible_truncation, + clippy::trivially_copy_pass_by_ref +)] +mod tests { + use std::io::Read; + + use super::*; + use crate::codec_read::CodecReader; + + /// Builds coder properties matching what `sevenz-rust2` / 7-Zip write: + /// `b0 = ncp | 0xC0`, `b1 = 0xFF` (16-byte salt and IV), then salt, then IV. + fn props(ncp: u8, salt: &[u8; 16], iv: &[u8; 16]) -> Vec { + let mut p = vec![(ncp & 0x3F) | 0xC0, 0xFF]; + p.extend_from_slice(salt); + p.extend_from_slice(iv); + p + } + + /// Independent AES-256-CBC encryption of a padded plaintext, for a round trip. + fn encrypt(key: &[u8; 32], iv: &[u8; 16], plaintext: &[u8]) -> Vec { + use aes::cipher::BlockModeEncrypt; + type Enc = cbc::Encryptor; + let mut enc = Enc::new(&Array::from(*key), &Array::from(*iv)); + let mut padded = plaintext.to_vec(); + while !padded.len().is_multiple_of(16) { + padded.push(0); + } + for chunk in padded.chunks_mut(16) { + let block: &mut Array = chunk.try_into().unwrap(); + enc.encrypt_block(block); + } + padded + } + + #[test] + fn parses_standard_properties() { + let salt = [7u8; 16]; + let iv = [9u8; 16]; + let params = AesParams::parse(&props(8, &salt, &iv)).unwrap(); + assert_eq!(params.ncp, 8); + assert_eq!(params.salt_len, 16); + assert_eq!(¶ms.salt, &salt); + assert_eq!(¶ms.iv, &iv); + } + + #[test] + fn password_utf16le_matches_lossy_reference() { + // The `utf8_chunks`-based encoder (which avoids an un-zeroized password copy) + // must be byte-identical to `String::from_utf8_lossy(..).encode_utf16()`, + // including one U+FFFD per maximal invalid subsequence. + let cases: [&[u8]; 5] = [ + b"correct horse", + &[0x66, 0x6f, 0xff, 0x6f], // valid, one invalid byte, valid + &[0xff, 0xfe, 0xfd], // all invalid + &[0xf0, 0x28, 0x8c, 0x28], // invalid lead + stray continuations + &[], + ]; + for pw in cases { + let reference: Vec = String::from_utf8_lossy(pw) + .encode_utf16() + .flat_map(u16::to_le_bytes) + .collect(); + assert_eq!(password_utf16le(pw), reference, "pw = {pw:02x?}"); + } + } + + #[test] + fn rejects_external_and_overlong_and_truncated() { + // External-key marker. + assert!(AesParams::parse(&[0x3F | 0xC0, 0xFF]).is_none()); + // Work factor above the DoS cap. + assert!(AesParams::parse(&props(30, &[0; 16], &[0; 16])).is_none()); + // Declared salt+IV longer than the property buffer. + assert!(AesParams::parse(&[0xC0, 0xFF, 1, 2, 3]).is_none()); + // Empty properties. + assert!(AesParams::parse(&[]).is_none()); + } + + #[test] + fn decrypts_round_trip_at_every_chunking() { + let salt = [0x11u8; 16]; + let iv = [0x22u8; 16]; + let ncp = 4; + let password = b"correct horse"; + let key = derive_key(ncp, &salt, &password_utf16le(password)); + let plaintext: Vec = (0..1000u32).map(|i| (i * 31 + 7) as u8).collect(); + let ciphertext = encrypt(&key, &iv, &plaintext); + + let params = AesParams::parse(&props(ncp, &salt, &iv)).unwrap(); + for chunk in [1usize, 3, 16, 17, 512, 4096] { + let reader = ChunkReader { + data: ciphertext.clone(), + pos: 0, + chunk, + }; + let decoder = AesDecoder::new(params, plaintext.len() as u64, password); + let mut cr = CodecReader::new(reader, decoder, "7z-aes"); + let mut out = Vec::new(); + cr.read_to_end(&mut out).unwrap(); + assert_eq!(out, plaintext, "mismatch at chunk {chunk}"); + } + } + + struct ChunkReader { + data: Vec, + pos: usize, + chunk: usize, + } + + impl Read for ChunkReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = (self.data.len() - self.pos).min(self.chunk).min(buf.len()); + buf[..n].copy_from_slice(&self.data[self.pos..self.pos + n]); + self.pos += n; + Ok(n) + } + } +} diff --git a/libarchive_oxide/src/filter/bcj.rs b/libarchive_oxide/src/filter/bcj.rs new file mode 100644 index 0000000..1984f4d --- /dev/null +++ b/libarchive_oxide/src/filter/bcj.rs @@ -0,0 +1,786 @@ +// SPDX-FileCopyrightText: 2026 libarchive_oxide contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Branch/Call/Jump (BCJ) decode filters as a sans-I/O [`Codec`]. +//! +//! BCJ filters convert position-relative branch targets back to their stored, +//! position-independent form so that repeated call/jump instructions compress +//! better. This module is decode-only: it inverts the transform 7z/XZ encoders +//! apply. The transform at each instruction is a pure function of the bytes at +//! that position, the running instruction pointer, and (for x86) a small running +//! mask, so the reconstructed bytes are independent of how the input is chunked. +//! +//! Each family scans on a fixed stride (x86 byte-by-byte with stateful E8/E9 +//! detection; ARM/ARM64/PPC/SPARC on 4-byte, ARM-Thumb on 2-byte, IA-64 on +//! 16-byte, RISC-V on 2/4/8-byte instructions). Because a branch instruction may +//! straddle a chunk boundary, [`BcjDecoder`] retains the trailing bytes that do +//! not yet form a complete instruction window and re-scans them once more input +//! arrives; at end of input any remaining tail is emitted untransformed, exactly +//! as the encoder left it. + +// The branch transforms are ported bit-for-bit from the reference filters and rely on wrapping, +// sign-reinterpreting integer casts throughout; individual casts are not independently meaningful. +#![allow( + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + clippy::cast_sign_loss +)] + +use libarchive_oxide_core::{ArchiveError, Codec, CodecStatus, CodecStep, EndOfInput}; + +/// Which instruction family a BCJ filter targets. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum BranchKind { + /// x86 `E8`/`E9` call/jump (stateful, byte-granular). + X86, + /// 32-bit ARM `BL`. + Arm, + /// ARM Thumb `BL`/`BLX`. + ArmThumb, + /// 64-bit ARM `BL`/`ADRP`. + Arm64, + /// PowerPC `bl`. + Ppc, + /// SPARC `call`. + Sparc, + /// Itanium (IA-64) bundles. + Ia64, + /// RISC-V `JAL`/`AUIPC`. + RiscV, +} + +/// Running BCJ transform state (instruction pointer plus x86 mask). +struct BcjFilter { + kind: BranchKind, + /// Running position of the first byte of the buffer handed to [`Self::decode`]. + pos: usize, + /// x86-only running branch mask carried across instructions. + prev_mask: u32, +} + +const MASK_TO_ALLOWED_STATUS: [bool; 8] = [true, true, true, false, true, false, false, false]; +const MASK_TO_BIT_NUMBER: [u32; 8] = [0, 1, 2, 2, 3, 3, 3, 3]; + +#[inline] +const fn test_86_ms_byte(b: u8) -> bool { + b == 0x00 || b == 0xFF +} + +impl BcjFilter { + fn new(kind: BranchKind, start_pos: usize) -> Self { + // Per-family starting offsets mirror the reference encoders so decode is + // their exact inverse. + let pos = match kind { + BranchKind::X86 => start_pos.wrapping_add(5), + BranchKind::Arm => start_pos.wrapping_add(8), + BranchKind::ArmThumb => start_pos.wrapping_add(4), + BranchKind::Arm64 + | BranchKind::Ppc + | BranchKind::Sparc + | BranchKind::Ia64 + | BranchKind::RiscV => start_pos, + }; + Self { + kind, + pos, + prev_mask: 0, + } + } + + /// Transforms a leading run of complete instructions in place and returns the + /// number of bytes committed. Trailing bytes that cannot yet hold a full + /// instruction window are left untouched for the next call. + fn decode(&mut self, buf: &mut [u8]) -> usize { + match self.kind { + BranchKind::X86 => self.x86(buf), + BranchKind::Arm => self.arm(buf), + BranchKind::ArmThumb => self.arm_thumb(buf), + BranchKind::Arm64 => self.arm64(buf), + BranchKind::Ppc => self.ppc(buf), + BranchKind::Sparc => self.sparc(buf), + BranchKind::Ia64 => self.ia64(buf), + BranchKind::RiscV => self.riscv(buf), + } + } + + fn x86(&mut self, buf: &mut [u8]) -> usize { + let len = buf.len(); + if len < 5 { + return 0; + } + let end = len - 5; + let mut prev_pos: isize = -1; + let mut prev_mask = self.prev_mask; + let mut i = 0usize; + while i <= end { + let b = buf[i]; + if b != 0xE9 && b != 0xE8 { + i += 1; + continue; + } + prev_pos = i as isize - prev_pos; + if (prev_pos & !3) != 0 { + prev_mask = 0; + } else { + prev_mask = (prev_mask << ((prev_pos - 1) as u32)) & 7; + if prev_mask != 0 + && (!MASK_TO_ALLOWED_STATUS[prev_mask as usize] + || test_86_ms_byte( + buf[i + 4 - MASK_TO_BIT_NUMBER[prev_mask as usize] as usize], + )) + { + prev_pos = i as isize; + prev_mask = (prev_mask << 1) | 1; + i += 1; + continue; + } + } + + prev_pos = i as isize; + if test_86_ms_byte(buf[i + 4]) { + let mut src = i32::from(buf[i + 1]) + | (i32::from(buf[i + 2]) << 8) + | (i32::from(buf[i + 3]) << 16) + | (i32::from(buf[i + 4]) << 24); + let mut dest: i32; + loop { + dest = src.wrapping_sub((self.pos.wrapping_add(i)) as i32); + if prev_mask == 0 { + break; + } + let index = MASK_TO_BIT_NUMBER[prev_mask as usize] * 8; + if !test_86_ms_byte(((dest >> (24 - index)) & 0xFF) as u8) { + break; + } + src = dest ^ ((1i32 << (32 - index)) - 1); + } + + buf[i + 1] = dest as u8; + buf[i + 2] = (dest >> 8) as u8; + buf[i + 3] = (dest >> 16) as u8; + buf[i + 4] = (!(((dest >> 24) & 1) - 1)) as u8; + i += 4; + } else { + prev_mask = (prev_mask << 1) | 1; + } + i += 1; + } + + prev_pos = i as isize - prev_pos; + prev_mask = if (prev_pos & !3) != 0 { + 0 + } else { + prev_mask << ((prev_pos - 1) as u32) + }; + + self.prev_mask = prev_mask; + self.pos = self.pos.wrapping_add(i); + i + } + + fn arm(&mut self, buf: &mut [u8]) -> usize { + let len = buf.len(); + if len < 4 { + return 0; + } + let end = len - 4; + let mut i = 0usize; + while i <= end { + if buf[i + 3] == 0xEB { + let b2 = i32::from(buf[i + 2]); + let b1 = i32::from(buf[i + 1]); + let b0 = i32::from(buf[i]); + + let src = ((b2 << 16) | (b1 << 8) | b0) << 2; + let p = self.pos.wrapping_add(i) as i32; + let dest = src.wrapping_sub(p) >> 2; + buf[i + 2] = ((dest >> 16) & 0xFF) as u8; + buf[i + 1] = ((dest >> 8) & 0xFF) as u8; + buf[i] = (dest & 0xFF) as u8; + } + i += 4; + } + self.pos = self.pos.wrapping_add(i); + i + } + + fn arm_thumb(&mut self, buf: &mut [u8]) -> usize { + let len = buf.len(); + if len < 4 { + return 0; + } + let end = len - 4; + let mut i = 0usize; + while i <= end { + let b1 = i32::from(buf[i + 1]); + let b3 = i32::from(buf[i + 3]); + if (b3 & 0xF8) == 0xF8 && (b1 & 0xF8) == 0xF0 { + let b2 = i32::from(buf[i + 2]); + let b0 = i32::from(buf[i]); + + let src = + ((b1 & 0x07) << 19) | ((b0 & 0xFF) << 11) | ((b3 & 0x07) << 8) | (b2 & 0xFF); + let src = src << 1; + let dest = src.wrapping_sub(self.pos.wrapping_add(i) as i32) >> 1; + buf[i + 1] = (0xF0 | ((dest >> 19) & 0x07)) as u8; + buf[i] = (dest >> 11) as u8; + buf[i + 3] = (0xF8 | ((dest >> 8) & 0x07)) as u8; + buf[i + 2] = (dest & 0xFF) as u8; + i += 2; + } + i += 2; + } + self.pos = self.pos.wrapping_add(i); + i + } + + fn arm64(&mut self, buf: &mut [u8]) -> usize { + let len = buf.len(); + if len < 4 { + return 0; + } + let end = len - 4; + let mut i = 0usize; + while i <= end { + let b3 = i32::from(buf[i + 3]); + let b2 = i32::from(buf[i + 2]); + let b1 = i32::from(buf[i + 1]); + let b0 = i32::from(buf[i]); + + let src = (b3 << 24) + (b2 << 16) + (b1 << 8) + b0; + let p = self.pos.wrapping_add(i) as i32; + + // BL + if ((src >> 26) & 0x3F) == 0x25 { + let dest_adr = src.wrapping_sub(p >> 2); + let dest = (dest_adr & 0x03FF_FFFF) | (0x94 << 24); + + buf[i + 3] = ((dest >> 24) & 0xFF) as u8; + buf[i + 2] = ((dest >> 16) & 0xFF) as u8; + buf[i + 1] = ((dest >> 8) & 0xFF) as u8; + buf[i] = (dest & 0xFF) as u8; + } + + // ADRP + if ((src >> 24) & 0x9F) == 0x90 { + let addr = ((src >> 29) & 3) | ((src >> 3) & 0x001F_FFFC); + + if 0 == (addr.wrapping_add(0x0002_0000) & 0x001C_0000) { + let dest = (0x90 << 24) | (src & 0x1F); + let addr = addr.wrapping_sub(p >> 12); + let dest = dest | ((addr & 3) << 29); + let dest = dest | ((addr & 0x0003_FFFC) << 3); + let dest = dest | (0i32.wrapping_sub(addr & 0x0002_0000) & 0x00E0_0000); + + buf[i + 3] = ((dest >> 24) & 0xFF) as u8; + buf[i + 2] = ((dest >> 16) & 0xFF) as u8; + buf[i + 1] = ((dest >> 8) & 0xFF) as u8; + buf[i] = (dest & 0xFF) as u8; + } + } + + i += 4; + } + self.pos = self.pos.wrapping_add(i); + i + } + + fn ppc(&mut self, buf: &mut [u8]) -> usize { + let len = buf.len(); + if len < 4 { + return 0; + } + let end = len - 4; + let mut i = 0usize; + while i <= end { + let b3 = i32::from(buf[i + 3]); + let b0 = i32::from(buf[i]); + + if (b0 & 0xFC) == 0x48 && (b3 & 0x03) == 0x01 { + let b2 = i32::from(buf[i + 2]); + let b1 = i32::from(buf[i + 1]); + + let src = + ((b0 & 0x03) << 24) | ((b1 & 0xFF) << 16) | ((b2 & 0xFF) << 8) | (b3 & 0xFC); + let p = self.pos.wrapping_add(i) as i32; + let dest = src.wrapping_sub(p); + + buf[i] = (0x48 | ((dest >> 24) & 0x03)) as u8; + buf[i + 1] = (dest >> 16) as u8; + buf[i + 2] = (dest >> 8) as u8; + buf[i + 3] = ((b3 & 0x03) | dest) as u8; + } + i += 4; + } + self.pos = self.pos.wrapping_add(i); + i + } + + fn sparc(&mut self, buf: &mut [u8]) -> usize { + let len = buf.len(); + if len < 4 { + return 0; + } + let end = len - 4; + let mut i = 0usize; + while i <= end { + let b0 = i32::from(buf[i]); + let b1 = i32::from(buf[i + 1]); + + if (b0 == 0x40 && (b1 & 0xC0) == 0x00) || (b0 == 0x7F && (b1 & 0xC0) == 0xC0) { + let b2 = i32::from(buf[i + 2]); + let b3 = i32::from(buf[i + 3]); + + let src = + ((b0 & 0xFF) << 24) | ((b1 & 0xFF) << 16) | ((b2 & 0xFF) << 8) | (b3 & 0xFF); + let src = src << 2; + let p = self.pos.wrapping_add(i) as i32; + let dest = src.wrapping_sub(p) >> 2; + let dest = (((0 - ((dest >> 22) & 1)) << 22) & 0x3FFF_FFFF) + | (dest & 0x3F_FFFF) + | 0x4000_0000; + + buf[i] = (dest >> 24) as u8; + buf[i + 1] = (dest >> 16) as u8; + buf[i + 2] = (dest >> 8) as u8; + buf[i + 3] = dest as u8; + } + i += 4; + } + self.pos = self.pos.wrapping_add(i); + i + } + + fn ia64(&mut self, buf: &mut [u8]) -> usize { + const BRANCH_TABLE: [u32; 32] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 6, 6, 0, 0, 7, 7, 4, 4, 0, 0, 4, + 4, 0, 0, + ]; + + let len = buf.len(); + if len < 16 { + return 0; + } + let end = len - 16; + let mut i = 0usize; + + while i <= end { + let instr_template = (buf[i] & 0x1F) as usize; + let mask = BRANCH_TABLE[instr_template]; + + for slot in 0..3 { + let bit_pos = 5 + slot * 41; + if ((mask >> slot) & 1) == 0 { + continue; + } + + let byte_pos = bit_pos >> 3; + let bit_res = bit_pos & 7; + + let mut instr: u64 = 0; + for j in 0..6 { + if i + byte_pos + j < buf.len() { + instr |= u64::from(buf[i + byte_pos + j]) << (8 * j); + } + } + + let instr_norm = instr >> bit_res; + + if ((instr_norm >> 37) & 0x0F) != 0x05 || ((instr_norm >> 9) & 0x07) != 0x00 { + continue; + } + + let mut src = ((instr_norm >> 13) & 0x0F_FFFF) as i32; + src |= (((instr_norm >> 36) & 1) as i32) << 20; + src <<= 4; + + let dest = src.wrapping_sub((self.pos as i32).wrapping_add(i as i32)); + let dest = (dest as u32) >> 4; + + let mut instr_norm = instr_norm; + instr_norm &= !(0x8F_FFFF_u64 << 13); + instr_norm |= u64::from(dest & 0x0F_FFFF) << 13; + instr_norm |= u64::from(dest & 0x10_0000) << (36 - 20); + + let mut instr = instr & ((1_u64 << bit_res) - 1); + instr |= instr_norm << bit_res; + + for j in 0..6 { + if i + byte_pos + j < buf.len() { + buf[i + byte_pos + j] = (instr >> (8 * j)) as u8; + } + } + } + + i += 16; + } + + self.pos = self.pos.wrapping_add(i); + i + } + + fn riscv(&mut self, buf: &mut [u8]) -> usize { + let len = buf.len(); + if len < 8 { + return 0; + } + let end = len - 8; + let mut i = 0usize; + + while i <= end { + let inst = u32::from(buf[i]); + + if inst == 0xEF { + // JAL + let b1 = u32::from(buf[i + 1]); + if (b1 & 0x0D) != 0 { + i += 2; + continue; + } + + let b2 = u32::from(buf[i + 2]); + let b3 = u32::from(buf[i + 3]); + let pc = (self.pos + i) as i32; + + let addr = ((b1 & 0xF0) << 13) | (b2 << 9) | (b3 << 1); + let addr = (addr as i32).wrapping_sub(pc); + + buf[i + 1] = ((b1 & 0x0F) | ((addr as u32 >> 8) & 0xF0)) as u8; + buf[i + 2] = + (((addr >> 16) & 0x0F) | ((addr >> 7) & 0x10) | ((addr << 4) & 0xE0)) as u8; + buf[i + 3] = (((addr >> 4) & 0x7F) | ((addr >> 13) & 0x80)) as u8; + + i += 4; + } else if (inst & 0x7F) == 0x17 { + // AUIPC + let mut inst_full = inst + | (u32::from(buf[i + 1]) << 8) + | (u32::from(buf[i + 2]) << 16) + | (u32::from(buf[i + 3]) << 24); + + if (inst_full & 0xE80) != 0 { + let inst2 = + u32::from_le_bytes([buf[i + 4], buf[i + 5], buf[i + 6], buf[i + 7]]); + + if (((inst_full << 8) ^ inst2) & 0xF_8003) != 3 { + i += 6; + continue; + } + + let addr = + ((inst_full & 0xFFFF_F000) as i32).wrapping_add((inst2 >> 20) as i32); + + inst_full = 0x17 | (2 << 7) | (inst2 << 12); + let inst2_new = addr; + + buf[i] = inst_full as u8; + buf[i + 1] = (inst_full >> 8) as u8; + buf[i + 2] = (inst_full >> 16) as u8; + buf[i + 3] = (inst_full >> 24) as u8; + + buf[i + 4] = inst2_new as u8; + buf[i + 5] = (inst2_new >> 8) as u8; + buf[i + 6] = (inst2_new >> 16) as u8; + buf[i + 7] = (inst2_new >> 24) as u8; + } else { + let fake_rs1 = inst_full >> 27; + + if ((inst_full.wrapping_sub(0x3100)) & 0x3F80) >= (fake_rs1 & 0x1D) { + i += 4; + continue; + } + + let addr = i32::from_be_bytes([buf[i + 4], buf[i + 5], buf[i + 6], buf[i + 7]]); + let addr = addr.wrapping_sub((self.pos + i) as i32); + + let inst2_rs1 = inst_full >> 27; + let inst2 = (inst_full >> 12) | ((addr as u32) << 20); + + inst_full = + 0x17 | (inst2_rs1 << 7) | ((addr.wrapping_add(0x800) as u32) & 0xFFFF_F000); + + buf[i] = inst_full as u8; + buf[i + 1] = (inst_full >> 8) as u8; + buf[i + 2] = (inst_full >> 16) as u8; + buf[i + 3] = (inst_full >> 24) as u8; + + buf[i + 4] = inst2 as u8; + buf[i + 5] = (inst2 >> 8) as u8; + buf[i + 6] = (inst2 >> 16) as u8; + buf[i + 7] = (inst2 >> 24) as u8; + } + + i += 8; + } else { + i += 2; + } + } + + self.pos = self.pos.wrapping_add(i); + i + } +} + +/// Incremental BCJ decoder driving a [`BcjFilter`] with a small carry buffer so +/// that instruction windows straddling input chunks are handled correctly. +pub(crate) struct BcjDecoder { + filter: BcjFilter, + /// Encoded bytes accepted but not yet committed as a complete instruction run. + buf: Vec, + /// Decoded bytes ready to emit. + out: Vec, + /// Emit cursor into `out`. + out_pos: usize, + finished: bool, +} + +impl BcjDecoder { + /// Builds a decoder for `kind`, with the running position seeded from `start`. + pub(crate) fn new(kind: BranchKind, start: usize) -> Self { + Self { + filter: BcjFilter::new(kind, start), + buf: Vec::new(), + out: Vec::new(), + out_pos: 0, + finished: false, + } + } +} + +impl core::fmt::Debug for BcjDecoder { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("BcjDecoder") + .field("kind", &self.filter.kind) + .field("buffered", &self.buf.len()) + .field("pending", &(self.out.len() - self.out_pos)) + .finish_non_exhaustive() + } +} + +impl Codec for BcjDecoder { + fn process( + &mut self, + input: &[u8], + output: &mut [u8], + end: EndOfInput, + ) -> Result { + // Phase A: drain decoded bytes already staged from an earlier call. + if self.out_pos < self.out.len() { + let available = self.out.len() - self.out_pos; + let count = available.min(output.len()); + output[..count].copy_from_slice(&self.out[self.out_pos..self.out_pos + count]); + self.out_pos += count; + if self.out_pos == self.out.len() { + self.out.clear(); + self.out_pos = 0; + } + return Ok(CodecStep { + consumed: 0, + produced: count, + status: CodecStatus::NeedOutput, + }); + } + if self.finished { + return Ok(CodecStep { + consumed: 0, + produced: 0, + status: CodecStatus::Done, + }); + } + + // Phase B: ingest this call's input and either filter it or, at true end + // of input, flush the untransformed tail. + let consumed = input.len(); + if consumed > 0 { + self.buf.extend_from_slice(input); + } + let at_end = matches!(end, EndOfInput::End) && consumed == 0; + if at_end { + // The trailing bytes that never formed a complete instruction window + // are emitted exactly as stored. + core::mem::swap(&mut self.out, &mut self.buf); + self.finished = true; + } else { + let filtered = self.filter.decode(&mut self.buf); + if filtered > 0 { + self.out.extend_from_slice(&self.buf[..filtered]); + self.buf.drain(..filtered); + } + } + + // Phase C: emit whatever the filter produced. + if self.out.is_empty() { + let status = if self.finished { + CodecStatus::Done + } else { + // Input consumed but no complete instruction yet; ask for more. + CodecStatus::NeedInput + }; + return Ok(CodecStep { + consumed, + produced: 0, + status, + }); + } + let count = self.out.len().min(output.len()); + output[..count].copy_from_slice(&self.out[..count]); + self.out_pos = count; + if self.out_pos == self.out.len() { + self.out.clear(); + self.out_pos = 0; + } + Ok(CodecStep { + consumed, + produced: count, + status: CodecStatus::NeedInput, + }) + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used)] +mod tests { + use std::io::{Cursor, Read, Write}; + + use lzma_rust2::filter::bcj::{BcjReader, BcjWriter}; + + use super::*; + use crate::codec_read::CodecReader; + + /// Deterministic pseudo-random bytes so branch opcodes actually occur and get transformed. + fn pseudo_random(len: usize) -> Vec { + let mut state: u64 = 0x1234_5678_9abc_def1; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state >> 24) as u8 + }) + .collect() + } + + /// A reader that yields at most `chunk` bytes per `read`, to stress input-boundary handling. + struct ChunkReader { + data: Vec, + pos: usize, + chunk: usize, + } + + impl Read for ChunkReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let remaining = self.data.len() - self.pos; + let n = remaining.min(self.chunk).min(buf.len()); + buf[..n].copy_from_slice(&self.data[self.pos..self.pos + n]); + self.pos += n; + Ok(n) + } + } + + fn encode(kind: BranchKind, data: &[u8]) -> Vec { + let cursor = Cursor::new(Vec::new()); + let mut writer = match kind { + BranchKind::X86 => BcjWriter::new_x86(cursor, 0), + BranchKind::Arm => BcjWriter::new_arm(cursor, 0), + BranchKind::ArmThumb => BcjWriter::new_arm_thumb(cursor, 0), + BranchKind::Arm64 => BcjWriter::new_arm64(cursor, 0), + BranchKind::Ppc => BcjWriter::new_ppc(cursor, 0), + BranchKind::Sparc => BcjWriter::new_sparc(cursor, 0), + BranchKind::Ia64 => BcjWriter::new_ia64(cursor, 0), + BranchKind::RiscV => BcjWriter::new_riscv(cursor, 0), + }; + writer.write_all(data).unwrap(); + writer.finish().unwrap().into_inner() + } + + fn reference_decode(kind: BranchKind, encoded: &[u8]) -> Vec { + let cursor = Cursor::new(encoded.to_vec()); + let mut reader = match kind { + BranchKind::X86 => BcjReader::new_x86(cursor, 0), + BranchKind::Arm => BcjReader::new_arm(cursor, 0), + BranchKind::ArmThumb => BcjReader::new_arm_thumb(cursor, 0), + BranchKind::Arm64 => BcjReader::new_arm64(cursor, 0), + BranchKind::Ppc => BcjReader::new_ppc(cursor, 0), + BranchKind::Sparc => BcjReader::new_sparc(cursor, 0), + BranchKind::Ia64 => BcjReader::new_ia64(cursor, 0), + BranchKind::RiscV => BcjReader::new_riscv(cursor, 0), + }; + let mut out = Vec::new(); + reader.read_to_end(&mut out).unwrap(); + out + } + + fn arca_decode(kind: BranchKind, encoded: &[u8], out_chunk: usize, in_chunk: usize) -> Vec { + let inner = ChunkReader { + data: encoded.to_vec(), + pos: 0, + chunk: in_chunk, + }; + let mut reader = CodecReader::new(inner, BcjDecoder::new(kind, 0), "bcj"); + let mut out = Vec::new(); + let mut buf = vec![0u8; out_chunk]; + loop { + let n = reader.read(&mut buf).unwrap(); + if n == 0 { + break; + } + out.extend_from_slice(&buf[..n]); + } + out + } + + #[test] + fn every_branch_kind_round_trips_against_lzma_rust2() { + let data = pseudo_random(150_003); + for kind in [ + BranchKind::X86, + BranchKind::Arm, + BranchKind::ArmThumb, + BranchKind::Arm64, + BranchKind::Ppc, + BranchKind::Sparc, + BranchKind::Ia64, + BranchKind::RiscV, + ] { + let encoded = encode(kind, &data); + // The reference decoder must reconstruct the input (encoder actually transformed it). + assert_eq!( + reference_decode(kind, &encoded), + data, + "reference round trip failed for {kind:?}" + ); + // A non-trivial transform occurred for at least the byte-granular x86 filter. + if kind == BranchKind::X86 { + assert_ne!(encoded, data, "x86 encoder made no change"); + } + // arca decodes byte-identically regardless of input/output chunking. + for (out_chunk, in_chunk) in [(1, 1), (3, 7), (64, 5), (4096, 64), (200_000, 200_000)] { + assert_eq!( + arca_decode(kind, &encoded, out_chunk, in_chunk), + data, + "arca decode mismatch for {kind:?} at out={out_chunk} in={in_chunk}" + ); + } + } + } + + #[test] + fn tiny_output_matches_bulk_output() { + // Empty and near-window-boundary lengths for the x86 filter's held-back tail. + for len in [0usize, 1, 4, 5, 6, 8, 4095, 4096, 4097] { + let data = pseudo_random(len); + let encoded = encode(BranchKind::X86, &data); + let bulk = arca_decode( + BranchKind::X86, + &encoded, + usize::max(len, 1), + usize::max(len, 1), + ); + let single = arca_decode(BranchKind::X86, &encoded, 1, 1); + assert_eq!(bulk, data, "bulk decode mismatch at len {len}"); + assert_eq!(single, data, "single-byte decode mismatch at len {len}"); + } + } +} diff --git a/libarchive_oxide/src/filter/delta.rs b/libarchive_oxide/src/filter/delta.rs new file mode 100644 index 0000000..9362e10 --- /dev/null +++ b/libarchive_oxide/src/filter/delta.rs @@ -0,0 +1,168 @@ +// SPDX-FileCopyrightText: 2026 libarchive_oxide contributors +// +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Delta decode filter as a sans-I/O [`Codec`]. +//! +//! The 7z/XZ delta filter reconstructs each output byte by adding the byte +//! `distance` positions back in the already-decoded history. It is fully +//! per-byte and carries no partial-instruction state, so any input chunking +//! yields the same output. `distance` is `1..=256`; the history ring is a fixed +//! 256-byte buffer indexed modulo 256. + +use libarchive_oxide_core::{ArchiveError, Codec, CodecStatus, CodecStep, EndOfInput}; + +/// The delta filter's fixed maximum distance (and history ring size). +const MAX_DISTANCE: usize = 256; +/// Mask folding a ring index into `0..MAX_DISTANCE`. +const DIS_MASK: usize = MAX_DISTANCE - 1; + +/// Incremental delta decoder. Decode-only: it inverts the encoder by adding the +/// historical byte back to each delta-coded input byte. +pub(crate) struct DeltaDecoder { + distance: usize, + history: [u8; MAX_DISTANCE], + pos: u8, +} + +impl DeltaDecoder { + /// Builds a decoder for `distance` (clamped to the valid `1..=256` range). + pub(crate) fn new(distance: usize) -> Self { + Self { + distance: distance.clamp(1, MAX_DISTANCE), + history: [0; MAX_DISTANCE], + pos: 0, + } + } +} + +impl core::fmt::Debug for DeltaDecoder { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("DeltaDecoder") + .field("distance", &self.distance) + .finish_non_exhaustive() + } +} + +impl Codec for DeltaDecoder { + fn process( + &mut self, + input: &[u8], + output: &mut [u8], + end: EndOfInput, + ) -> Result { + let count = input.len().min(output.len()); + for (dst, &coded) in output[..count].iter_mut().zip(&input[..count]) { + let pos = self.pos as usize; + let history = self.history[self.distance.wrapping_add(pos) & DIS_MASK]; + let value = coded.wrapping_add(history); + *dst = value; + self.history[pos & DIS_MASK] = value; + self.pos = self.pos.wrapping_sub(1); + } + if count > 0 { + // `consumed == produced`; the status is only consulted on zero progress. + return Ok(CodecStep { + consumed: count, + produced: count, + status: CodecStatus::NeedInput, + }); + } + // Output is always non-empty here, so zero progress means the input is empty. + let status = match end { + EndOfInput::End => CodecStatus::Done, + EndOfInput::More => CodecStatus::NeedInput, + }; + Ok(CodecStep { + consumed: 0, + produced: 0, + status, + }) + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::cast_possible_truncation)] +mod tests { + use std::io::{Cursor, Read, Write}; + + use lzma_rust2::filter::delta::{DeltaReader, DeltaWriter}; + + use super::*; + use crate::codec_read::CodecReader; + + fn pseudo_random(len: usize) -> Vec { + let mut state: u64 = 0x0f1e_2d3c_4b5a_6978; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state >> 24) as u8 + }) + .collect() + } + + struct ChunkReader { + data: Vec, + pos: usize, + chunk: usize, + } + + impl Read for ChunkReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = (self.data.len() - self.pos).min(self.chunk).min(buf.len()); + buf[..n].copy_from_slice(&self.data[self.pos..self.pos + n]); + self.pos += n; + Ok(n) + } + } + + fn encode(distance: usize, data: &[u8]) -> Vec { + let mut writer = DeltaWriter::new(Cursor::new(Vec::new()), distance); + writer.write_all(data).unwrap(); + writer.into_inner().into_inner() + } + + fn arca_decode(distance: usize, encoded: &[u8], out_chunk: usize, in_chunk: usize) -> Vec { + let inner = ChunkReader { + data: encoded.to_vec(), + pos: 0, + chunk: in_chunk, + }; + let mut reader = CodecReader::new(inner, DeltaDecoder::new(distance), "delta"); + let mut out = Vec::new(); + let mut buf = vec![0u8; out_chunk]; + loop { + let n = reader.read(&mut buf).unwrap(); + if n == 0 { + break; + } + out.extend_from_slice(&buf[..n]); + } + out + } + + #[test] + fn delta_round_trips_against_lzma_rust2_at_every_distance() { + let data = pseudo_random(70_007); + for distance in [1usize, 2, 3, 4, 16, 255, 256] { + let encoded = encode(distance, &data); + // Reference reader reconstructs the input. + let mut reference = Vec::new(); + DeltaReader::new(Cursor::new(encoded.clone()), distance) + .read_to_end(&mut reference) + .unwrap(); + assert_eq!(reference, data, "reference mismatch at distance {distance}"); + // arca decodes byte-identically at any chunking. + for (out_chunk, in_chunk) in [(1, 1), (3, 7), (256, 5), (65536, 65536)] { + assert_eq!( + arca_decode(distance, &encoded, out_chunk, in_chunk), + data, + "arca mismatch at distance {distance}, out={out_chunk}, in={in_chunk}" + ); + } + } + } +} diff --git a/libarchive_oxide/src/filter/gzip.rs b/libarchive_oxide/src/filter/gzip.rs index d1a87ba..0c6418d 100644 --- a/libarchive_oxide/src/filter/gzip.rs +++ b/libarchive_oxide/src/filter/gzip.rs @@ -362,6 +362,157 @@ impl Codec for GzipDecoder { } } +/// Raw DEFLATE (RFC 1951) inflate with no framing. +/// +/// The gzip decoder above wraps this same `miniz_oxide` raw-inflate core in RFC 1952 +/// header/trailer handling; the 7z Deflate coder, by contrast, stores bare deflate +/// blocks with no header, trailer, or checksum, so it drives the raw core directly. +/// The window is a fixed 32 KiB, so the decoder's working set is inherently bounded; +/// a `decoded_total` guard adds decompression-bomb defense on top. +#[cfg(feature = "sevenz")] +pub(crate) struct RawInflateDecoder { + inflate: Box, + limits: Limits, + decoded_total: u64, + done: bool, +} + +#[cfg(feature = "sevenz")] +impl core::fmt::Debug for RawInflateDecoder { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("RawInflateDecoder") + .field("done", &self.done) + .finish_non_exhaustive() + } +} + +#[cfg(feature = "sevenz")] +impl RawInflateDecoder { + /// Creates a raw-DEFLATE decompressor with mandatory resource limits. + #[must_use] + pub(crate) fn new(limits: Limits) -> Self { + Self { + inflate: Box::new(InflateState::new(DataFormat::Raw)), + limits, + decoded_total: 0, + done: false, + } + } + + fn account_output(&mut self, produced: usize) -> Result<()> { + if produced == 0 { + return Ok(()); + } + let next = self + .decoded_total + .checked_add(produced as u64) + .ok_or_else(|| gzip_error(ErrorKind::Limit, "decoded byte count overflow"))?; + if self + .limits + .decoded_total() + .is_some_and(|maximum| next > maximum) + { + return Err(gzip_error( + ErrorKind::Limit, + "decoded stream exceeds configured limit", + )); + } + self.decoded_total = next; + Ok(()) + } +} + +#[cfg(feature = "sevenz")] +impl Codec for RawInflateDecoder { + fn process( + &mut self, + input: &[u8], + output: &mut [u8], + end: EndOfInput, + ) -> core::result::Result { + if self.done { + if input.is_empty() { + return Ok(CodecStep { + consumed: 0, + produced: 0, + status: CodecStatus::Done, + }); + } + return Err(gzip_error( + ErrorKind::Malformed, + "data follows the completed deflate stream", + )); + } + // `miniz_oxide`'s streaming `inflate` errors on empty input under `MZFlush::None`, so an + // empty chunk is handled explicitly: flush the tail at end-of-input, otherwise wait. + if input.is_empty() { + if !matches!(end, EndOfInput::End) { + return Ok(CodecStep { + consumed: 0, + produced: 0, + status: CodecStatus::NeedInput, + }); + } + let res = inflate(&mut self.inflate, &[], output, MZFlush::Finish); + let produced = res.bytes_written; + self.account_output(produced)?; + return match res.status { + Ok(MZStatus::StreamEnd) => { + self.done = true; + CodecStep { + consumed: 0, + produced, + status: CodecStatus::Done, + } + .validate(input.len(), output.len()) + }, + Ok(_) if produced > 0 => CodecStep { + consumed: 0, + produced, + status: CodecStatus::NeedOutput, + } + .validate(input.len(), output.len()), + _ => Err(gzip_error(ErrorKind::Malformed, "truncated deflate stream")), + }; + } + + let res = inflate(&mut self.inflate, input, output, MZFlush::None); + let consumed = res.bytes_consumed; + let produced = res.bytes_written; + self.account_output(produced)?; + match res.status { + Ok(MZStatus::StreamEnd) => { + self.done = true; + return CodecStep { + consumed, + produced, + status: CodecStatus::Done, + } + .validate(input.len(), output.len()); + }, + Ok(_) | Err(MZError::Buf) => {}, + Err(_) => return Err(gzip_error(ErrorKind::Malformed, "inflate error")), + } + // Report progress in a shape the driver can always advance on: output-full asks for more + // output; input-drained asks for more input; a genuine no-progress stall is malformed. + let status = if produced == output.len() { + CodecStatus::NeedOutput + } else if consumed == input.len() { + CodecStatus::NeedInput + } else if consumed == 0 && produced == 0 { + return Err(gzip_error(ErrorKind::Malformed, "deflate stream stalled")); + } else { + CodecStatus::NeedOutput + }; + CodecStep { + consumed, + produced, + status, + } + .validate(input.len(), output.len()) + } +} + const GZIP_HEADER: [u8; 10] = [0x1f, 0x8b, 0x08, 0x00, 0, 0, 0, 0, 0x00, 0xff]; #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/libarchive_oxide/src/filter/mod.rs b/libarchive_oxide/src/filter/mod.rs index d30b145..30b356b 100644 --- a/libarchive_oxide/src/filter/mod.rs +++ b/libarchive_oxide/src/filter/mod.rs @@ -4,6 +4,12 @@ //! Compression filter implementations and runtime dispatch. +#[cfg(all(feature = "sevenz", feature = "aes"))] +pub(crate) mod aes7z; +#[cfg(feature = "sevenz")] +pub(crate) mod bcj; +#[cfg(feature = "sevenz")] +pub(crate) mod delta; pub mod gzip; #[cfg(all(feature = "lz4", not(feature = "native-codecs")))] pub(crate) mod lz4; diff --git a/libarchive_oxide/src/filtered_io.rs b/libarchive_oxide/src/filtered_io.rs index 81faa9d..f2da0a6 100644 --- a/libarchive_oxide/src/filtered_io.rs +++ b/libarchive_oxide/src/filtered_io.rs @@ -14,12 +14,34 @@ use libarchive_oxide_core::Codec; use libarchive_oxide_core::filter::FilterId; use libarchive_oxide_core::{ArchiveError, CodecStatus, EndOfInput, ErrorKind, Limits}; +#[cfg(any(feature = "zstd", feature = "lz4"))] +use crate::codec_read::CodecReader; #[cfg(not(feature = "native-codecs"))] use crate::filter::gzip::GzipEncoder; use crate::pipeline_codec::PipelineCodec; const BUFFER: usize = 64 * 1024; +/// A bounded outer-filter reader: the shared [`CodecReader`] engine driving a [`PipelineCodec`]. +#[cfg(any(feature = "zstd", feature = "lz4"))] +type PipelineRead = CodecReader; + +/// Builds a [`PipelineRead`] for an outer filter, constructing its [`PipelineCodec`] first. +#[cfg(any(feature = "zstd", feature = "lz4"))] +fn pipeline_reader( + input: R, + filter: FilterId, + limits: Limits, +) -> io::Result> { + let name = match filter { + FilterId::Zstd => "zstd", + FilterId::Lz4 => "LZ4", + _ => "codec", + }; + let codec = PipelineCodec::new(filter, limits).map_err(codec_archive_io)?; + Ok(CodecReader::new(input, codec, name)) +} + /// Auto-detecting streaming input filter. /// /// gzip uses the common sans-I/O codec. The other variants use bounded @@ -96,7 +118,7 @@ impl FilterReader { if available.starts_with(&[0x28, 0xb5, 0x2f, 0xfd]) { #[cfg(feature = "zstd")] { - return PipelineRead::new(input, FilterId::Zstd, limits) + return pipeline_reader(input, FilterId::Zstd, limits) .map(Box::new) .map(FilterReaderInner::Zstd) .map(|inner| Self { @@ -130,7 +152,7 @@ impl FilterReader { } if available.starts_with(&[0x04, 0x22, 0x4d, 0x18]) { #[cfg(feature = "lz4")] - return PipelineRead::new(input, FilterId::Lz4, limits) + return pipeline_reader(input, FilterId::Lz4, limits) .map(Box::new) .map(FilterReaderInner::Lz4) .map(|inner| Self { @@ -192,131 +214,6 @@ impl Read for PrefixReader { } } -#[cfg(any(feature = "zstd", feature = "lz4"))] -struct PipelineRead { - input: R, - decoder: PipelineCodec, - filter_name: &'static str, - buffer: Vec, - start: usize, - end: usize, - eof: bool, - done: bool, - failed: bool, -} - -#[cfg(any(feature = "zstd", feature = "lz4"))] -impl PipelineRead { - fn new(input: R, filter: FilterId, limits: Limits) -> io::Result { - let filter_name = match filter { - FilterId::Zstd => "zstd", - FilterId::Lz4 => "LZ4", - _ => "codec", - }; - Ok(Self { - input, - decoder: PipelineCodec::new(filter, limits).map_err(codec_archive_io)?, - filter_name, - buffer: vec![0; BUFFER], - start: 0, - end: 0, - eof: false, - done: false, - failed: false, - }) - } - - fn into_inner(self) -> R { - self.input - } - - fn fill(&mut self) -> io::Result<()> { - if self.start != 0 { - self.buffer.copy_within(self.start..self.end, 0); - self.end -= self.start; - self.start = 0; - } - if self.end == self.buffer.len() || self.eof { - return Ok(()); - } - let read = self.input.read(&mut self.buffer[self.end..])?; - if read == 0 { - self.eof = true; - } else { - self.end += read; - } - Ok(()) - } - - fn fail(&mut self, error: io::Error) -> io::Result { - self.failed = true; - Err(error) - } -} - -#[cfg(any(feature = "zstd", feature = "lz4"))] -impl Read for PipelineRead { - fn read(&mut self, output: &mut [u8]) -> io::Result { - if output.is_empty() || self.done { - return Ok(0); - } - if self.failed { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("{} reader is in a failed state", self.filter_name), - )); - } - loop { - if self.start == self.end && !self.eof { - self.fill()?; - } - let input_length = self.end - self.start; - let end = if self.eof { - EndOfInput::End - } else { - EndOfInput::More - }; - let step = match self - .decoder - .process(&self.buffer[self.start..self.end], output, end) - .and_then(|step| step.validate(input_length, output.len())) - { - Ok(step) => step, - Err(error) => return self.fail(codec_archive_io(error)), - }; - self.start += step.consumed; - if step.produced != 0 { - return Ok(step.produced); - } - match step.status { - CodecStatus::Done => { - self.done = true; - return Ok(0); - }, - CodecStatus::NeedInput if self.start == self.end && !self.eof => { - self.fill()?; - }, - CodecStatus::NeedOutput if step.consumed != 0 => {}, - CodecStatus::NeedInput if self.eof => { - return self.fail(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "{} decoder requested input after the source ended", - self.filter_name - ), - )); - }, - _ => { - return self.fail(io::Error::new( - io::ErrorKind::InvalidData, - format!("{} reader made no progress", self.filter_name), - )); - }, - } - } - } -} - #[cfg(feature = "xz")] struct XzRead { input: R, diff --git a/libarchive_oxide/src/lib.rs b/libarchive_oxide/src/lib.rs index 4f22b54..f0da4ec 100644 --- a/libarchive_oxide/src/lib.rs +++ b/libarchive_oxide/src/lib.rs @@ -32,6 +32,8 @@ pub mod async_seek; #[cfg(feature = "async")] pub mod async_stream; mod cab; +#[cfg(any(feature = "zstd", feature = "lz4", feature = "sevenz"))] +mod codec_read; pub mod create; pub mod engine; pub mod extractor; diff --git a/libarchive_oxide/src/pipeline_codec.rs b/libarchive_oxide/src/pipeline_codec.rs index 059f5fb..b1c7b41 100644 --- a/libarchive_oxide/src/pipeline_codec.rs +++ b/libarchive_oxide/src/pipeline_codec.rs @@ -23,6 +23,10 @@ pub(crate) enum PipelineCodec { Gzip(ExternalDecoder), #[cfg(not(feature = "native-codecs"))] Gzip(Box), + /// Raw DEFLATE (no gzip framing) — the 7z Deflate coder. Backed by the same + /// `miniz_oxide` raw-inflate core the gzip decoder sits on. + #[cfg(feature = "sevenz")] + Deflate(Box), #[cfg(feature = "bzip2")] Bzip2(ExternalDecoder), #[cfg(all(feature = "zstd", feature = "native-codecs"))] @@ -55,6 +59,10 @@ impl PipelineCodec { Ok(Self::Gzip(Box::new(GzipDecoder::new(limits)))) } }, + #[cfg(feature = "sevenz")] + FilterId::Deflate => Ok(Self::Deflate(Box::new( + crate::filter::gzip::RawInflateDecoder::new(limits), + ))), FilterId::Bzip2 => { #[cfg(feature = "bzip2")] { @@ -133,6 +141,8 @@ impl PipelineCodec { Self::Gzip(codec) => codec.process(input, output, end), #[cfg(not(feature = "native-codecs"))] Self::Gzip(codec) => codec.process(input, output, end), + #[cfg(feature = "sevenz")] + Self::Deflate(codec) => codec.process(input, output, end), #[cfg(feature = "bzip2")] Self::Bzip2(codec) => codec.process(input, output, end), #[cfg(feature = "zstd")] @@ -162,6 +172,8 @@ impl PipelineCodec { Self::Gzip(codec) => codec.poll_process(input, output, end, waker), #[cfg(not(feature = "native-codecs"))] Self::Gzip(codec) => codec.poll_process(input, output, end, waker), + #[cfg(feature = "sevenz")] + Self::Deflate(codec) => codec.poll_process(input, output, end, waker), #[cfg(feature = "bzip2")] Self::Bzip2(codec) => codec.poll_process(input, output, end, waker), #[cfg(feature = "zstd")] @@ -174,6 +186,19 @@ impl PipelineCodec { } } +impl Codec for PipelineCodec { + fn process( + &mut self, + input: &[u8], + output: &mut [u8], + end: EndOfInput, + ) -> Result { + // Delegates to the inherent method (inherent resolution wins, so this does not recurse), + // letting `PipelineCodec` drive the generic `CodecReader<_, PipelineCodec>`. + PipelineCodec::process(self, input, output, end) + } +} + #[cfg(all(feature = "zstd", feature = "native-codecs"))] fn native_zstd_decoder(limits: Limits) -> Result { let Some(memory_limit) = limits.codec_memory() else { @@ -200,6 +225,7 @@ fn disabled(filter: FilterId) -> ArchiveError { const fn filter_name(filter: FilterId) -> &'static str { match filter { FilterId::Gzip => "gzip", + FilterId::Deflate => "deflate", FilterId::Bzip2 => "bzip2", FilterId::Zstd => "zstd", FilterId::Xz => "xz", diff --git a/libarchive_oxide/src/seek_stream.rs b/libarchive_oxide/src/seek_stream.rs index e777ef6..6454a76 100644 --- a/libarchive_oxide/src/seek_stream.rs +++ b/libarchive_oxide/src/seek_stream.rs @@ -316,7 +316,7 @@ impl SeekArchiveReader { { return Ok(Self { inner: SeekDispatch::SevenZ(Box::new(crate::sevenz::SevenZSeekReader::new( - input, limits, + input, limits, password, )?)), }); } diff --git a/libarchive_oxide/src/sevenz.rs b/libarchive_oxide/src/sevenz.rs index 31ddf22..4585c9f 100644 --- a/libarchive_oxide/src/sevenz.rs +++ b/libarchive_oxide/src/sevenz.rs @@ -4,18 +4,31 @@ //! Seek-capable, payload-streaming 7z reader and writer. //! -//! Supports one folder, pack stream, and coder. Writers emit LZMA2. Readers -//! accept LZMA2 or LZMA, including encoded headers. BCJ, delta, AES, `PPMd`, -//! multiple folders, multiple coders, and coder graphs are unsupported. +//! Readers parse every folder's full coder graph (any number of coders, bind +//! pairs, and packed streams) so listing works for any archive. Decoding is +//! limited to linear chains over a single pack stream whose coders are each +//! LZMA2/LZMA, a decode-only byte filter (delta, BCJ), a general-purpose coder +//! reused from the shared codec dispatch (Deflate, `BZip2`, Zstd), or — under the +//! `aes` feature, given a password — the AES-256/SHA-256 decryption coder: such a +//! folder streams normally, while richer graphs (BCJ2, `PPMd`, multi-pack) list +//! but report `Unsupported` on extraction. Exactly one folder decode chain is live at a +//! time, so memory stays bounded regardless of folder count. Writers still emit +//! a single solid LZMA2 folder. Each substream's stored CRC-32 is verified after decode, +//! which also distinguishes a correct 7z AES password from a wrong one. use std::io::{Read, Seek, SeekFrom, Take, Write}; +use libarchive_oxide_core::filter::FilterId; use libarchive_oxide_core::{ ArchiveError, ArchiveMetadata, ArchivePath, EntryKind, EntryMetadata, EntryTimes, ErrorKind, Extension, Limits, Owner, PathEncoding, Timestamp, }; -use crate::{ReaderEvent, StreamError}; +use crate::codec_read::CodecReader; +use crate::filter::bcj::{BcjDecoder, BranchKind}; +use crate::filter::delta::DeltaDecoder; +use crate::pipeline_codec::PipelineCodec; +use crate::{ReaderEvent, SecretBytes, StreamError}; type Result = core::result::Result; @@ -60,6 +73,41 @@ const K_START_POS: u8 = 0x18; const METHOD_LZMA2: u8 = 0x21; /// LZMA (v1) coder method id (3 bytes) — how mainstream 7-Zip compresses encoded headers and folders. const METHOD_LZMA: [u8; 3] = [0x03, 0x01, 0x01]; +/// Delta filter coder method id (1 byte); its single property byte is `distance - 1`. +const METHOD_DELTA: u8 = 0x03; +/// BCJ x86 filter coder method id. +const METHOD_BCJ_X86: [u8; 4] = [0x03, 0x03, 0x01, 0x03]; +/// BCJ PowerPC filter coder method id. +const METHOD_BCJ_PPC: [u8; 4] = [0x03, 0x03, 0x02, 0x05]; +/// BCJ IA-64 filter coder method id. +const METHOD_BCJ_IA64: [u8; 4] = [0x03, 0x03, 0x04, 0x01]; +/// BCJ ARM filter coder method id. +const METHOD_BCJ_ARM: [u8; 4] = [0x03, 0x03, 0x05, 0x01]; +/// BCJ ARM-Thumb filter coder method id. +const METHOD_BCJ_ARMT: [u8; 4] = [0x03, 0x03, 0x07, 0x01]; +/// BCJ SPARC filter coder method id. +const METHOD_BCJ_SPARC: [u8; 4] = [0x03, 0x03, 0x08, 0x05]; +/// BCJ ARM64 filter coder method id (1 byte). +const METHOD_BCJ_ARM64: u8 = 0x0A; +/// BCJ RISC-V filter coder method id (1 byte). +const METHOD_BCJ_RISCV: u8 = 0x0B; +/// Deflate coder method id (3 bytes) — RAW DEFLATE (RFC 1951), no gzip/zlib framing. +const METHOD_DEFLATE: [u8; 3] = [0x04, 0x01, 0x08]; +/// `BZip2` coder method id (3 bytes). +const METHOD_BZIP2: [u8; 3] = [0x04, 0x02, 0x02]; +/// Zstandard coder method id (4 bytes), as used by the 7-Zip-zstd fork and `sevenz-rust2`. +const METHOD_ZSTD: [u8; 4] = [0x04, 0xF7, 0x11, 0x01]; +/// AES-256/SHA-256 encryption coder method id (4 bytes). +const METHOD_AES256: [u8; 4] = [0x06, 0xF1, 0x07, 0x01]; + +/// Peak decompression working set for a bzip2 stream at the largest (900 KiB) block size. +/// libbzip2 documents ~3.7 MiB for `-9` decompression; rounded up. A folder's bzip2 coder is +/// refused when the configured codec-memory budget cannot hold it. +const BZIP2_DECODE_WORKSPACE: usize = 4 * 1024 * 1024; +/// Smallest zstd window (the format's minimum `Window_Log` of 10 → 1 KiB) the decoder must be +/// allowed to allocate. The native zstd backend additionally caps the frame window to the +/// codec-memory budget inside [`PipelineCodec`]; this floor rejects a nonsensically tiny budget. +const ZSTD_MIN_WINDOW: usize = 1024; /// `FILE_ATTRIBUTE_DIRECTORY`. const ATTR_DIRECTORY: u32 = 0x10; @@ -81,31 +129,175 @@ const WRITER_PRESET: u32 = 6; // Parsed structures // ════════════════════════════════════════════════════════════════════════════════════════════════ -/// The single-coder codec of a folder. Reading supports both LZMA2 (what this crate writes) and plain -/// LZMA (what 7-Zip and `sevenz-rust2` use for compressed/encoded headers and folders). +/// One coder's decode method as parsed from a folder graph. Decoding supports LZMA2 (what this +/// crate writes), plain LZMA (what 7-Zip and `sevenz-rust2` use for compressed/encoded headers and +/// folders), the decode-only delta and BCJ byte filters, and the general-purpose Deflate/BZip2/Zstd +/// coders reused from the shared [`PipelineCodec`] dispatch. Every other method id (`PPMd`, BCJ2, …) +/// parses into [`CoderMethod::Unsupported`] so listing works, but its payload cannot be decoded here. #[derive(Debug, Clone, Copy)] -enum FolderCoder { +enum CoderMethod { /// LZMA2, carrying its one-byte dictionary-size property. Lzma2 { dict_prop: u8 }, /// LZMA (v1), carrying its 5 property bytes: `lc/lp/pb` byte + little-endian `u32` dict size. Lzma { props: [u8; 5] }, + /// A decode-only byte-to-byte filter (delta or BCJ) applied after its inner coder. + Filter(SevenFilter), + /// AES-256/SHA-256 decryption, carrying its parsed (archive-public) salt/IV/work + /// factor. The secret key is derived only at decode time from the caller's password. + #[cfg(feature = "aes")] + Aes(crate::filter::aes7z::AesParams), /// A coder whose metadata can be listed but whose payload cannot be decoded. Unsupported, } -/// The restricted single-folder description parsed from a `StreamsInfo`. +/// A 7z coder that arca decodes by reusing an existing codec. Delta and BCJ are decode-only byte +/// filters applied in place after the coder feeding them; [`SevenFilter::Pipeline`] instead reuses +/// a caller-driven [`PipelineCodec`] (Deflate/BZip2/Zstd) as a decompression coder in the chain. +#[derive(Debug, Clone, Copy)] +enum SevenFilter { + /// Delta filter with a byte `distance` in `1..=256`. + Delta { distance: usize }, + /// Branch/Call/Jump filter for a given instruction family, with a running start position. + Bcj { kind: BranchKind, start: usize }, + /// A general-purpose decompression coder reused from the shared [`PipelineCodec`] dispatch: + /// 7z Deflate (raw), `BZip2`, or Zstd, identified by its [`FilterId`]. + Pipeline { filter: FilterId }, +} + +/// One coder in a folder's graph, with its input/output stream arity. +#[derive(Debug, Clone, Copy)] +struct Coder { + method: CoderMethod, + /// Number of input streams the coder consumes. + num_in: usize, + /// Number of output streams the coder produces. + num_out: usize, +} + +/// A bind pair links one coder's output stream to another coder's input stream, both addressed by +/// global stream index within the folder. +#[derive(Debug, Clone, Copy)] +struct BindPair { + in_index: usize, + out_index: usize, +} + +/// A parsed folder coder graph. Spec-complete: it may describe more coders, bind pairs, or packed +/// streams than the linear LZMA/LZMA2 chains this crate can decode, so listing always works. +#[derive(Debug, Clone)] +struct Folder { + coders: Vec, + bind_pairs: Vec, + /// Global input-stream indices fed directly by pack streams, in pack-stream order. + packed_indices: Vec, +} + +impl Folder { + fn total_in(&self) -> usize { + self.coders.iter().map(|coder| coder.num_in).sum() + } + + fn total_out(&self) -> usize { + self.coders.iter().map(|coder| coder.num_out).sum() + } + + /// First global input-stream index owned by coder `index`. + fn in_base(&self, index: usize) -> usize { + self.coders[..index].iter().map(|coder| coder.num_in).sum() + } + + /// First global output-stream index owned by coder `index`. + fn out_base(&self, index: usize) -> usize { + self.coders[..index].iter().map(|coder| coder.num_out).sum() + } + + /// The coder that owns global output stream `out`. + fn coder_of_out(&self, out: usize) -> Option { + let mut base = 0; + for (index, coder) in self.coders.iter().enumerate() { + base += coder.num_out; + if out < base { + return Some(index); + } + } + None + } + + /// The single output stream not consumed by a bind pair — the folder's result. + fn final_out_index(&self) -> Result { + (0..self.total_out()) + .find(|out| !self.bind_pairs.iter().any(|pair| pair.out_index == *out)) + .ok_or(HeaderError::Malformed( + "7z: folder has no final output stream", + )) + } + + /// True when the graph is a linear chain of single-in/single-out coders fed by exactly one + /// pack stream — the shape [`build_folder_reader`] can fold into a decode chain. + fn is_linear_chain(&self) -> bool { + self.packed_indices.len() == 1 + && self + .coders + .iter() + .all(|coder| coder.num_in == 1 && coder.num_out == 1) + } +} + +/// One folder's fully-resolved decode description: its coder graph, pack-stream layout, output +/// sizes, and the topological decode order computed by [`resolve_folder`]. An archive carries a +/// `Vec`; each folder decodes independently from its own pack stream(s). #[derive(Debug, Clone)] struct FolderInfo { - /// Absolute offset of the packed stream within the archive bytes. + /// The parsed coder graph (spec-complete, possibly beyond what can be decoded). + graph: Folder, + /// Absolute offset of this folder's first pack stream within the archive bytes. pack_offset: usize, - /// Length of the packed stream. - pack_size: usize, - /// Uncompressed size of the whole folder. + /// This folder's pack-stream sizes, in `graph.packed_indices` order. + pack_sizes: Vec, + /// Uncompressed size of the folder's final output stream (what a reader decodes from it). unpack_size: u64, - /// The folder's single coder and its properties. - coder: FolderCoder, + /// Per global output-stream uncompressed sizes, in coder order. + output_sizes: Vec, + /// Coder indices in execution order (dependencies first), from [`resolve_folder`]. + decode_order: Vec, /// Per-substream (per content file) uncompressed sizes, in file order. substream_sizes: Vec, + /// Per-substream stored CRC-32 (`None` where the archive defines none), in file order. + substream_crcs: Vec>, +} + +impl FolderInfo { + /// Total packed span across every pack stream this folder owns. + fn pack_span(&self) -> usize { + self.pack_sizes.iter().sum() + } + + /// Whether the folder can be decoded here: a linear chain of LZMA/LZMA2 coders over a single + /// pack stream. Non-linear graphs (BCJ2, multi-pack) and unsupported methods list but do not + /// decode. + fn is_decodable(&self) -> bool { + self.graph.is_linear_chain() + && self + .graph + .coders + .iter() + .all(|coder| !matches!(coder.method, CoderMethod::Unsupported)) + } + + /// Whether the folder chains an AES-256/SHA-256 coder (so decoding needs a password). + #[cfg(feature = "aes")] + fn is_encrypted(&self) -> bool { + self.graph + .coders + .iter() + .any(|coder| matches!(coder.method, CoderMethod::Aes(_))) + } + + #[cfg(not(feature = "aes"))] + #[allow(clippy::unused_self)] + fn is_encrypted(&self) -> bool { + false + } } /// A parsed directory-entry-like record: name, kind, permissions, mtime, and (for content files) @@ -119,15 +311,20 @@ struct FileRec { anti: bool, start_position: Option, has_stream: bool, + /// Which folder holds this file's content stream (`None` for empty-stream entries). + folder_index: Option, + /// Byte offset of this file's substream within its own folder's decoded output. stream_offset: usize, size: usize, + /// Stored CRC-32 of this file's decoded substream, verified after decode (`None` if absent). + crc: Option, } /// Bounded parser for the 7z next-header metadata. #[derive(Debug, Default)] struct HeaderParser { files: Vec, - folder: Option, + folders: Vec, header_extensions: Vec, } @@ -135,24 +332,24 @@ impl HeaderParser { fn new() -> Self { Self { files: Vec::new(), - folder: None, + folders: Vec::new(), header_extensions: Vec::new(), } } /// Parses a plain `kHeader` body: optional main streams info, then files info. fn parse_header(&mut self, r: &mut ByteReader<'_>) -> Result<()> { - let mut folder: Option = None; + let mut folders: Vec = Vec::new(); loop { match r.u8()? { K_END => break, K_ARCHIVE_PROPERTIES => self.parse_archive_properties(r)?, - K_MAIN_STREAMS_INFO => folder = Some(parse_streams_info(r)?), - K_FILES_INFO => self.parse_files_info(r, folder.as_ref())?, + K_MAIN_STREAMS_INFO => folders = parse_streams_info(r)?, + K_FILES_INFO => self.parse_files_info(r, &folders)?, _ => return Err(HeaderError::Unsupported("7z: unsupported header property")), } } - self.folder = folder; + self.folders = folders; Ok(()) } @@ -175,11 +372,7 @@ impl HeaderParser { /// Parses `FilesInfo`, assembling [`FileRec`]s and assigning content-file windows in order. #[allow(clippy::too_many_lines, clippy::needless_range_loop)] - fn parse_files_info( - &mut self, - r: &mut ByteReader<'_>, - folder: Option<&FolderInfo>, - ) -> Result<()> { + fn parse_files_info(&mut self, r: &mut ByteReader<'_>, folders: &[FolderInfo]) -> Result<()> { let num_files = usize_of(r.number()?)?; if num_files as u64 > MAX_FILES { return Err(HeaderError::LimitExceeded("7z: too many files")); @@ -237,11 +430,24 @@ impl HeaderParser { )); } - // Assemble records. Content files consume substream windows in file order; empty-stream - // files are directories unless flagged as empty files. - let sizes = folder.map_or(&[][..], |f| f.substream_sizes.as_slice()); + // Assemble records. Content files consume substream windows in folder-then-file order: + // a flat list of `(folder_index, size)` pairs, one per substream across every folder in + // order, is drawn down as content files appear. The per-folder byte offset resets whenever + // the folder changes. Empty-stream files are directories unless flagged as empty files. + let flat: Vec<(usize, u64, Option)> = folders + .iter() + .enumerate() + .flat_map(|(index, folder)| { + folder + .substream_sizes + .iter() + .zip(folder.substream_crcs.iter()) + .map(move |(&size, &crc)| (index, size, crc)) + }) + .collect(); let mut content_index = 0usize; let mut running = 0usize; + let mut current_folder: Option = None; let mut empty_index = 0usize; for i in 0..num_files { @@ -249,10 +455,15 @@ impl HeaderParser { let full_mode = modes.get(i).copied().flatten(); let has_stream = !empty_stream[i]; - let (kind, offset, size, is_anti) = if has_stream { - let size = usize_of(*sizes.get(content_index).ok_or(HeaderError::Malformed( - "7z: content stream index out of range", - ))?)?; + let (kind, folder_index, offset, size, is_anti, crc) = if has_stream { + let (fi, raw_size, crc) = *flat.get(content_index).ok_or( + HeaderError::Malformed("7z: content stream index out of range"), + )?; + let size = usize_of(raw_size)?; + if current_folder != Some(fi) { + running = 0; + current_folder = Some(fi); + } let offset = running; running = running .checked_add(size) @@ -263,7 +474,7 @@ impl HeaderParser { } else { EntryKind::File }; - (kind, offset, size, false) + (kind, Some(fi), offset, size, false, crc) } else { let is_empty_file = empty_file.get(empty_index).copied().unwrap_or(false); let is_anti = anti_empty.get(empty_index).copied().unwrap_or(false); @@ -273,7 +484,7 @@ impl HeaderParser { } else { EntryKind::Dir }; - (kind, 0, 0, is_anti) + (kind, None, 0, 0, is_anti, None) }; let mode = permission_bits(full_mode, kind); @@ -290,36 +501,71 @@ impl HeaderParser { anti: is_anti, start_position: start_positions.get(i).copied().flatten(), has_stream, + folder_index, stream_offset: offset, size, + crc, }); } Ok(()) } } -enum SevenDecoder { - Lzma2(lzma_rust2::Lzma2Reader>), - Lzma(lzma_rust2::LzmaReader>), +/// One stage of a folder's decode chain, recursively wrapping the stage below it. The innermost +/// stage is the raw pack stream (`Source`); each coder in the folder's decode order wraps the +/// stage below with its own reader, keeping dispatch static (no trait objects). Stages are +/// LZMA/LZMA2 coders or decode-only byte filters (delta, BCJ); a folder is built as a chain only +/// when it is a linear graph of such coders over one pack stream. +enum SevenStage { + /// The base pack stream, bounded to the folder's packed bytes. + Source(Take), + /// An LZMA2 coder reading the stage below. + Lzma2(Box>>), + /// An LZMA (v1) coder reading the stage below. + Lzma(Box>>), + /// A delta decode filter reading the stage below. + Delta(Box, DeltaDecoder>>), + /// A BCJ decode filter reading the stage below. + Bcj(Box, BcjDecoder>>), + /// A general-purpose decompression coder (Deflate/BZip2/Zstd) reused from [`PipelineCodec`], + /// reading the stage below. + Pipeline(Box, PipelineCodec>>), + /// An AES-256/SHA-256 decryption coder reading the stage below. + #[cfg(feature = "aes")] + Aes(Box, crate::filter::aes7z::AesDecoder>>), } enum SevenInput { Source(R), - Decoder(Box>), + Decoder(Box>), } -impl SevenDecoder { +impl SevenStage { + /// Borrows the raw source at the bottom of the chain. fn source_ref(&self) -> &R { match self { - Self::Lzma2(reader) => reader.inner().get_ref(), - Self::Lzma(reader) => reader.inner().get_ref(), + Self::Source(take) => take.get_ref(), + Self::Lzma2(reader) => reader.inner().source_ref(), + Self::Lzma(reader) => reader.inner().source_ref(), + Self::Delta(reader) => reader.get_ref().source_ref(), + Self::Bcj(reader) => reader.get_ref().source_ref(), + Self::Pipeline(reader) => reader.get_ref().source_ref(), + #[cfg(feature = "aes")] + Self::Aes(reader) => reader.get_ref().source_ref(), } } - fn into_inner(self) -> R { + /// Unwinds the whole chain, reclaiming the raw source (and freeing every codec workspace). + fn into_source(self) -> R { match self { - Self::Lzma2(reader) => reader.into_inner().into_inner(), - Self::Lzma(reader) => reader.into_inner().into_inner(), + Self::Source(take) => take.into_inner(), + Self::Lzma2(reader) => reader.into_inner().into_source(), + Self::Lzma(reader) => reader.into_inner().into_source(), + Self::Delta(reader) => reader.into_inner().into_source(), + Self::Bcj(reader) => reader.into_inner().into_source(), + Self::Pipeline(reader) => reader.into_inner().into_source(), + #[cfg(feature = "aes")] + Self::Aes(reader) => reader.into_inner().into_source(), } } } @@ -331,13 +577,27 @@ impl SevenInput { Self::Decoder(decoder) => decoder.source_ref(), } } + + /// Reclaims the raw source, dropping any live folder decode chain (and its codec workspaces). + fn into_source(self) -> R { + match self { + Self::Source(source) => source, + Self::Decoder(decoder) => (*decoder).into_source(), + } + } } -impl Read for SevenDecoder { +impl Read for SevenStage { fn read(&mut self, output: &mut [u8]) -> std::io::Result { match self { + Self::Source(take) => take.read(output), Self::Lzma2(reader) => reader.read(output), Self::Lzma(reader) => reader.read(output), + Self::Delta(reader) => reader.read(output), + Self::Bcj(reader) => reader.read(output), + Self::Pipeline(reader) => reader.read(output), + #[cfg(feature = "aes")] + Self::Aes(reader) => reader.read(output), } } } @@ -352,17 +612,32 @@ enum SevenPhase { } /// Seek-capable 7z reader used by the opaque runtime dispatch. +/// +/// The reader owns the raw source through `input`; at most one folder decoder is live at a +/// time. `input` is `None` only transiently while [`SevenZSeekReader::set_active_folder`] +/// swaps the source between decoders, so the raw `R` is always reclaimable between calls. pub(crate) struct SevenZSeekReader { - input: SevenInput, + input: Option>, limits: Limits, + /// Zeroizing password for AES folders (`None` when the caller supplied none). Never logged. + password: Option, archive_metadata: Option, files: Vec, - folder: Option, + folders: Vec, + /// The folder whose decoder is currently live (`None` before the first content stream). + active_folder: Option, next_file: usize, phase: SevenPhase, event_data: Vec, + /// Decoded byte position within the currently active folder (reset on every folder switch). decoded_position: usize, decoded_total: u64, + /// Running CRC-32 and the stored digest for the open content entry, present only when the + /// entry declares a CRC and a decoder is live. Verified when the entry's data is exhausted. + entry_crc: Option<(crate::filter::Crc32, u32)>, + /// Whether the active folder contains an AES coder, so a CRC mismatch can be reported as a + /// likely wrong password rather than generic corruption. + active_folder_encrypted: bool, } impl std::fmt::Debug for SevenZSeekReader { @@ -378,39 +653,31 @@ impl std::fmt::Debug for SevenZSeekReader { } impl SevenZSeekReader { - pub(crate) fn new(mut input: R, limits: Limits) -> core::result::Result { - let (archive_metadata, files, folder) = parse_seek_layout(&mut input, limits)?; - validate_seek_layout(&files, folder.as_ref(), limits)?; - let input = match folder.as_ref().map(|folder| folder.coder) { - None | Some(FolderCoder::Unsupported) => SevenInput::Source(input), - Some(coder) => { - let folder = folder - .as_ref() - .ok_or_else(|| seven_error(ErrorKind::Protocol, "folder state disappeared"))?; - input - .seek(SeekFrom::Start(u64::try_from(folder.pack_offset).map_err( - |_| seven_error(ErrorKind::Limit, "pack offset exceeds u64"), - )?)) - .map_err(StreamError::io)?; - let take = input.take( - u64::try_from(folder.pack_size) - .map_err(|_| seven_error(ErrorKind::Limit, "pack size exceeds u64"))?, - ); - let decoder = build_seven_decoder(take, coder, folder.unpack_size, limits)?; - SevenInput::Decoder(Box::new(decoder)) - }, - }; + pub(crate) fn new( + mut input: R, + limits: Limits, + password: Option, + ) -> core::result::Result { + let (archive_metadata, files, folders) = + parse_seek_layout(&mut input, limits, password.as_ref())?; + validate_seek_layout(&files, &folders, limits)?; + // Folder decoders are built lazily, one at a time, when the first file of each folder is + // reached. That keeps exactly one LZMA/LZMA2 workspace resident regardless of folder count. Ok(Self { - input, + input: Some(SevenInput::Source(input)), limits, + password, archive_metadata: Some(archive_metadata), files, - folder, + folders, + active_folder: None, next_file: 0, phase: SevenPhase::Idle, event_data: Vec::with_capacity(64 * 1024), decoded_position: 0, decoded_total: 0, + entry_crc: None, + active_folder_encrypted: false, }) } @@ -423,7 +690,7 @@ impl SevenZSeekReader { match self.phase { SevenPhase::Idle => { let Some(record) = self.files.get(self.next_file).cloned() else { - self.verify_folder_end()?; + self.finalize_active_folder()?; self.phase = SevenPhase::Done; return Ok(ReaderEvent::Done); }; @@ -433,6 +700,8 @@ impl SevenZSeekReader { return Ok(ReaderEvent::Entry(metadata)); }, SevenPhase::Data { remaining: 0 } => { + // The whole substream has been delivered; the stored CRC-32 (if any) must match. + self.verify_entry_crc()?; self.phase = SevenPhase::EndEntry; }, SevenPhase::Data { remaining } => { @@ -446,6 +715,10 @@ impl SevenZSeekReader { )); } self.event_data.truncate(count); + // Fold the freshly decoded content into the running CRC (disjoint field borrows). + if let Some((crc, _)) = &mut self.entry_crc { + crc.update(&self.event_data[..count]); + } self.phase = SevenPhase::Data { remaining: remaining - count, }; @@ -479,8 +752,13 @@ impl SevenZSeekReader { "folder ended while skipping a substream", )); } + // Even a skipped substream is CRC-checked — the bytes are decoded anyway. + if let Some((crc, _)) = &mut self.entry_crc { + crc.update(&scratch[..count]); + } remaining -= count; } + self.verify_entry_crc()?; self.phase = SevenPhase::EndEntry; Ok(()) }, @@ -496,15 +774,21 @@ impl SevenZSeekReader { } } + // `input` is only `None` transiently inside `set_active_folder`; it is always `Some` at any + // point a caller can observe the reader, so the fallbacks below are unreachable in practice. + #[allow(clippy::expect_used)] pub(crate) fn into_inner(self) -> R { - match self.input { - SevenInput::Source(source) => source, - SevenInput::Decoder(decoder) => (*decoder).into_inner(), - } + self.input + .map(SevenInput::into_source) + .expect("7z reader source is present outside a folder switch") } + #[allow(clippy::expect_used)] pub(crate) fn source_ref(&self) -> &R { - self.input.source_ref() + self.input + .as_ref() + .map(SevenInput::source_ref) + .expect("7z reader source is present outside a folder switch") } fn prepare_record( @@ -512,11 +796,26 @@ impl SevenZSeekReader { _index: usize, record: &FileRec, ) -> core::result::Result { - if record.has_stream && self.has_decoder() { + // Move the live decoder onto this file's folder before touching its bytes. `has_decoder` + // is false when the folder's coder is unsupported (its payload is then not decodable). + let has_decoder = match (record.has_stream, record.folder_index) { + (true, Some(folder_index)) => self.set_active_folder(folder_index)?, + _ => false, + }; + if record.has_stream && has_decoder { self.drain_to(record.stream_offset)?; } + // Arm per-substream CRC verification for this entry's decoded content (drain bytes above are + // deliberately excluded). Absent a stored digest, decode proceeds unverified. + self.entry_crc = if record.has_stream && has_decoder { + record + .crc + .map(|expected| (crate::filter::Crc32::new(), expected)) + } else { + None + }; let mut link_target = None; - if record.has_stream && record.kind == EntryKind::Symlink && self.has_decoder() { + if record.has_stream && record.kind == EntryKind::Symlink && has_decoder { if self .limits .path_bytes() @@ -529,10 +828,14 @@ impl SevenZSeekReader { } let mut target = vec![0; record.size]; self.read_decoded_exact(&mut target)?; + if let Some((crc, _)) = &mut self.entry_crc { + crc.update(&target); + } + self.verify_entry_crc()?; link_target = Some(ArchivePath::from_encoded(target, PathEncoding::Utf8)); self.phase = SevenPhase::EndEntry; } else if record.has_stream { - self.phase = if self.has_decoder() { + self.phase = if has_decoder { SevenPhase::Data { remaining: record.size, } @@ -618,10 +921,10 @@ impl SevenZSeekReader { amount: usize, ) -> core::result::Result { let count = match &mut self.input { - SevenInput::Decoder(decoder) => decoder + Some(SevenInput::Decoder(decoder)) => decoder .read(&mut self.event_data[..amount]) .map_err(seven_decode_error)?, - SevenInput::Source(_) => { + _ => { return Err(seven_error( ErrorKind::Unsupported, "payload coder is unsupported", @@ -663,14 +966,40 @@ impl SevenZSeekReader { Ok(()) } - fn verify_folder_end(&mut self) -> core::result::Result<(), StreamError> { - let Some(folder) = &self.folder else { + /// Consumes the running per-substream CRC-32 (if armed) and compares it to the stored digest. + /// A mismatch on an encrypted folder is reported as a likely wrong password; otherwise as + /// generic corruption. Never a no-op silently: a `None` accumulator simply means the archive + /// stored no digest to check against. + fn verify_entry_crc(&mut self) -> core::result::Result<(), StreamError> { + if let Some((crc, expected)) = self.entry_crc.take() { + if crc.finalize() != expected { + let context = if self.active_folder_encrypted { + "wrong password or corrupt data" + } else { + "substream CRC-32 mismatch" + }; + return Err(seven_error(ErrorKind::Integrity, context)); + } + } + Ok(()) + } + + /// Verifies that the currently active folder decoded exactly to its declared size and + /// produced nothing past it. A no-op when no decoder is live (no folder yet, or an + /// unsupported coder). Called before switching folders and once the last file is consumed. + fn finalize_active_folder(&mut self) -> core::result::Result<(), StreamError> { + let Some(index) = self.active_folder else { return Ok(()); }; if !self.has_decoder() { return Ok(()); } - let expected = usize::try_from(folder.unpack_size) + let unpack_size = self + .folders + .get(index) + .ok_or_else(|| seven_error(ErrorKind::Protocol, "active folder index out of range"))? + .unpack_size; + let expected = usize::try_from(unpack_size) .map_err(|_| seven_error(ErrorKind::Limit, "folder size exceeds address space"))?; if self.decoded_position != expected { return Err(seven_error( @@ -688,14 +1017,81 @@ impl SevenZSeekReader { Ok(()) } - const fn has_decoder(&self) -> bool { - matches!(&self.input, SevenInput::Decoder(_)) + /// Makes `folder_index` the active folder, building its decoder if the current one is a + /// different folder. The previous folder is finalized first, then its decoder dropped so + /// only one codec workspace is ever resident. Returns whether a decoder is now live + /// (`false` when the folder's coder is unsupported and its payload cannot be decoded). + fn set_active_folder( + &mut self, + folder_index: usize, + ) -> core::result::Result { + if self.active_folder == Some(folder_index) { + return Ok(self.has_decoder()); + } + self.finalize_active_folder()?; + // Read the folder's decode parameters (all `Copy`), releasing the `self.folders` borrow + // before touching `self.input`. + let folder = self + .folders + .get(folder_index) + .ok_or_else(|| seven_error(ErrorKind::Protocol, "folder index out of range"))?; + let decodable = folder.is_decodable(); + let encrypted = folder.is_encrypted(); + let pack_offset = folder.pack_offset; + let pack_size = folder.pack_sizes.first().copied(); + // Reject a missing password before reclaiming the source, so the raw `R` stays reachable + // (via `into_inner`/`source_ref`) after the error. The secret is never touched here. + if decodable && encrypted && self.password.is_none() { + return Err(seven_error( + ErrorKind::Unsupported, + "7z AES entry requires a password", + )); + } + let mut source = self.take_source()?; + self.decoded_position = 0; + self.active_folder = Some(folder_index); + self.active_folder_encrypted = encrypted; + if !decodable { + self.input = Some(SevenInput::Source(source)); + return Ok(false); + } + // A decodable folder is a linear chain over exactly one pack stream, so its size is present. + let pack_size = pack_size.ok_or_else(|| { + seven_error(ErrorKind::Protocol, "decodable folder has no pack stream") + })?; + let offset = u64::try_from(pack_offset) + .map_err(|_| seven_error(ErrorKind::Limit, "pack offset exceeds u64"))?; + let size = u64::try_from(pack_size) + .map_err(|_| seven_error(ErrorKind::Limit, "pack size exceeds u64"))?; + if let Err(error) = source.seek(SeekFrom::Start(offset)) { + // Keep the source reachable so `into_inner`/`source_ref` stay valid after the error. + self.input = Some(SevenInput::Source(source)); + return Err(StreamError::io(error)); + } + let take = source.take(size); + let folder = &self.folders[folder_index]; + let decoder = build_folder_reader(take, folder, self.limits, self.password.as_ref())?; + self.input = Some(SevenInput::Decoder(Box::new(decoder))); + Ok(true) + } + + /// Reclaims the raw source, dropping any live folder decoder. Leaves `input` as `None` + /// for the caller to restore before returning. + fn take_source(&mut self) -> core::result::Result { + self.input + .take() + .map(SevenInput::into_source) + .ok_or_else(|| seven_error(ErrorKind::Protocol, "7z reader source is unavailable")) + } + + fn has_decoder(&self) -> bool { + matches!(&self.input, Some(SevenInput::Decoder(_))) } - fn decoder_mut(&mut self) -> Option<&mut SevenDecoder> { + fn decoder_mut(&mut self) -> Option<&mut SevenStage> { match &mut self.input { - SevenInput::Source(_) => None, - SevenInput::Decoder(decoder) => Some(decoder.as_mut()), + Some(SevenInput::Decoder(decoder)) => Some(decoder.as_mut()), + _ => None, } } } @@ -703,7 +1099,9 @@ impl SevenZSeekReader { fn parse_seek_layout( input: &mut (impl Read + Seek), limits: Limits, -) -> core::result::Result<(ArchiveMetadata, Vec, Option), StreamError> { + password: Option<&SecretBytes>, +) -> core::result::Result<(ArchiveMetadata, Vec, Vec), StreamError> { + #![allow(clippy::too_many_lines)] let image_length = input.seek(SeekFrom::End(0)).map_err(StreamError::io)?; input.seek(SeekFrom::Start(0)).map_err(StreamError::io)?; let mut signature = [0_u8; SIGNATURE_HEADER_SIZE]; @@ -722,7 +1120,7 @@ fn parse_seek_layout( let header_size = u64_le(&signature, 20).map_err(seven_legacy_error)?; let header_crc = u32_le(&signature, 28).map_err(seven_legacy_error)?; if header_size == 0 { - return Ok((ArchiveMetadata::new(), Vec::new(), None)); + return Ok((ArchiveMetadata::new(), Vec::new(), Vec::new())); } if limits .metadata_bytes() @@ -766,14 +1164,22 @@ fn parse_seek_layout( Some(K_HEADER) => header, Some(K_ENCODED_HEADER) => { let mut encoded = ByteReader::new(&header[1..]); - let folder = parse_streams_info(&mut encoded).map_err(seven_legacy_error)?; - if matches!(folder.coder, FolderCoder::Unsupported) { + let folders = parse_streams_info(&mut encoded).map_err(seven_legacy_error)?; + // The encoded next-header is itself decoded through the same folder machinery, but + // mainstream 7-Zip always writes it as a single folder; keep that restriction here. + let [folder] = folders.as_slice() else { + return Err(seven_error( + ErrorKind::Unsupported, + "encoded header must be a single folder", + )); + }; + if !folder.is_decodable() { return Err(seven_error( ErrorKind::Unsupported, "encoded-header coder is unsupported", )); } - decode_seek_header(input, &folder, limits, image_length)? + decode_seek_header(input, folder, limits, image_length, password)? }, _ => { return Err(seven_error( @@ -793,14 +1199,14 @@ fn parse_seek_layout( parser .parse_header(&mut bytes) .map_err(seven_legacy_error)?; - if let Some(folder) = &parser.folder { + for folder in &parser.folders { validate_folder_range(folder, image_length)?; } let archive_metadata = parser .header_extensions .into_iter() .fold(ArchiveMetadata::new(), ArchiveMetadata::with_extension); - Ok((archive_metadata, parser.files, parser.folder)) + Ok((archive_metadata, parser.files, parser.folders)) } fn decode_seek_header( @@ -808,6 +1214,7 @@ fn decode_seek_header( folder: &FolderInfo, limits: Limits, image_length: u64, + password: Option<&SecretBytes>, ) -> core::result::Result, StreamError> { if limits .metadata_bytes() @@ -819,16 +1226,20 @@ fn decode_seek_header( )); } validate_folder_range(folder, image_length)?; + let pack_size = + folder.pack_sizes.first().copied().ok_or_else(|| { + seven_error(ErrorKind::Unsupported, "encoded-header has no pack stream") + })?; input .seek(SeekFrom::Start(u64::try_from(folder.pack_offset).map_err( |_| seven_error(ErrorKind::Limit, "encoded-header offset exceeds u64"), )?)) .map_err(StreamError::io)?; let take = input.take( - u64::try_from(folder.pack_size) + u64::try_from(pack_size) .map_err(|_| seven_error(ErrorKind::Limit, "encoded-header pack size exceeds u64"))?, ); - let mut decoder = build_seven_decoder(take, folder.coder, folder.unpack_size, limits)?; + let mut decoder = build_folder_reader(take, folder, limits, password)?; let length = usize::try_from(folder.unpack_size) .map_err(|_| seven_error(ErrorKind::Limit, "decoded header exceeds address space"))?; let mut output = vec![0; length]; @@ -847,7 +1258,7 @@ fn decode_seek_header( fn validate_seek_layout( files: &[FileRec], - folder: Option<&FolderInfo>, + folders: &[FolderInfo], limits: Limits, ) -> core::result::Result<(), StreamError> { if limits @@ -893,27 +1304,31 @@ fn validate_seek_layout( "file metadata exceeds configured limit", )); } - if let Some(folder) = folder { - if limits - .decoded_total() - .is_some_and(|maximum| folder.unpack_size > maximum) - { - return Err(seven_error( - ErrorKind::Limit, - "folder output exceeds decoded-total limit", - )); - } - let total = files - .iter() - .filter(|file| file.has_stream) - .try_fold(0usize, |sum, file| sum.checked_add(file.size)) - .ok_or_else(|| seven_error(ErrorKind::Malformed, "substream size sum overflow"))?; - if u64::try_from(total).ok() != Some(folder.unpack_size) { - return Err(seven_error( - ErrorKind::Malformed, - "substream sizes do not equal folder size", - )); - } + // The sum of every folder's uncompressed output is what the reader will ever decode; bound it + // against the decoded-total budget, and confirm it equals the total of the file substreams. + let total_unpack = folders + .iter() + .try_fold(0u64, |sum, folder| sum.checked_add(folder.unpack_size)) + .ok_or_else(|| seven_error(ErrorKind::Malformed, "folder size sum overflow"))?; + if limits + .decoded_total() + .is_some_and(|maximum| total_unpack > maximum) + { + return Err(seven_error( + ErrorKind::Limit, + "folder output exceeds decoded-total limit", + )); + } + let total_content = files + .iter() + .filter(|file| file.has_stream) + .try_fold(0usize, |sum, file| sum.checked_add(file.size)) + .ok_or_else(|| seven_error(ErrorKind::Malformed, "substream size sum overflow"))?; + if u64::try_from(total_content).ok() != Some(total_unpack) { + return Err(seven_error( + ErrorKind::Malformed, + "substream sizes do not equal folder size", + )); } Ok(()) } @@ -924,7 +1339,7 @@ fn validate_folder_range( ) -> core::result::Result<(), StreamError> { let start = u64::try_from(folder.pack_offset) .map_err(|_| seven_error(ErrorKind::Limit, "pack offset exceeds u64"))?; - let size = u64::try_from(folder.pack_size) + let size = u64::try_from(folder.pack_span()) .map_err(|_| seven_error(ErrorKind::Limit, "pack size exceeds u64"))?; if start.checked_add(size).is_none_or(|end| end > image_length) { return Err(seven_error( @@ -935,41 +1350,125 @@ fn validate_folder_range( Ok(()) } -fn build_seven_decoder( - input: Take, - coder: FolderCoder, +/// Folds a folder's linear decode chain over its single pack stream, wrapping each coder in +/// [`resolve_folder`] order (dependencies first) so the outermost stage yields the folder's final +/// output. Only linear LZMA/LZMA2 folders are buildable; anything else reports `Unsupported`, which +/// leaves the folder listable but not extractable. +fn build_folder_reader( + source: Take, + folder: &FolderInfo, + limits: Limits, + password: Option<&SecretBytes>, +) -> core::result::Result, StreamError> { + if !folder.is_decodable() { + return Err(seven_error( + ErrorKind::Unsupported, + "payload coder is unsupported", + )); + } + let mut stage = SevenStage::Source(source); + for &coder_index in &folder.decode_order { + let coder = folder.graph.coders[coder_index]; + // Each stage's uncompressed size is the size of the coder's (single) output stream. + let out_size = folder + .output_sizes + .get(folder.graph.out_base(coder_index)) + .copied() + .ok_or_else(|| seven_error(ErrorKind::Malformed, "coder output size missing"))?; + stage = wrap_coder(stage, coder.method, out_size, limits, password)?; + } + Ok(stage) +} + +/// Wraps `inner` with a single coder's reader, producing the next chain stage. +fn wrap_coder( + inner: SevenStage, + method: CoderMethod, unpack_size: u64, limits: Limits, -) -> core::result::Result, StreamError> { - match coder { - FolderCoder::Lzma2 { dict_prop } => { + #[cfg_attr(not(feature = "aes"), allow(unused_variables))] password: Option<&SecretBytes>, +) -> core::result::Result, StreamError> { + match method { + CoderMethod::Lzma2 { dict_prop } => { let dictionary = lzma2_dict_size(dict_prop).map_err(seven_legacy_error)?; validate_dictionary(dictionary, limits)?; - Ok(SevenDecoder::Lzma2(lzma_rust2::Lzma2Reader::new( - input, dictionary, None, - ))) + Ok(SevenStage::Lzma2(Box::new(lzma_rust2::Lzma2Reader::new( + inner, dictionary, None, + )))) }, - FolderCoder::Lzma { props } => { + CoderMethod::Lzma { props } => { let dictionary = u32::from_le_bytes([props[1], props[2], props[3], props[4]]); validate_dictionary(dictionary, limits)?; - Ok(SevenDecoder::Lzma( + Ok(SevenStage::Lzma(Box::new( lzma_rust2::LzmaReader::new_with_props( - input, + inner, unpack_size, props[0], dictionary, None, ) .map_err(|_| seven_error(ErrorKind::Malformed, "LZMA decoder setup failed"))?, - )) + ))) }, - FolderCoder::Unsupported => Err(seven_error( + CoderMethod::Filter(SevenFilter::Delta { distance }) => Ok(SevenStage::Delta(Box::new( + CodecReader::new(inner, DeltaDecoder::new(distance), "7z-delta"), + ))), + CoderMethod::Filter(SevenFilter::Bcj { kind, start }) => Ok(SevenStage::Bcj(Box::new( + CodecReader::new(inner, BcjDecoder::new(kind, start), "7z-bcj"), + ))), + CoderMethod::Filter(SevenFilter::Pipeline { filter }) => { + validate_pipeline_memory(filter, limits)?; + let codec = PipelineCodec::new(filter, limits).map_err(StreamError::archive)?; + Ok(SevenStage::Pipeline(Box::new(CodecReader::new( + inner, + codec, + "7z-pipeline", + )))) + }, + #[cfg(feature = "aes")] + CoderMethod::Aes(params) => { + // A password is mandatory to decode an AES folder. Its absence is a typed + // capability error; the secret itself is never placed in any message. + let password = password.ok_or_else(|| { + seven_error(ErrorKind::Unsupported, "7z AES entry requires a password") + })?; + let decoder = + crate::filter::aes7z::AesDecoder::new(params, unpack_size, password.expose()); + Ok(SevenStage::Aes(Box::new(CodecReader::new( + inner, decoder, "7z-aes", + )))) + }, + CoderMethod::Unsupported => Err(seven_error( ErrorKind::Unsupported, "payload coder is unsupported", )), } } +/// Bounds a reused [`PipelineCodec`] coder's working set against the configured codec-memory +/// budget, mirroring [`validate_dictionary`] for LZMA. Deflate has a fixed 32 KiB window and is +/// always within budget; `BZip2` and Zstd are refused when the budget cannot hold their workspace. +fn validate_pipeline_memory( + filter: FilterId, + limits: Limits, +) -> core::result::Result<(), StreamError> { + let Some(budget) = limits.codec_memory() else { + return Ok(()); + }; + let needed = match filter { + FilterId::Bzip2 => BZIP2_DECODE_WORKSPACE, + FilterId::Zstd => ZSTD_MIN_WINDOW, + _ => return Ok(()), + }; + if budget < needed { + return Err(seven_error( + ErrorKind::Limit, + "7z codec workspace exceeds codec-memory limit", + )); + } + Ok(()) +} + fn validate_dictionary(dictionary: u32, limits: Limits) -> core::result::Result<(), StreamError> { if limits .codec_memory() @@ -1011,36 +1510,52 @@ fn seven_error(kind: ErrorKind, context: &'static str) -> StreamError { ) } -/// Parses a `StreamsInfo` (`PackInfo`, `UnpackInfo`, `SubStreamsInfo`) restricted to one pack, one folder, -/// and one LZMA2 coder. -fn parse_streams_info(r: &mut ByteReader<'_>) -> Result { +/// Parses a `StreamsInfo` (`PackInfo`, `UnpackInfo`, `SubStreamsInfo`) into one [`FolderInfo`] +/// per folder. Each folder's full coder graph is parsed (any number of coders, bind pairs, and +/// packed streams) so listing works for every archive; `CodersUnpackSize` carries one size per +/// output stream across all folders, and pack streams lie back-to-back from the pack base, each +/// folder consuming as many as it has packed inputs. Decodability (linear LZMA/LZMA2) is resolved +/// separately at build time. +fn parse_streams_info(r: &mut ByteReader<'_>) -> Result> { + #![allow(clippy::too_many_lines)] let mut pack_pos = 0u64; - let mut pack_size: Option = None; - let mut coder: Option = None; - let mut unpack_size = 0u64; - let mut num_substreams = 1usize; - let mut substream_sizes: Option> = None; - // Whether the folder's UnpackInfo carried a CRC. Mainstream 7-Zip and `sevenz-rust2` do NOT put - // the folder CRC here — they store it only in SubStreamsInfo — so a single-substream folder still - // carries a digest in SubStreamsInfo. Tracking this is what makes real compressed headers parse. - let mut folder_has_crc = false; + let mut pack_sizes: Option> = None; + let mut graphs: Vec = Vec::new(); + // One uncompressed size per output stream, flat across every folder in order. + let mut output_sizes_all: Vec = Vec::new(); + let mut num_folders = 0usize; + // Per folder: its UnpackInfo-defined CRC (`None` when absent). Mainstream 7-Zip and `sevenz-rust2` + // do NOT put the folder CRC here — they store it only in SubStreamsInfo — so a single-substream + // folder still carries a digest there. Tracking this drives the SubStreamsInfo digest count and + // supplies the substream CRC for the single-substream + folder-CRC case. + let mut folder_crcs: Vec> = Vec::new(); + let mut substream_sizes: Option>> = None; + let mut substream_crcs: Option>>> = None; loop { match r.u8()? { K_END => break, K_PACK_INFO => { pack_pos = r.number()?; - let num_pack = r.number()?; - if num_pack != 1 { - return Err(HeaderError::Unsupported( - "7z: only a single pack stream is supported", + let num_pack = usize_of(r.number()?)?; + if num_pack > r.remaining() { + return Err(HeaderError::Malformed( + "7z: pack-stream count exceeds header size", )); } loop { match r.u8()? { K_END => break, - K_SIZE => pack_size = Some(r.number()?), - K_CRC => read_digests(r, 1)?, + K_SIZE => { + let mut sizes = Vec::with_capacity(num_pack); + for _ in 0..num_pack { + sizes.push(r.number()?); + } + pack_sizes = Some(sizes); + }, + K_CRC => { + let _ = read_digests(r, num_pack)?; + }, _ => { return Err(HeaderError::Unsupported( "7z: unsupported pack-info property", @@ -1053,25 +1568,39 @@ fn parse_streams_info(r: &mut ByteReader<'_>) -> Result { if r.u8()? != K_FOLDER { return Err(HeaderError::Malformed("7z: unpack info missing kFolder")); } - if r.number()? != 1 { - return Err(HeaderError::Unsupported( - "7z: only a single folder is supported", - )); - } + num_folders = usize_of(r.number()?)?; if r.u8()? != 0 { return Err(HeaderError::Unsupported("7z: external folder definitions")); } - coder = Some(read_folder(r)?); + if num_folders > r.remaining() { + return Err(HeaderError::Malformed( + "7z: folder count exceeds header size", + )); + } + graphs = Vec::with_capacity(num_folders); + for _ in 0..num_folders { + graphs.push(read_folder(r)?); + } if r.u8()? != K_CODERS_UNPACK_SIZE { return Err(HeaderError::Malformed("7z: missing coders-unpack-size")); } - unpack_size = r.number()?; + // One unpack size per output stream of every coder, in folder-then-coder order. + let total_outputs = graphs.iter().map(Folder::total_out).sum::(); + if total_outputs > r.remaining() { + return Err(HeaderError::Malformed( + "7z: output-stream count exceeds header size", + )); + } + output_sizes_all = Vec::with_capacity(total_outputs); + for _ in 0..total_outputs { + output_sizes_all.push(r.number()?); + } + folder_crcs = vec![None; num_folders]; loop { match r.u8()? { K_END => break, K_CRC => { - folder_has_crc = true; - read_digests(r, 1)?; + folder_crcs = read_digests(r, num_folders)?; }, _ => { return Err(HeaderError::Unsupported( @@ -1082,9 +1611,11 @@ fn parse_streams_info(r: &mut ByteReader<'_>) -> Result { } }, K_SUBSTREAMS_INFO => { - let (n, sizes) = parse_substreams_info(r, unpack_size, folder_has_crc)?; - num_substreams = n; + // SubStreamsInfo needs each folder's final-output (unpack) size. + let folder_unpack = folder_unpack_sizes(&graphs, &output_sizes_all)?; + let (sizes, crcs) = parse_substreams_info(r, &folder_unpack, &folder_crcs)?; substream_sizes = Some(sizes); + substream_crcs = Some(crcs); }, _ => { return Err(HeaderError::Unsupported( @@ -1094,112 +1625,510 @@ fn parse_streams_info(r: &mut ByteReader<'_>) -> Result { } } - let pack_size = pack_size.ok_or(HeaderError::Malformed("7z: missing pack size"))?; - let coder = coder.ok_or(HeaderError::Malformed("7z: missing coder properties"))?; + if num_folders == 0 { + return Err(HeaderError::Malformed("7z: missing folder definitions")); + } + if graphs.len() != num_folders { + return Err(HeaderError::Malformed("7z: inconsistent folder table")); + } + let total_outputs = graphs.iter().map(Folder::total_out).sum::(); + if output_sizes_all.len() != total_outputs { + return Err(HeaderError::Malformed("7z: coder output-size mismatch")); + } + let folder_unpack = folder_unpack_sizes(&graphs, &output_sizes_all)?; + let pack_sizes = pack_sizes.ok_or(HeaderError::Malformed("7z: missing pack sizes"))?; + // Every folder draws one pack stream per packed input, so the pack streams must exactly + // cover every folder's packed inputs. + let total_packed = graphs + .iter() + .map(|folder| folder.packed_indices.len()) + .sum::(); + if pack_sizes.len() != total_packed { + return Err(HeaderError::Malformed( + "7z: pack-stream count does not match folder inputs", + )); + } let substream_sizes = match substream_sizes { - Some(s) => s, - None => vec![unpack_size], + Some(sizes) => sizes, + None => folder_unpack.iter().map(|&size| vec![size]).collect(), }; - let _ = num_substreams; - let pack_offset = SIGNATURE_HEADER_SIZE + if substream_sizes.len() != num_folders { + return Err(HeaderError::Malformed( + "7z: substream folder-count mismatch", + )); + } + // Absent SubStreamsInfo: every folder is single-substream, its CRC (if any) taken from the + // folder-level digest. + let substream_crcs = match substream_crcs { + Some(crcs) => crcs, + None => folder_crcs.iter().map(|&crc| vec![crc]).collect(), + }; + if substream_crcs.len() != num_folders { + return Err(HeaderError::Malformed( + "7z: substream crc folder-count mismatch", + )); + } + + let base = SIGNATURE_HEADER_SIZE .checked_add(usize_of(pack_pos)?) .ok_or(HeaderError::Malformed("7z: pack offset overflow"))?; + let mut folders = Vec::with_capacity(num_folders); + let mut running_pack = base; + let mut pack_cursor = 0usize; + let mut out_cursor = 0usize; + for (index, graph) in graphs.into_iter().enumerate() { + let packed_count = graph.packed_indices.len(); + let total_out = graph.total_out(); + // This folder's pack streams and their absolute offsets. + let pack_offset = running_pack; + let mut folder_pack_sizes = Vec::with_capacity(packed_count); + for &raw_pack_size in &pack_sizes[pack_cursor..pack_cursor + packed_count] { + let pack_size = usize_of(raw_pack_size)?; + folder_pack_sizes.push(pack_size); + running_pack = running_pack + .checked_add(pack_size) + .ok_or(HeaderError::Malformed("7z: pack offset overflow"))?; + } + pack_cursor += packed_count; + // This folder's per-output sizes. + let output_sizes = output_sizes_all[out_cursor..out_cursor + total_out].to_vec(); + out_cursor += total_out; + // Resolve the decode order (also validates cycles and stream overlaps). + let decode_order = resolve_folder(&graph)?; + folders.push(FolderInfo { + graph, + pack_offset, + pack_sizes: folder_pack_sizes, + unpack_size: folder_unpack[index], + output_sizes, + decode_order, + substream_sizes: substream_sizes[index].clone(), + substream_crcs: substream_crcs[index].clone(), + }); + } + Ok(folders) +} - Ok(FolderInfo { - pack_offset, - pack_size: usize_of(pack_size)?, - unpack_size, - coder, - substream_sizes, - }) +/// Each folder's final-output (uncompressed) size, drawn from the flat per-output-stream sizes. +fn folder_unpack_sizes(graphs: &[Folder], output_sizes_all: &[u64]) -> Result> { + let mut sizes = Vec::with_capacity(graphs.len()); + let mut out_cursor = 0usize; + for graph in graphs { + let total_out = graph.total_out(); + let slice = output_sizes_all + .get(out_cursor..out_cursor + total_out) + .ok_or(HeaderError::Malformed("7z: coder output-size mismatch"))?; + let final_out = graph.final_out_index()?; + let size = *slice + .get(final_out) + .ok_or(HeaderError::Malformed("7z: final output size missing"))?; + sizes.push(size); + out_cursor += total_out; + } + Ok(sizes) } -/// Parses a single folder definition, returning its coder (LZMA2 or plain LZMA) and properties. -fn read_folder(r: &mut ByteReader<'_>) -> Result { - if r.number()? != 1 { - return Err(HeaderError::Unsupported( - "7z: only a single coder is supported", +/// Parses a single folder's coder graph: its coders, bind pairs, and packed-stream indices. +fn read_folder(r: &mut ByteReader<'_>) -> Result { + let num_coders = usize_of(r.number()?)?; + if num_coders == 0 { + return Err(HeaderError::Malformed("7z: folder has no coders")); + } + if num_coders > r.remaining() { + return Err(HeaderError::Malformed( + "7z: coder count exceeds header size", )); } + let mut coders = Vec::with_capacity(num_coders); + for _ in 0..num_coders { + coders.push(read_coder(r)?); + } + let total_in = coders.iter().map(|coder| coder.num_in).sum::(); + let total_out = coders.iter().map(|coder| coder.num_out).sum::(); + if total_out == 0 { + return Err(HeaderError::Malformed("7z: folder has no output streams")); + } + // A folder has (numOutStreams - 1) bind pairs, one per non-final output stream. + let num_bind_pairs = total_out - 1; + if num_bind_pairs > r.remaining() { + return Err(HeaderError::Malformed( + "7z: bind-pair count exceeds header size", + )); + } + let mut bind_pairs = Vec::with_capacity(num_bind_pairs); + for _ in 0..num_bind_pairs { + let in_index = usize_of(r.number()?)?; + let out_index = usize_of(r.number()?)?; + bind_pairs.push(BindPair { + in_index, + out_index, + }); + } + let num_packed = total_in + .checked_sub(num_bind_pairs) + .filter(|&packed| packed != 0) + .ok_or(HeaderError::Malformed( + "7z: folder has no packed input streams", + ))?; + let packed_indices = if num_packed == 1 { + // The single packed stream is implicit: the one input not consumed by a bind pair. + let index = (0..total_in) + .find(|input| !bind_pairs.iter().any(|pair| pair.in_index == *input)) + .ok_or(HeaderError::Malformed("7z: no free packed input stream"))?; + vec![index] + } else { + let mut indices = Vec::with_capacity(num_packed); + for _ in 0..num_packed { + indices.push(usize_of(r.number()?)?); + } + indices + }; + Ok(Folder { + coders, + bind_pairs, + packed_indices, + }) +} + +/// Parses a single coder: its method id, stream arity, and properties. +fn read_coder(r: &mut ByteReader<'_>) -> Result { let flags = r.u8()?; let id_size = usize::from(flags & 0x0F); let is_complex = flags & 0x10 != 0; let has_attributes = flags & 0x20 != 0; if flags & 0x80 != 0 { - return Err(HeaderError::Unsupported("7z: reserved coder flag set")); - } - let codec = r.bytes(id_size)?; - if is_complex { return Err(HeaderError::Unsupported( - "7z: complex coders are not supported", + "7z: alternative coder methods are unsupported", )); } - if !has_attributes { - return Err(HeaderError::Unsupported("7z: coder without properties")); - } - let prop_size = usize_of(r.number()?)?; - let props = r.bytes(prop_size)?; + // Copy the method id out before further reads reborrow the byte reader. + let codec = r.bytes(id_size)?.to_vec(); + let (num_in, num_out) = if is_complex { + let num_in = usize_of(r.number()?)?; + let num_out = usize_of(r.number()?)?; + // Bound the stream arity against the remaining header so it cannot force a huge allocation. + if num_in > r.remaining() || num_out > r.remaining() { + return Err(HeaderError::Malformed( + "7z: coder stream count exceeds header size", + )); + } + if num_in == 0 || num_out == 0 { + return Err(HeaderError::Malformed( + "7z: coder has no input or output streams", + )); + } + (num_in, num_out) + } else { + (1, 1) + }; + let props = if has_attributes { + let prop_size = usize_of(r.number()?)?; + r.bytes(prop_size)?.to_vec() + } else { + Vec::new() + }; + Ok(Coder { + method: classify_method(&codec, &props)?, + num_in, + num_out, + }) +} + +/// Maps a coder method id and its properties onto a [`CoderMethod`]. LZMA2 and LZMA decode; every +/// other id lists as [`CoderMethod::Unsupported`]. +fn classify_method(codec: &[u8], props: &[u8]) -> Result { if codec == [METHOD_LZMA2] { - if prop_size != 1 { + if props.len() != 1 { return Err(HeaderError::Unsupported( "7z: unexpected LZMA2 property size", )); } - Ok(FolderCoder::Lzma2 { + Ok(CoderMethod::Lzma2 { dict_prop: props[0], }) } else if codec == METHOD_LZMA { - if prop_size != 5 { + if props.len() != 5 { return Err(HeaderError::Unsupported( "7z: unexpected LZMA property size", )); } let mut p = [0u8; 5]; p.copy_from_slice(props); - Ok(FolderCoder::Lzma { props: p }) + Ok(CoderMethod::Lzma { props: p }) + } else if codec == [METHOD_DELTA] { + // The delta filter stores one property byte: `distance - 1`. An absent + // property (rare) means distance 1; any other length is not a valid delta. + let distance = match props { + [] => 1, + [prop] => usize::from(*prop) + 1, + _ => return Ok(CoderMethod::Unsupported), + }; + Ok(CoderMethod::Filter(SevenFilter::Delta { distance })) + } else if let Some(kind) = bcj_branch_kind(codec) { + // BCJ filters carry either no property or a 4-byte little-endian start offset. + let start = match props { + [] => 0, + [a, b, c, d] => usize_of(u64::from(u32::from_le_bytes([*a, *b, *c, *d])))?, + _ => return Ok(CoderMethod::Unsupported), + }; + Ok(CoderMethod::Filter(SevenFilter::Bcj { kind, start })) + } else if codec == METHOD_DEFLATE { + // Raw DEFLATE, always available under the `sevenz` feature (it shares the gzip codec's + // `miniz_oxide` core). The coder carries no decode-relevant properties. + Ok(CoderMethod::Filter(SevenFilter::Pipeline { + filter: FilterId::Deflate, + })) + } else if codec == METHOD_BZIP2 { + // BZip2 lists on every build but only decodes when the `bzip2` codec is compiled in. + #[cfg(feature = "bzip2")] + { + Ok(CoderMethod::Filter(SevenFilter::Pipeline { + filter: FilterId::Bzip2, + })) + } + #[cfg(not(feature = "bzip2"))] + { + Ok(CoderMethod::Unsupported) + } + } else if codec == METHOD_ZSTD { + // Zstd lists on every build but only decodes when the `zstd` codec is compiled in. + #[cfg(feature = "zstd")] + { + Ok(CoderMethod::Filter(SevenFilter::Pipeline { + filter: FilterId::Zstd, + })) + } + #[cfg(not(feature = "zstd"))] + { + Ok(CoderMethod::Unsupported) + } + } else if codec == METHOD_AES256 { + // AES-256/SHA-256 lists on every build but only decodes when the `aes` feature is + // compiled in and the caller supplies a password. Properties that name an external + // key or an out-of-range work factor parse as `Unsupported` (listable, not decodable). + #[cfg(feature = "aes")] + { + match crate::filter::aes7z::AesParams::parse(props) { + Some(params) => Ok(CoderMethod::Aes(params)), + None => Ok(CoderMethod::Unsupported), + } + } + #[cfg(not(feature = "aes"))] + { + Ok(CoderMethod::Unsupported) + } + } else { + Ok(CoderMethod::Unsupported) + } +} + +/// Maps a BCJ coder method id onto its [`BranchKind`], or `None` if it is not a BCJ filter. +fn bcj_branch_kind(codec: &[u8]) -> Option { + if codec == METHOD_BCJ_X86 { + Some(BranchKind::X86) + } else if codec == METHOD_BCJ_ARM { + Some(BranchKind::Arm) + } else if codec == METHOD_BCJ_ARMT { + Some(BranchKind::ArmThumb) + } else if codec == [METHOD_BCJ_ARM64] { + Some(BranchKind::Arm64) + } else if codec == METHOD_BCJ_PPC { + Some(BranchKind::Ppc) + } else if codec == METHOD_BCJ_SPARC { + Some(BranchKind::Sparc) + } else if codec == METHOD_BCJ_IA64 { + Some(BranchKind::Ia64) + } else if codec == [METHOD_BCJ_RISCV] { + Some(BranchKind::RiscV) } else { - Ok(FolderCoder::Unsupported) + None + } +} + +/// Resolves a folder's coder graph into a topological decode order (coder indices, +/// dependencies first). Detects cycles (DFS in-progress marks) and stream overlaps — an input fed +/// by more than one source, an input fed by none, or an output bound more than once — all reported +/// as typed `Malformed`. The order is computed for every folder; decodability is a build-time +/// concern handled by [`build_folder_reader`]. +fn resolve_folder(folder: &Folder) -> Result> { + let total_in = folder.total_in(); + let total_out = folder.total_out(); + // Each input stream must have exactly one source; each output at most one consumer. + let mut in_sources = vec![0usize; total_in]; + let mut out_consumers = vec![0usize; total_out]; + for pair in &folder.bind_pairs { + if pair.in_index >= total_in || pair.out_index >= total_out { + return Err(HeaderError::Malformed( + "7z: bind pair references a missing stream", + )); + } + in_sources[pair.in_index] += 1; + out_consumers[pair.out_index] += 1; + } + for &packed in &folder.packed_indices { + if packed >= total_in { + return Err(HeaderError::Malformed( + "7z: packed stream references a missing input", + )); + } + in_sources[packed] += 1; + } + if in_sources.iter().any(|&count| count != 1) { + return Err(HeaderError::Malformed( + "7z: overlapping or unfed folder input stream", + )); + } + if out_consumers.iter().any(|&count| count > 1) { + return Err(HeaderError::Malformed( + "7z: overlapping folder output stream", + )); + } + let final_out = folder.final_out_index()?; + let sink = folder + .coder_of_out(final_out) + .ok_or(HeaderError::Malformed("7z: final output has no coder"))?; + + let mut marks = vec![DfsMark::Unvisited; folder.coders.len()]; + let mut order = Vec::with_capacity(folder.coders.len()); + visit_coder(folder, sink, &mut marks, &mut order)?; + Ok(order) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum DfsMark { + Unvisited, + InProgress, + Done, +} + +/// Post-order DFS over the coder graph: a coder is emitted after every coder feeding its inputs, +/// so the resulting order runs dependencies first. Revisiting an in-progress coder is a cycle. +fn visit_coder( + folder: &Folder, + coder: usize, + marks: &mut [DfsMark], + order: &mut Vec, +) -> Result<()> { + match marks[coder] { + DfsMark::Done => return Ok(()), + DfsMark::InProgress => return Err(HeaderError::Malformed("7z: cyclic coder graph")), + DfsMark::Unvisited => {}, + } + marks[coder] = DfsMark::InProgress; + let in_base = folder.in_base(coder); + for local in 0..folder.coders[coder].num_in { + let in_index = in_base + local; + // Bind-paired inputs depend on the producing coder; packed inputs are leaves. + if let Some(pair) = folder + .bind_pairs + .iter() + .find(|pair| pair.in_index == in_index) + { + let producer = folder + .coder_of_out(pair.out_index) + .ok_or(HeaderError::Malformed("7z: bind pair output has no coder"))?; + visit_coder(folder, producer, marks, order)?; + } } + marks[coder] = DfsMark::Done; + order.push(coder); + Ok(()) } -/// Parses a `SubStreamsInfo` block for the single folder, returning `(num_substreams, sizes)`. +/// Per-folder substream sizes paired with per-folder substream CRC-32s (`None` where undefined), +/// both indexed `[folder][substream]`. +type SubStreamTables = (Vec>, Vec>>); + +/// Parses a `SubStreamsInfo` block covering every folder, returning per-folder substream sizes and +/// per-folder substream CRC-32s (`None` where undefined). /// -/// `folder_has_crc` says whether the folder's `UnpackInfo` already defined a CRC: a single-substream -/// folder repeats no digest here only when it did, matching the 7z rule -/// `numDigests = Σ folders where (numSubstreams != 1 || !folderCrcDefined)`. +/// `folder_unpack[i]` is folder `i`'s uncompressed size and `folder_crcs[i]` its `UnpackInfo`-level +/// CRC (if any). The digest count follows the 7z rule +/// `numDigests = Σ_folders (numSubstreams(i) == 1 && folderCrcDefined(i) ? 0 : numSubstreams(i))`; +/// a single-substream folder whose CRC is already defined at the folder level reuses that value and +/// contributes no digest here. Single-substream folders also omit their stored size (it equals the +/// folder size). +#[allow(clippy::too_many_lines)] fn parse_substreams_info( r: &mut ByteReader<'_>, - folder_unpack: u64, - folder_has_crc: bool, -) -> Result<(usize, Vec)> { - let mut num = 1usize; - let mut sizes: Vec = Vec::new(); - let mut have_sizes = false; + folder_unpack: &[u64], + folder_crcs: &[Option], +) -> Result { + let num_folders = folder_unpack.len(); + let mut counts: Vec = vec![1; num_folders]; + let mut sizes: Option>> = None; + let mut crcs: Option>>> = None; loop { match r.u8()? { K_END => break, - K_NUM_UNPACK_STREAM => num = usize_of(r.number()?)?, + K_NUM_UNPACK_STREAM => { + let mut parsed = Vec::with_capacity(num_folders); + for _ in 0..num_folders { + parsed.push(usize_of(r.number()?)?); + } + counts = parsed; + }, K_SIZE => { - let mut sum = 0u64; - for _ in 0..num.saturating_sub(1) { - let s = r.number()?; - sum = sum - .checked_add(s) - .ok_or(HeaderError::Malformed("7z: substream size overflow"))?; - sizes.push(s); + let mut per_folder = Vec::with_capacity(num_folders); + for (index, &folder_size) in folder_unpack.iter().enumerate() { + let count = counts[index]; + if count == 0 { + per_folder.push(Vec::new()); + continue; + } + // Only (count - 1) sizes are stored; each needs at least one header byte. + if count.saturating_sub(1) > r.remaining() { + return Err(HeaderError::Malformed( + "7z: substream count exceeds header size", + )); + } + let mut folder_sizes = Vec::with_capacity(count); + let mut sum = 0u64; + for _ in 0..count - 1 { + let size = r.number()?; + sum = sum + .checked_add(size) + .ok_or(HeaderError::Malformed("7z: substream size overflow"))?; + folder_sizes.push(size); + } + let last = folder_size + .checked_sub(sum) + .ok_or(HeaderError::Malformed("7z: substream sizes exceed folder"))?; + folder_sizes.push(last); + per_folder.push(folder_sizes); } - let last = folder_unpack - .checked_sub(sum) - .ok_or(HeaderError::Malformed("7z: substream sizes exceed folder"))?; - sizes.push(last); - have_sizes = true; + sizes = Some(per_folder); }, K_CRC => { - // A digest is present for every substream except the single-substream case whose CRC - // is already defined on the folder (then it is not repeated). - let unknown = if num == 1 && folder_has_crc { 0 } else { num }; - read_digests(r, unknown)?; + // A digest is present for every substream except a single-substream folder whose + // CRC is already defined on the folder (that one is not repeated here). + let unknown = counts + .iter() + .enumerate() + .fold(0usize, |total, (index, &n)| { + let already = n == 1 && folder_crcs.get(index).copied().flatten().is_some(); + total.saturating_add(if already { 0 } else { n }) + }); + let digests = read_digests(r, unknown)?; + // Distribute digests over substreams in folder order, reusing the folder-level CRC + // for a single-substream folder that carried one. + let mut per_folder = Vec::with_capacity(num_folders); + let mut next = 0usize; + for (index, &n) in counts.iter().enumerate() { + let folder_crc = folder_crcs.get(index).copied().flatten(); + if n == 1 && folder_crc.is_some() { + per_folder.push(vec![folder_crc]); + } else { + let mut folder = Vec::with_capacity(n); + for _ in 0..n { + folder.push(digests.get(next).copied().flatten()); + next += 1; + } + per_folder.push(folder); + } + } + crcs = Some(per_folder); }, _ => { return Err(HeaderError::Unsupported( @@ -1209,35 +2138,71 @@ fn parse_substreams_info( } } - if !have_sizes { - if num == 1 { - sizes = vec![folder_unpack]; - } else { - return Err(HeaderError::Malformed("7z: missing substream sizes")); + let sizes = if let Some(sizes) = sizes { + sizes + } else { + // No kSize: every folder must be single-substream, taking the whole folder as its size. + let mut per_folder = Vec::with_capacity(num_folders); + for (index, &folder_size) in folder_unpack.iter().enumerate() { + match counts[index] { + 0 => per_folder.push(Vec::new()), + 1 => per_folder.push(vec![folder_size]), + _ => return Err(HeaderError::Malformed("7z: missing substream sizes")), + } + } + per_folder + }; + // Absent a SubStreamsInfo kCRC block, a substream's CRC is only known when its folder carried a + // single-substream folder-level digest. + let crcs = crcs.unwrap_or_else(|| { + counts + .iter() + .enumerate() + .map(|(index, &n)| { + let folder_crc = folder_crcs.get(index).copied().flatten(); + if n == 1 && folder_crc.is_some() { + vec![folder_crc] + } else { + vec![None; n] + } + }) + .collect() + }); + for (index, folder_sizes) in sizes.iter().enumerate() { + if folder_sizes.len() != counts[index] { + return Err(HeaderError::Malformed("7z: substream count mismatch")); + } + if crcs[index].len() != counts[index] { + return Err(HeaderError::Malformed("7z: substream crc count mismatch")); } } - if sizes.len() != num { - return Err(HeaderError::Malformed("7z: substream count mismatch")); - } - Ok((num, sizes)) + Ok((sizes, crcs)) } /// Reads a `Digests` structure (`AllAreDefined` byte + optional bit vector + one CRC per defined -/// item), discarding the values after bounds-checking them. -fn read_digests(r: &mut ByteReader<'_>, count: usize) -> Result<()> { +/// item), returning the per-item CRC-32 where defined (`None` otherwise). Retaining the values lets +/// the reader verify each substream after decode — the check that distinguishes a correct 7z AES +/// password from a wrong one. +fn read_digests(r: &mut ByteReader<'_>, count: usize) -> Result>> { if count == 0 { - return Ok(()); + return Ok(Vec::new()); + } + if count > r.remaining() { + return Err(HeaderError::Malformed( + "7z: digest count exceeds header size", + )); } let all_defined = r.u8()?; let defined = if all_defined != 0 { - count + vec![true; count] } else { - r.bit_vector(count)?.iter().filter(|&&b| b).count() + r.bit_vector(count)? }; - for _ in 0..defined { - let _ = r.u32()?; + let mut out = Vec::with_capacity(count); + for &is_defined in &defined { + out.push(if is_defined { Some(r.u32()?) } else { None }); } - Ok(()) + Ok(out) } /// Parses the `kName` property: an external byte (must be 0) then null-terminated UTF-16LE names. diff --git a/libarchive_oxide/tests/fuzz_replay.rs b/libarchive_oxide/tests/fuzz_replay.rs index 23470b6..66a2884 100644 --- a/libarchive_oxide/tests/fuzz_replay.rs +++ b/libarchive_oxide/tests/fuzz_replay.rs @@ -154,7 +154,7 @@ fn arbitrary_seeds_uphold_invariants() { } } - assert_eq!(TARGETS.len(), 17, "all fuzz targets are wired"); + assert_eq!(TARGETS.len(), 18, "all fuzz targets are wired"); assert_eq!(runs, TARGETS.len() * LENGTHS.len() * STREAMS); } diff --git a/libarchive_oxide/tests/sevenz_differential.rs b/libarchive_oxide/tests/sevenz_differential.rs index 32d940d..ae27a89 100644 --- a/libarchive_oxide/tests/sevenz_differential.rs +++ b/libarchive_oxide/tests/sevenz_differential.rs @@ -19,7 +19,8 @@ clippy::expect_used, clippy::panic, clippy::doc_markdown, - clippy::many_single_char_names + clippy::many_single_char_names, + clippy::cast_possible_truncation )] use std::io::Cursor; @@ -30,6 +31,7 @@ use libarchive_oxide_core::{ArchivePath, EntryKind, EntryMetadata, FormatId, Lim use sevenz_rust2::{ ArchiveEntry, ArchiveReader as SevenReader, ArchiveWriter as SevenWriter, EncoderConfiguration, EncoderMethod, Password, SourceReader, + encoder_options::{DeltaOptions, EncoderOptions}, }; mod common; @@ -155,6 +157,78 @@ fn arca_reads_sevenz_rust2_lzma_folder() { assert_eq!(got, expected); } +/// Non-solid archives put every file in its own folder (its own pack stream and coder). `sevenz-rust2` +/// produces exactly that when each entry is pushed with the singular `push_archive_entry`. arca must +/// walk the resulting `Vec`, activating one decoder at a time, and reproduce every file's +/// bytes — the core of RM-303 step 1. Directory and empty-file entries (which carry no folder) are +/// interleaved to confirm the file->folder mapping skips them correctly. +#[test] +fn arca_reads_sevenz_rust2_multi_folder() { + let a = b"alpha folder payload\n".to_vec(); + let b = b"beta folder payload, repeated for compressibility\n".repeat(30); + let c = b"gamma folder payload with different bytes\n".repeat(12); + + let mut w = SevenWriter::new(Cursor::new(Vec::new())).unwrap(); + // A directory and an empty file carry no content stream, so they never open a folder. + w.push_archive_entry::<&[u8]>(ArchiveEntry::new_directory("pkg"), None) + .unwrap(); + w.push_archive_entry(ArchiveEntry::new_file("pkg/a.txt"), Some(a.as_slice())) + .unwrap(); + w.push_archive_entry(ArchiveEntry::new_file("pkg/b.txt"), Some(b.as_slice())) + .unwrap(); + w.push_archive_entry::<&[u8]>(ArchiveEntry::new_file("pkg/empty.txt"), None) + .unwrap(); + w.push_archive_entry(ArchiveEntry::new_file("pkg/c.txt"), Some(c.as_slice())) + .unwrap(); + let bytes = w.finish().unwrap().into_inner(); + + // Three distinct content files => three separate folders/blocks (non-solid). + let reader = SevenReader::new(Cursor::new(bytes.clone()), Password::empty()) + .expect("sevenz-rust2 opens its own multi-folder archive"); + assert!( + !reader.archive().is_solid, + "push_archive_entry should produce a non-solid (multi-folder) archive" + ); + assert!( + reader.archive().blocks.len() >= 3, + "expected one folder per content file, got {} folders", + reader.archive().blocks.len() + ); + + let got = read_with_arca(&bytes); + let expected = vec![ + EntryShape::new(b"pkg".to_vec(), EntryKind::Dir, Vec::new()), + EntryShape::new(b"pkg/a.txt".to_vec(), EntryKind::File, a.clone()), + EntryShape::new(b"pkg/b.txt".to_vec(), EntryKind::File, b.clone()), + EntryShape::new(b"pkg/empty.txt".to_vec(), EntryKind::File, Vec::new()), + EntryShape::new(b"pkg/c.txt".to_vec(), EntryKind::File, c.clone()), + ]; + assert_eq!(got, expected); +} + +/// The same non-solid multi-folder shape, but with the plain-LZMA coder: each file is its own +/// folder, so arca must tear down one `LzmaReader` and seek+build the next between entries. +#[test] +fn arca_reads_sevenz_rust2_multi_folder_lzma() { + let a = b"first lzma folder\n".repeat(8); + let b = b"second lzma folder, a good deal longer than the first\n".repeat(25); + + let mut w = SevenWriter::new(Cursor::new(Vec::new())).unwrap(); + w.set_content_methods(vec![EncoderConfiguration::new(EncoderMethod::LZMA)]); + w.push_archive_entry(ArchiveEntry::new_file("a.bin"), Some(a.as_slice())) + .unwrap(); + w.push_archive_entry(ArchiveEntry::new_file("b.bin"), Some(b.as_slice())) + .unwrap(); + let bytes = w.finish().unwrap().into_inner(); + + let got = read_with_arca(&bytes); + let expected = vec![ + EntryShape::new(b"a.bin".to_vec(), EntryKind::File, a.clone()), + EntryShape::new(b"b.bin".to_vec(), EntryKind::File, b.clone()), + ]; + assert_eq!(got, expected); +} + /// A compressed (`kEncodedHeader`) next header is what mainstream 7-Zip / `sevenz-rust2` emit once an /// archive carries more than a trivial number of entries: `sevenz-rust2` LZMA-compresses the header /// whenever that shrinks it. Enough long, repetitive names force that path, so this archive lands on @@ -200,3 +274,255 @@ fn arca_reads_sevenz_rust2_compressed_header() { assert_eq!(g.content(), e.1.as_slice()); } } + +/// Deterministic pseudo-random bytes; branch opcodes occur naturally so the BCJ filters transform. +fn pseudo_random(len: usize) -> Vec { + let mut state: u64 = 0xdead_beef_cafe_0007; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state >> 24) as u8 + }) + .collect() +} + +/// Encodes one file through `sevenz-rust2` with an explicit content-coder chain. The chain is given +/// innermost-first (the compressor, then the filters that wrap it), matching `set_content_methods`. +fn sevenz_with_methods(methods: Vec, name: &str, data: &[u8]) -> Vec { + let mut w = SevenWriter::new(Cursor::new(Vec::new())).unwrap(); + w.set_content_methods(methods); + w.push_archive_entry(ArchiveEntry::new_file(name), Some(data)) + .unwrap(); + w.finish().unwrap().into_inner() +} + +/// A folder whose coder graph chains the delta filter with LZMA2 must decode in arca: LZMA2 +/// decompresses the pack stream, then the delta stage reconstructs the original bytes. `sevenz-rust2` +/// (using `lzma_rust2`'s independent delta implementation) is the producer; arca is the consumer. +#[test] +fn arca_reads_sevenz_rust2_delta_lzma2() { + let data = pseudo_random(40_003); + for distance in [1u32, 4, 256] { + let methods = vec![ + EncoderConfiguration::new(EncoderMethod::LZMA2), + EncoderConfiguration::new(EncoderMethod::DELTA_FILTER) + .with_options(EncoderOptions::Delta(DeltaOptions::from_distance(distance))), + ]; + let bytes = sevenz_with_methods(methods, "delta.bin", &data); + // sevenz-rust2 must agree the archive is a valid delta+LZMA2 chain it can also read back. + let mut reader = SevenReader::new(Cursor::new(bytes.clone()), Password::empty()) + .expect("sevenz-rust2 opens its own delta+LZMA2 archive"); + assert_eq!(reader.read_file("delta.bin").unwrap(), data); + + let got = read_with_arca(&bytes); + let expected = vec![EntryShape::new( + b"delta.bin".to_vec(), + EntryKind::File, + data.clone(), + )]; + assert_eq!( + got, expected, + "arca delta+LZMA2 mismatch at distance {distance}" + ); + } +} + +/// The same shape with a BCJ branch filter: LZMA2 over a BCJ-filtered payload. arca must resolve the +/// two-coder graph, decompress LZMA2, then invert the branch transform. Exercised for the x86 +/// (stateful) filter and a couple of fixed-stride RISC families. +#[test] +fn arca_reads_sevenz_rust2_bcj_lzma2() { + let data = pseudo_random(60_011); + let filters = [ + ("x86", EncoderMethod::BCJ_X86_FILTER), + ("arm", EncoderMethod::BCJ_ARM_FILTER), + ("ppc", EncoderMethod::BCJ_PPC_FILTER), + ("sparc", EncoderMethod::BCJ_SPARC_FILTER), + ]; + for (label, method) in filters { + let methods = vec![ + EncoderConfiguration::new(EncoderMethod::LZMA2), + EncoderConfiguration::new(method), + ]; + let bytes = sevenz_with_methods(methods, "bcj.bin", &data); + let mut reader = SevenReader::new(Cursor::new(bytes.clone()), Password::empty()) + .unwrap_or_else(|e| panic!("sevenz-rust2 opens its own {label} archive: {e}")); + assert_eq!( + reader.read_file("bcj.bin").unwrap(), + data, + "{label} self-read" + ); + + let got = read_with_arca(&bytes); + let expected = vec![EntryShape::new( + b"bcj.bin".to_vec(), + EntryKind::File, + data.clone(), + )]; + assert_eq!(got, expected, "arca BCJ+LZMA2 mismatch for {label}"); + } +} + +/// A payload that mixes a highly compressible run with pseudo-random noise, so the general-purpose +/// coders below (Deflate/BZip2/Zstd) emit real, non-degenerate compressed folders. +fn mixed_payload() -> Vec { + let mut data = b"the quick brown fox jumps over the lazy dog\n".repeat(400); + data.extend(pseudo_random(20_000)); + data.extend(b"trailing repeated tail tail tail tail tail\n".repeat(120)); + data +} + +/// Round-trips one file through `sevenz-rust2` with a single content coder `method`, asserting the +/// producer can read its own archive and that arca reproduces the original bytes exactly. +#[track_caller] +fn assert_arca_reads_method(method: EncoderMethod, label: &str) { + let data = mixed_payload(); + let bytes = sevenz_with_methods( + vec![EncoderConfiguration::new(method)], + "payload.bin", + &data, + ); + let mut reader = SevenReader::new(Cursor::new(bytes.clone()), Password::empty()) + .unwrap_or_else(|e| panic!("sevenz-rust2 opens its own {label} archive: {e}")); + assert_eq!( + reader.read_file("payload.bin").unwrap(), + data, + "{label} self-read" + ); + + let got = read_with_arca(&bytes); + let expected = vec![EntryShape::new( + b"payload.bin".to_vec(), + EntryKind::File, + data.clone(), + )]; + assert_eq!(got, expected, "arca {label} decode mismatch"); +} + +/// The 7z Deflate coder (method `04 01 08`) is RAW DEFLATE — no gzip header, trailer, or checksum. +/// arca reuses the shared `PipelineCodec` raw-inflate arm (the same `miniz_oxide` core the gzip +/// decoder sits on) to decode it. Always available under the `sevenz` feature. +#[test] +fn arca_reads_sevenz_rust2_deflate() { + assert_arca_reads_method(EncoderMethod::DEFLATE, "deflate"); +} + +/// The 7z BZip2 coder (method `04 02 02`), reusing arca's existing BZip2 codec through the folder +/// coder graph. Gated on arca's `bzip2` feature; `sevenz-rust2`'s bzip2 support is on by default. +#[cfg(feature = "bzip2")] +#[test] +fn arca_reads_sevenz_rust2_bzip2() { + assert_arca_reads_method(EncoderMethod::BZIP2, "bzip2"); +} + +/// The 7z Zstd coder (method `04 F7 11 01`), reusing arca's existing Zstd codec (portable `ruzstd` +/// or native `compression-codecs`) through the folder coder graph. Gated on arca's `zstd` feature. +#[cfg(feature = "zstd")] +#[test] +fn arca_reads_sevenz_rust2_zstd() { + assert_arca_reads_method(EncoderMethod::ZSTD, "zstd"); +} + +/// AES-256/SHA-256 (method `06 F1 07 01`) differential coverage. `sevenz-rust2` (with its default +/// `aes256` feature) produces a password-protected single-coder folder; arca decrypts it through the +/// coder graph, deriving the key with 7-Zip's own UTF-16LE + SHA-256 KDF, and verifies the stored +/// per-substream CRC-32 after decode. Gated on arca's `aes` feature. +#[cfg(feature = "aes")] +mod aes { + use libarchive_oxide::{ReaderEvent, SecretBytes, SeekArchiveReader, StreamError}; + use libarchive_oxide_core::{ArchiveError, ErrorKind}; + use sevenz_rust2::encoder_options::AesEncoderOptions; + + use super::*; + + /// Builds a `.7z` whose single content folder is AES-256/SHA-256 encrypted under `password`. + /// `encrypt_header` selects between `sevenz-rust2`'s default encrypted next-header (an AES-coded + /// `kEncodedHeader`, so even listing needs the password) and a plain header (so the encryption is + /// confined to the content folder — the shape that exercises the per-substream CRC check). + fn build_aes(password: &str, name: &str, data: &[u8], encrypt_header: bool) -> Vec { + let mut w = SevenWriter::new(Cursor::new(Vec::new())).unwrap(); + w.set_encrypt_header(encrypt_header); + let pw: Password = password.into(); + w.set_content_methods(vec![AesEncoderOptions::new(pw).into()]); + w.push_archive_entry(ArchiveEntry::new_file(name), Some(data)) + .unwrap(); + w.finish().unwrap().into_inner() + } + + /// Reads the whole (single-entry) archive's content through arca's seek reader with `password`. + fn read_aes(bytes: &[u8], password: &str) -> Result, StreamError> { + let mut reader = SeekArchiveReader::with_password( + Cursor::new(bytes.to_vec()), + SecretBytes::new(password.as_bytes().to_vec()), + )?; + let mut content = Vec::new(); + loop { + match reader.next_event()? { + ReaderEvent::Data(d) => content.extend_from_slice(d), + ReaderEvent::Done => return Ok(content), + _ => {}, + } + } + } + + /// The correct password (including a non-ASCII code point, to exercise the UTF-16LE encoding) + /// decrypts to the original plaintext, matching what `sevenz-rust2` reads back from its own file. + /// Uses the default *encrypted* next-header, so this also proves the AES-coded `kEncodedHeader` + /// path decodes with the caller's password. + #[test] + fn correct_password_decodes_encrypted_header_and_content() { + let data = mixed_payload(); + let password = "córrèct-horse-battery-\u{1F510}"; + let bytes = build_aes(password, "secret.bin", &data, true); + + // The independent producer must read its own archive with the same password. + let mut reference = + SevenReader::new(Cursor::new(bytes.clone()), Password::from(password)).unwrap(); + assert_eq!(reference.read_file("secret.bin").unwrap(), data); + + assert_eq!(read_aes(&bytes, password).unwrap(), data); + } + + /// A wrong password decrypts the content to garbage of the right length; the stored per-substream + /// CRC-32 mismatch is surfaced as a typed `Integrity` error rather than silently returning corrupt + /// bytes. A plain header keeps the failure at the content coder (the CRC path), not the header. + #[test] + fn wrong_password_trips_integrity() { + let data = b"top secret payload, not for prying eyes\n".repeat(16); + let bytes = build_aes("correct-horse", "s.bin", &data, false); + + // The fixture is sound: the correct password decodes it cleanly. + assert_eq!(read_aes(&bytes, "correct-horse").unwrap(), data); + + let error = read_aes(&bytes, "wrong-horse").expect_err("wrong password must fail"); + assert_eq!( + error.archive_error().map(ArchiveError::kind), + Some(ErrorKind::Integrity), + "wrong 7z AES password must be reported as an integrity failure" + ); + } + + /// Opening a plain-header AES archive without any password lists metadata but reports a typed + /// capability error (never a panic, never leaked secrets) when the encrypted content is reached. + #[test] + fn missing_password_is_unsupported() { + let data = b"needs a key to read\n".repeat(8); + let bytes = build_aes("pw", "s.bin", &data, false); + + let mut reader = SeekArchiveReader::new(Cursor::new(bytes.clone())).unwrap(); + let error = loop { + match reader.next_event() { + Ok(ReaderEvent::Done) => panic!("expected a typed error for the missing password"), + Ok(_) => {}, + Err(error) => break error, + } + }; + assert_eq!( + error.archive_error().map(ArchiveError::kind), + Some(ErrorKind::Unsupported), + "a missing 7z AES password must be a typed capability error" + ); + } +}