From 47e1f9e678e91f364f7069d6bcc1c61ebdbe991b Mon Sep 17 00:00:00 2001 From: alnfedorov <18511496+alnfedorov@users.noreply.github.com> Date: Sat, 27 Dec 2025 19:01:48 +0000 Subject: [PATCH 1/7] compress --- Cargo.toml | 4 + workspace/compress/src/adapter/boxed_sync.rs | 40 +++++++ workspace/compress/src/adapter/mod.rs | 2 + workspace/compress/src/decode/adapter.rs | 20 ++++ workspace/compress/src/decode/bgzf.rs | 72 +++++++++++++ workspace/compress/src/decode/decode.rs | 18 ++++ workspace/compress/src/decode/decoder.rs | 59 +++++++++++ workspace/compress/src/decode/deflate.rs | 81 ++++++++++++++ workspace/compress/src/decode/gzip.rs | 105 +++++++++++++++++++ workspace/compress/src/decode/identity.rs | 66 ++++++++++++ workspace/compress/src/decode/mod.rs | 27 +++++ workspace/compress/src/encode/adapter.rs | 12 +++ workspace/compress/src/encode/bgzf.rs | 64 +++++++++++ workspace/compress/src/encode/deflate.rs | 49 +++++++++ workspace/compress/src/encode/encode.rs | 6 ++ workspace/compress/src/encode/encoder.rs | 50 +++++++++ workspace/compress/src/encode/gzip.rs | 43 ++++++++ workspace/compress/src/encode/identity.rs | 17 +++ workspace/compress/src/encode/mod.rs | 27 +++++ workspace/compress/src/format.rs | 98 +++++++++++++++++ workspace/multilock/src/lock.rs | 2 +- workspace/multilock/src/multi_guard.rs | 2 +- 22 files changed, 862 insertions(+), 2 deletions(-) create mode 100644 workspace/compress/src/adapter/boxed_sync.rs create mode 100644 workspace/compress/src/adapter/mod.rs create mode 100644 workspace/compress/src/decode/adapter.rs create mode 100644 workspace/compress/src/decode/bgzf.rs create mode 100644 workspace/compress/src/decode/decode.rs create mode 100644 workspace/compress/src/decode/decoder.rs create mode 100644 workspace/compress/src/decode/deflate.rs create mode 100644 workspace/compress/src/decode/gzip.rs create mode 100644 workspace/compress/src/decode/identity.rs create mode 100644 workspace/compress/src/decode/mod.rs create mode 100644 workspace/compress/src/encode/adapter.rs create mode 100644 workspace/compress/src/encode/bgzf.rs create mode 100644 workspace/compress/src/encode/deflate.rs create mode 100644 workspace/compress/src/encode/encode.rs create mode 100644 workspace/compress/src/encode/encoder.rs create mode 100644 workspace/compress/src/encode/gzip.rs create mode 100644 workspace/compress/src/encode/identity.rs create mode 100644 workspace/compress/src/encode/mod.rs create mode 100644 workspace/compress/src/format.rs diff --git a/Cargo.toml b/Cargo.toml index efde9ee..114885a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,9 +13,13 @@ categories = ["science", "algorithms"] [workspace] members = [ + "workspace/compress", "workspace/multilock", ] resolver = "3" [workspace.dependencies] lock_api = { version = "0.4.14", default-features = false } +flate2 = { version = "1.1.5", features = ["zlib-rs"], default-features = false } +noodles-bgzf = { version = "0.45.0", default-features = false } +bitcode = { version = "0.6.9", features = ["derive"], default-features = false } diff --git a/workspace/compress/src/adapter/boxed_sync.rs b/workspace/compress/src/adapter/boxed_sync.rs new file mode 100644 index 0000000..8eaa332 --- /dev/null +++ b/workspace/compress/src/adapter/boxed_sync.rs @@ -0,0 +1,40 @@ +use crate::{decode, encode}; +use std::io::{BufRead, Error, Read}; + +pub struct BoxedSync; + +impl<'a, In> decode::AdaptRead<'a, In> for BoxedSync +where + In: Read + Send + Sync + 'a, +{ + fn wrap(self, internal: In) -> Result, Error> { + Ok(Box::new(internal)) + } +} + +impl<'a, In> decode::AdaptBufRead<'a, In> for BoxedSync +where + In: BufRead + Send + Sync + 'a, +{ + fn wrap(self, internal: In) -> Result, Error> { + Ok(Box::new(internal)) + } +} + +impl decode::Adapter for BoxedSync { + type Read<'a> = Box; + type BufRead<'a> = Box; +} + +impl<'a, In> encode::AdaptWrite<'a, In> for BoxedSync +where + In: std::io::Write + Send + Sync + 'a, +{ + fn wrap(self, internal: In) -> Result, Error> { + Ok(Box::new(internal)) + } +} + +impl encode::Adapter for BoxedSync { + type Write<'a> = Box; +} \ No newline at end of file diff --git a/workspace/compress/src/adapter/mod.rs b/workspace/compress/src/adapter/mod.rs new file mode 100644 index 0000000..a2e1539 --- /dev/null +++ b/workspace/compress/src/adapter/mod.rs @@ -0,0 +1,2 @@ +mod boxed_sync; +pub use boxed_sync::BoxedSync; diff --git a/workspace/compress/src/decode/adapter.rs b/workspace/compress/src/decode/adapter.rs new file mode 100644 index 0000000..6f21871 --- /dev/null +++ b/workspace/compress/src/decode/adapter.rs @@ -0,0 +1,20 @@ +use std::io::{BufRead, Error, Read}; + +pub trait AdaptRead<'a, In>: Adapter { + /// A helper trait to allow introducing flexible constraints on the wrapped reader. + /// + /// This is normally where you would put constraints on `In` and implement the actual wrapping logic. + fn wrap(self, internal: In) -> Result, Error>; +} + +pub trait AdaptBufRead<'a, In>: Adapter { + /// A helper trait to allow introducing flexible constraints on the wrapped buffered reader. + /// + /// This is normally where you would put constraints on `In` and implement the actual wrapping logic. + fn wrap(self, internal: In) -> Result, Error>; +} + +pub trait Adapter { + type Read<'a>: Read + 'a; + type BufRead<'a>: BufRead + 'a; +} diff --git a/workspace/compress/src/decode/bgzf.rs b/workspace/compress/src/decode/bgzf.rs new file mode 100644 index 0000000..27caab9 --- /dev/null +++ b/workspace/compress/src/decode/bgzf.rs @@ -0,0 +1,72 @@ +use super::adapter::{AdaptBufRead, AdaptRead, Adapter}; +use super::decode::{DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead}; +use std::io::{BufRead, BufReader, Error, Read}; +use std::num::NonZeroUsize; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Bgzf { + out_bufsize: NonZeroUsize, +} + +impl Bgzf { + pub const DEFAULT: Bgzf = Bgzf { + out_bufsize: NonZeroUsize::new(8 * 1024).unwrap(), + }; + + pub fn new(out_bufsize: NonZeroUsize) -> Self { + Self { out_bufsize } + } + + pub fn out_bufsize(&self) -> NonZeroUsize { + self.out_bufsize + } +} + +impl Default for Bgzf { + fn default() -> Self { + Self::DEFAULT + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoRead<'a, R, A> for Bgzf +where + A: AdaptRead<'a, flate2::read::MultiGzDecoder>, +{ + fn decode_read_into_read(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::read::MultiGzDecoder::new(reader); + adapter.wrap(reader) + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoBufRead<'a, R, A> for Bgzf +where + A: AdaptBufRead<'a, BufReader>>, +{ + fn decode_read_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::read::MultiGzDecoder::new(reader); + let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoRead<'a, R, A> for Bgzf +where + A: AdaptRead<'a, flate2::bufread::MultiGzDecoder>, +{ + fn decode_bufread_into_read(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::bufread::MultiGzDecoder::new(reader); + adapter.wrap(reader) + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoBufRead<'a, R, A> for Bgzf +where + A: AdaptBufRead<'a, BufReader>>, +{ + fn decode_bufread_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::bufread::MultiGzDecoder::new(reader); + let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } +} diff --git a/workspace/compress/src/decode/decode.rs b/workspace/compress/src/decode/decode.rs new file mode 100644 index 0000000..ca80dca --- /dev/null +++ b/workspace/compress/src/decode/decode.rs @@ -0,0 +1,18 @@ +use super::adapter::Adapter; +use std::io::{BufRead, Error, Read}; + +pub trait DecodeReadIntoRead<'a, R: Read + 'a, A: Adapter> { + fn decode_read_into_read(&self, reader: R, adapter: A) -> Result, Error>; +} + +pub trait DecodeReadIntoBufRead<'a, R: Read + 'a, A: Adapter> { + fn decode_read_into_bufread(&self, reader: R, adapter: A) -> Result, Error>; +} + +pub trait DecodeBufReadIntoRead<'a, R: BufRead + 'a, A: Adapter> { + fn decode_bufread_into_read(&self, reader: R, adapter: A) -> Result, Error>; +} + +pub trait DecodeBufReadIntoBufRead<'a, R: BufRead + 'a, A: Adapter> { + fn decode_bufread_into_bufread(&self, reader: R, adapter: A) -> Result, Error>; +} diff --git a/workspace/compress/src/decode/decoder.rs b/workspace/compress/src/decode/decoder.rs new file mode 100644 index 0000000..59bbcfd --- /dev/null +++ b/workspace/compress/src/decode/decoder.rs @@ -0,0 +1,59 @@ +use super::adapter::Adapter; +use super::decode::{DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead}; +use std::io::{BufRead, Error, Read}; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Decoder { + /// No compression (identity) decoding algorithm. + /// + /// The option is useful to uniformly handle both compressed and uncompressed data streams. + Identity(crate::decode::Identity), + + /// DEFLATE compression algorithm + #[cfg(feature = "decode-deflate")] + Deflate(crate::decode::Deflate), + + /// GZIP container; Only supports DEFLATE compression. + #[cfg(feature = "decode-gzip")] + Gzip(crate::decode::Gzip), + + /// BGZF container; Only supports DEFLATE compression. + #[cfg(feature = "decode-bgzf")] + Bgzf(crate::decode::Bgzf), +} + +impl Default for Decoder { + fn default() -> Self { + Decoder::Identity(Default::default()) + } +} + +macro_rules! impl_decompression_trait { + ($read:ident, $out:ident, $trait:ident, $method:ident) => { + impl<'a, R: $read + 'a, A: Adapter> $trait<'a, R, A> for Decoder + where + crate::decode::Identity: $trait<'a, R, A>, + crate::decode::Deflate: $trait<'a, R, A>, + crate::decode::Gzip: $trait<'a, R, A>, + crate::decode::Bgzf: $trait<'a, R, A>, + { + fn $method(&self, reader: R, adapter: A) -> Result, Error> { + match self { + Decoder::Identity(codec) => codec.$method(reader, adapter), + #[cfg(feature = "decode-deflate")] + Decoder::Deflate(codec) => codec.$method(reader, adapter), + #[cfg(feature = "decode-gzip")] + Decoder::Gzip(codec) => codec.$method(reader, adapter), + #[cfg(feature = "decode-bgzf")] + Decoder::Bgzf(codec) => codec.$method(reader, adapter), + } + } + } + }; +} + +impl_decompression_trait!(Read, Read, DecodeReadIntoRead, decode_read_into_read); +impl_decompression_trait!(Read, BufRead, DecodeReadIntoBufRead, decode_read_into_bufread); +impl_decompression_trait!(BufRead, Read, DecodeBufReadIntoRead, decode_bufread_into_read); +impl_decompression_trait!(BufRead, BufRead, DecodeBufReadIntoBufRead, decode_bufread_into_bufread); diff --git a/workspace/compress/src/decode/deflate.rs b/workspace/compress/src/decode/deflate.rs new file mode 100644 index 0000000..dad45b6 --- /dev/null +++ b/workspace/compress/src/decode/deflate.rs @@ -0,0 +1,81 @@ +use super::adapter::{AdaptBufRead, AdaptRead, Adapter}; +use super::decode::{DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead}; +use std::io; +use std::io::{BufRead, Error, Read}; +use std::num::NonZeroUsize; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Deflate { + in_bufsize: NonZeroUsize, + out_bufsize: NonZeroUsize, +} + +impl Deflate { + pub const DEFAULT: Deflate = Deflate { + in_bufsize: NonZeroUsize::new(32 * 1024).unwrap(), + out_bufsize: NonZeroUsize::new(8 * 1024).unwrap(), + }; + + pub fn new(deflate_bufsize: NonZeroUsize, out_bufsize: NonZeroUsize) -> Self { + Self { + in_bufsize: deflate_bufsize, + out_bufsize, + } + } + + pub fn in_bufsize(&self) -> usize { + self.in_bufsize.get() + } + + pub fn out_bufsize(&self) -> usize { + self.out_bufsize.get() + } +} + +impl Default for Deflate { + fn default() -> Self { + Deflate::DEFAULT + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoRead<'a, R, A> for Deflate +where + A: AdaptRead<'a, flate2::read::DeflateDecoder>, +{ + fn decode_read_into_read(&self, reader: R, adapter: A) -> Result, Error> { + adapter.wrap(flate2::read::DeflateDecoder::new(reader)) + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoBufRead<'a, R, A> for Deflate +where + A: AdaptBufRead<'a, io::BufReader>>, +{ + fn decode_read_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::read::DeflateDecoder::new(reader); + let bufreader = io::BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoRead<'a, R, A> for Deflate +where + A: AdaptRead<'a, flate2::bufread::DeflateDecoder>, +{ + fn decode_bufread_into_read(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::bufread::DeflateDecoder::new(reader); + adapter.wrap(reader) + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoBufRead<'a, R, A> for Deflate +where + A: AdaptBufRead<'a, io::BufReader>>, +{ + fn decode_bufread_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::bufread::DeflateDecoder::new(reader); + let bufreader = io::BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } +} \ No newline at end of file diff --git a/workspace/compress/src/decode/gzip.rs b/workspace/compress/src/decode/gzip.rs new file mode 100644 index 0000000..f7feede --- /dev/null +++ b/workspace/compress/src/decode/gzip.rs @@ -0,0 +1,105 @@ +use super::decode::{DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead}; +use super::adapter::{AdaptBufRead, AdaptRead, Adapter}; +use std::io::{BufRead, BufReader, Error, Read}; +use std::num::NonZeroUsize; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Gzip { + out_bufsize: NonZeroUsize, + read_multi: bool, +} + +impl Gzip { + pub const DEFAULT: Gzip = Gzip { + out_bufsize: NonZeroUsize::new(8 * 1024).unwrap(), + read_multi: true, + }; + + pub fn new(out_bufsize: NonZeroUsize, read_multi: bool) -> Self { + Self { out_bufsize, read_multi } + } + + pub fn out_bufsize(&self) -> NonZeroUsize { + self.out_bufsize + } + + pub fn read_multi(&self) -> bool { + self.read_multi + } +} + +impl Default for Gzip { + fn default() -> Self { + Self::DEFAULT + } +} + + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoRead<'a, R, A> for Gzip +where + A: AdaptRead<'a, flate2::read::GzDecoder>, + A: AdaptRead<'a, flate2::read::MultiGzDecoder>, +{ + fn decode_read_into_read(&self, reader: R, adapter: A) -> Result, Error> { + if self.read_multi { + let reader = flate2::read::MultiGzDecoder::new(reader); + adapter.wrap(reader) + } else { + let reader = flate2::read::GzDecoder::new(reader); + adapter.wrap(reader) + } + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoBufRead<'a, R, A> for Gzip +where + A: AdaptBufRead<'a, BufReader>>, + A: AdaptBufRead<'a, BufReader>>, +{ + fn decode_read_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + if self.read_multi { + let reader = flate2::read::MultiGzDecoder::new(reader); + let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } else { + let reader = flate2::read::GzDecoder::new(reader); + let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoRead<'a, R, A> for Gzip +where + A: AdaptRead<'a, flate2::bufread::GzDecoder>, + A: AdaptRead<'a, flate2::bufread::MultiGzDecoder>, +{ + fn decode_bufread_into_read(&self, reader: R, adapter: A) -> Result, Error> { + if self.read_multi { + let reader = flate2::bufread::MultiGzDecoder::new(reader); + adapter.wrap(reader) + } else { + let reader = flate2::bufread::GzDecoder::new(reader); + adapter.wrap(reader) + } + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoBufRead<'a, R, A> for Gzip +where + A: AdaptBufRead<'a, BufReader>>, + A: AdaptBufRead<'a, BufReader>>, +{ + fn decode_bufread_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + if self.read_multi { + let reader = flate2::bufread::MultiGzDecoder::new(reader); + let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } else { + let reader = flate2::bufread::GzDecoder::new(reader); + let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } + } +} diff --git a/workspace/compress/src/decode/identity.rs b/workspace/compress/src/decode/identity.rs new file mode 100644 index 0000000..4a44375 --- /dev/null +++ b/workspace/compress/src/decode/identity.rs @@ -0,0 +1,66 @@ +use super::adapter::{AdaptBufRead, AdaptRead, Adapter}; +use super::decode::{DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead}; +use std::io::{BufRead, BufReader, Error, Read}; +use std::num::NonZeroUsize; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Identity { + out_bufsize: NonZeroUsize, +} + +impl Identity { + pub const DEFAULT: Identity = Identity { + out_bufsize: NonZeroUsize::new(8 * 1024).unwrap() + }; + + pub fn new(out_bufsize: NonZeroUsize) -> Self { + Self { out_bufsize } + } + + pub fn out_bufsize(&self) -> NonZeroUsize { + self.out_bufsize + } +} +impl Default for Identity { + fn default() -> Self { + Self::DEFAULT + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoRead<'a, R, A> for Identity +where + A: AdaptRead<'a, R>, +{ + fn decode_read_into_read(&self, reader: R, adapter: A) -> Result, Error> { + adapter.wrap(reader) + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoBufRead<'a, R, A> for Identity +where + A: AdaptBufRead<'a, BufReader>, +{ + fn decode_read_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + adapter.wrap(BufReader::with_capacity(self.out_bufsize.get(), reader)) + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoRead<'a, R, A> for Identity +where + A: AdaptRead<'a, R>, +{ + fn decode_bufread_into_read(&self, reader: R, adapter: A) -> Result, Error> { + adapter.wrap(reader) + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoBufRead<'a, R, A> for Identity +where + A: AdaptBufRead<'a, R>, +{ + fn decode_bufread_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + adapter.wrap(reader) + } +} + diff --git a/workspace/compress/src/decode/mod.rs b/workspace/compress/src/decode/mod.rs new file mode 100644 index 0000000..97d4d67 --- /dev/null +++ b/workspace/compress/src/decode/mod.rs @@ -0,0 +1,27 @@ +mod adapter; +pub use adapter::{AdaptBufRead, AdaptRead, Adapter}; + +#[allow(clippy::module_inception)] +mod decode; +pub use decode::{DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead}; + +mod identity; +pub use identity::Identity; + +#[cfg(feature = "decode-deflate")] +mod deflate; +#[cfg(feature = "decode-deflate")] +pub use deflate::Deflate; + +#[cfg(feature = "decode-gzip")] +mod gzip; +#[cfg(feature = "decode-gzip")] +pub use gzip::Gzip; + +#[cfg(feature = "decode-bgzf")] +mod bgzf; +#[cfg(feature = "decode-bgzf")] +pub use bgzf::Bgzf; + +mod decoder; +pub use decoder::Decoder; diff --git a/workspace/compress/src/encode/adapter.rs b/workspace/compress/src/encode/adapter.rs new file mode 100644 index 0000000..7e6d352 --- /dev/null +++ b/workspace/compress/src/encode/adapter.rs @@ -0,0 +1,12 @@ +use std::io::{Error, Write}; + +pub trait AdaptWrite<'a, In>: Adapter { + /// A helper trait to allow introducing flexible constraints on the wrapped reader. + /// + /// This is normally where you would put constraints on `In` and implement the actual wrapping logic. + fn wrap(self, internal: In) -> Result, Error>; +} + +pub trait Adapter { + type Write<'a>: Write + 'a; +} diff --git a/workspace/compress/src/encode/bgzf.rs b/workspace/compress/src/encode/bgzf.rs new file mode 100644 index 0000000..77b1e4a --- /dev/null +++ b/workspace/compress/src/encode/bgzf.rs @@ -0,0 +1,64 @@ +use super::adapter::{AdaptWrite, Adapter}; +use super::encode::Encode; +use noodles_bgzf as bgzf; +use std::io; +use std::io::{Error, Write}; +use std::num::NonZeroUsize; + +use super::deflate::Deflate; + + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Bgzf { + deflate: Deflate, + threads: Option, +} + +impl Bgzf { + pub const DEFAULT: Bgzf = Bgzf { + deflate: Deflate::DEFAULT, + threads: None, + }; + + pub fn new(deflate: Deflate, threads: Option) -> io::Result { + Ok(Self { deflate, threads }) + } + + pub fn deflate(&self) -> &Deflate { + &self.deflate + } + + pub fn threads(&self) -> Option { + self.threads + } +} + +impl Default for Bgzf { + fn default() -> Self { + Bgzf::DEFAULT + } +} + +impl<'a, W: Write + 'a, A: Adapter> Encode<'a, W, A> for Bgzf +where + A: AdaptWrite<'a, bgzf::io::Writer>, +{ + fn encode(&self, writer: W, adapter: A) -> Result, Error> { + let level = + bgzf::io::writer::CompressionLevel::new(self.deflate().level()).unwrap(); + if let Some(_threads) = self.threads() { + Err(Error::other("Multithreaded BGZF is not supported")) + // let writer = bgzf::io::multithreaded_writer::Builder::default() + // .set_compression_level(level) + // .set_worker_count(threads) + // .build_from_writer(writer); + // adapter.wrap(writer) + } else { + let writer = bgzf::io::writer::Builder::default() + .set_compression_level(level) + .build_from_writer(writer); + adapter.wrap(writer) + } + } +} diff --git a/workspace/compress/src/encode/deflate.rs b/workspace/compress/src/encode/deflate.rs new file mode 100644 index 0000000..ad53c22 --- /dev/null +++ b/workspace/compress/src/encode/deflate.rs @@ -0,0 +1,49 @@ +use super::adapter::{AdaptWrite, Adapter}; +use super::encode::Encode; +use std::io; +use std::io::{Error, Write}; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Deflate { + level: u8, +} + +impl Deflate { + pub const NONE: Deflate = Deflate { level: 0 }; + pub const FAST: Deflate = Deflate { level: 1 }; + pub const DEFAULT: Deflate = Deflate { level: 6 }; + pub const BEST: Deflate = Deflate { level: 9 }; + + pub fn new(level: u8) -> io::Result { + if level > 9 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("Invalid DEFLATE compress level: {}", level), + )); + } + Ok(Self { level }) + } + + pub fn level(&self) -> u8 { + self.level + } +} + +impl Default for Deflate { + fn default() -> Self { + Deflate::DEFAULT + } +} + +impl<'a, W: Write + 'a, A: Adapter> Encode<'a, W, A> for Deflate +where + A: AdaptWrite<'a, flate2::write::DeflateEncoder>, +{ + fn encode(&self, writer: W, adapter: A) -> Result, Error> { + adapter.wrap(flate2::write::DeflateEncoder::new( + writer, + flate2::Compression::new(self.level as u32), + )) + } +} diff --git a/workspace/compress/src/encode/encode.rs b/workspace/compress/src/encode/encode.rs new file mode 100644 index 0000000..fcce8b8 --- /dev/null +++ b/workspace/compress/src/encode/encode.rs @@ -0,0 +1,6 @@ +use super::adapter::Adapter; +use std::io::{Error, Write}; + +pub trait Encode<'a, W: Write + 'a, A: Adapter> { + fn encode(&self, writer: W, adapter: A) -> Result, Error>; +} diff --git a/workspace/compress/src/encode/encoder.rs b/workspace/compress/src/encode/encoder.rs new file mode 100644 index 0000000..40fafe0 --- /dev/null +++ b/workspace/compress/src/encode/encoder.rs @@ -0,0 +1,50 @@ +use super::adapter::Adapter; +use super::encode::Encode; +use std::io::{Error, Write}; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Encoder { + /// No compression (identity) encoding algorithm. + /// + /// The option is useful to uniformly handle both compressed and uncompressed data streams. + Identity(crate::encode::Identity), + + /// DEFLATE compression algorithm + #[cfg(feature = "encode-deflate")] + Deflate(crate::encode::Deflate), + + /// GZIP container; Only supports DEFLATE compression. + #[cfg(feature = "encode-gzip")] + Gzip(crate::encode::Gzip), + + /// BGZF container; Only supports DEFLATE compression. + #[cfg(feature = "encode-bgzf")] + Bgzf(crate::encode::Bgzf), +} + +impl Default for Encoder { + fn default() -> Self { + Encoder::Identity(Default::default()) + } +} + +impl<'a, W: Write + 'a, A: Adapter> Encode<'a, W, A> for Encoder +where + crate::encode::Identity: Encode<'a, W, A>, + crate::encode::Deflate: Encode<'a, W, A>, + crate::encode::Gzip: Encode<'a, W, A>, + crate::encode::Bgzf: Encode<'a, W, A>, +{ + fn encode(&self, writer: W, adapter: A) -> Result, Error> { + match self { + Encoder::Identity(codec) => codec.encode(writer, adapter), + #[cfg(feature = "encode-deflate")] + Encoder::Deflate(codec) => codec.encode(writer, adapter), + #[cfg(feature = "encode-gzip")] + Encoder::Gzip(codec) => codec.encode(writer, adapter), + #[cfg(feature = "encode-bgzf")] + Encoder::Bgzf(codec) => codec.encode(writer, adapter), + } + } +} diff --git a/workspace/compress/src/encode/gzip.rs b/workspace/compress/src/encode/gzip.rs new file mode 100644 index 0000000..479dc3d --- /dev/null +++ b/workspace/compress/src/encode/gzip.rs @@ -0,0 +1,43 @@ +use super::adapter::{AdaptWrite, Adapter}; +use super::encode::Encode; +use super::deflate::Deflate; +use flate2::Compression; +use std::io::{Error, Write}; + + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Gzip { + deflate: Deflate, +} + +impl Gzip { + pub const DEFAULT: Gzip = Gzip { + deflate: Deflate::DEFAULT + }; + + pub fn new(deflate: Deflate) -> Self { + Gzip { deflate } + } + + pub fn deflate(&self) -> &Deflate { + &self.deflate + } +} + +impl Default for Gzip { + fn default() -> Self { + Self::DEFAULT + } +} + +impl<'a, W: Write + 'a, A: Adapter> Encode<'a, W, A> for Gzip +where + A: AdaptWrite<'a, flate2::write::GzEncoder>, +{ + fn encode(&self, writer: W, adapter: A) -> Result, Error> { + adapter.wrap( + flate2::write::GzEncoder::new(writer, Compression::new(self.deflate.level() as u32)) + ) + } +} diff --git a/workspace/compress/src/encode/identity.rs b/workspace/compress/src/encode/identity.rs new file mode 100644 index 0000000..b50b8d4 --- /dev/null +++ b/workspace/compress/src/encode/identity.rs @@ -0,0 +1,17 @@ +use super::adapter::{AdaptWrite, Adapter}; +use super::encode::Encode; +use std::io::{Error, Write}; + + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Identity {} + +impl<'a, W: Write + 'a, A: Adapter> Encode<'a, W, A> for Identity +where + A: AdaptWrite<'a, W>, +{ + fn encode(&self, writer: W, adapter: A) -> Result, Error> { + adapter.wrap(writer) + } +} diff --git a/workspace/compress/src/encode/mod.rs b/workspace/compress/src/encode/mod.rs new file mode 100644 index 0000000..40b04e6 --- /dev/null +++ b/workspace/compress/src/encode/mod.rs @@ -0,0 +1,27 @@ +mod adapter; +pub use adapter::{AdaptWrite, Adapter}; + +#[allow(clippy::module_inception)] +mod encode; +pub use encode::Encode; + +mod identity; +pub use identity::Identity; + +#[cfg(feature = "encode-deflate")] +mod deflate; +#[cfg(feature = "encode-deflate")] +pub use deflate::Deflate; + +#[cfg(feature = "encode-gzip")] +mod gzip; +#[cfg(feature = "encode-gzip")] +pub use gzip::Gzip; + +#[cfg(feature = "encode-bgzf")] +mod bgzf; +#[cfg(feature = "encode-bgzf")] +pub use bgzf::Bgzf; + +mod encoder; +pub use encoder::Encoder; diff --git a/workspace/compress/src/format.rs b/workspace/compress/src/format.rs new file mode 100644 index 0000000..14787f4 --- /dev/null +++ b/workspace/compress/src/format.rs @@ -0,0 +1,98 @@ +use crate::{decode, encode}; +use std::path::Path; + +const DEFLATE: &[&str] = &[]; +const GZIP: &[&str] = &["gz", "gzip"]; +const BGZF: &[&str] = &["bgz", "bgzf", "bgzip"]; + + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Format { + Uncompressed, + #[cfg(any(feature = "decode-deflate", feature = "encode-deflate"))] + Deflate, + #[cfg(any(feature = "decode-gzip", feature = "encode-gzip"))] + Gzip, + #[cfg(any(feature = "decode-bgzf", feature = "encode-bgzf"))] + Bgzf, +} + +impl Format { + pub fn from_extension(ext: &str, uncompressed_exts: &[&str]) -> Option { + match ext { + // Empty extension can't be used to infer format + ext if ext.is_empty() => None, + + // Uncompressed files + ext if uncompressed_exts.contains(&ext) => Some(Self::Uncompressed), + + // DEFLATE compressed files + #[cfg(any(feature = "decode-deflate", feature = "encode-deflate"))] + ext if DEFLATE.contains(&ext) => Some(Self::Deflate), + + // GZIP compressed files + #[cfg(any(feature = "decode-gzip", feature = "encode-gzip"))] + ext if GZIP.contains(&ext) => Some(Self::Gzip), + + // BGZF compressed files + #[cfg(any(feature = "decode-bgzf", feature = "encode-bgzf"))] + ext if BGZF.contains(&ext) => Some(Self::Bgzf), + + _ => None + } + } + + pub fn from_path(path: impl AsRef, uncompressed_exts: &[&str]) -> Option { + let path = path.as_ref().extension()?.to_str()?; + Self::from_extension(path, uncompressed_exts) + } + + pub fn encoder(&self) -> encode::Encoder { + match self { + Self::Uncompressed => encode::Encoder::Identity(Default::default()), + #[cfg(feature = "encode-deflate")] + Self::Deflate => encode::Encoder::Deflate(Default::default()), + #[cfg(feature = "encode-gzip")] + Self::Gzip => encode::Encoder::Gzip(Default::default()), + #[cfg(feature = "encode-bgzf")] + Self::Bgzf => encode::Encoder::Bgzf(Default::default()), + } + } + + pub fn decoder(&self) -> decode::Decoder { + match self { + Self::Uncompressed => decode::Decoder::Identity(Default::default()), + #[cfg(feature = "decode-deflate")] + Self::Deflate => decode::Decoder::Deflate(Default::default()), + #[cfg(feature = "decode-gzip")] + Self::Gzip => decode::Decoder::Gzip(Default::default()), + #[cfg(feature = "decode-bgzf")] + Self::Bgzf => decode::Decoder::Bgzf(Default::default()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_format_from_extension() { + let uncompressed_exts = &["txt", "csv"]; + + // Uncompressed extensions + assert_eq!(Format::from_extension("txt", uncompressed_exts), Some(Format::Uncompressed)); + assert_eq!(Format::from_extension(".txt", uncompressed_exts), None); + + // Known compressed extensions + #[cfg(feature = "decode-gzip")] + assert_eq!(Format::from_extension("gz", uncompressed_exts), Some(Format::Gzip)); + + #[cfg(feature = "decode-bgzf")] + assert_eq!(Format::from_extension("bgzf", uncompressed_exts), Some(Format::Bgzf)); + + // Unknown or empty extensions + assert_eq!(Format::from_extension("unknown", uncompressed_exts), None); + assert_eq!(Format::from_extension("", uncompressed_exts), None); + } +} \ No newline at end of file diff --git a/workspace/multilock/src/lock.rs b/workspace/multilock/src/lock.rs index f6544e3..97cba61 100644 --- a/workspace/multilock/src/lock.rs +++ b/workspace/multilock/src/lock.rs @@ -28,7 +28,7 @@ impl<'a> LockId<'a> { pub trait Unlock { /// Unlock a previously acquired lock. /// - /// # Safety: + /// # Safety /// /// Callers must ensure that the lock is currently held by the calling thread. /// Some lock implementations may exhibit undefined behavior if this is not the case. diff --git a/workspace/multilock/src/multi_guard.rs b/workspace/multilock/src/multi_guard.rs index 9c1ca93..d727ec0 100644 --- a/workspace/multilock/src/multi_guard.rs +++ b/workspace/multilock/src/multi_guard.rs @@ -161,7 +161,7 @@ impl<'a> MultiGuard<'a> { .get(&lock) .ok_or(MultiLockError::LockNotHeld) .and_then(|cell| cell.try_borrow_mut().map_err(MultiLockError::from)) - .map(move |x| WriteLockProof::WithBorrow(x)) + .map(WriteLockProof::WithBorrow) } } From cb99c8e536475a827cf7cec5d3b94d9c7c54072d Mon Sep 17 00:00:00 2001 From: alnfedorov <18511496+alnfedorov@users.noreply.github.com> Date: Sun, 28 Dec 2025 11:19:07 +0000 Subject: [PATCH 2/7] compress --- workspace/compress/Cargo.toml | 51 ++++++++++++++++++++++++ workspace/compress/src/lib.rs | 74 +++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 workspace/compress/Cargo.toml create mode 100644 workspace/compress/src/lib.rs diff --git a/workspace/compress/Cargo.toml b/workspace/compress/Cargo.toml new file mode 100644 index 0000000..7b27ff9 --- /dev/null +++ b/workspace/compress/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "compress" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +description.workspace = true +readme.workspace = true +documentation.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +categories.workspace = true + +[dependencies] +flate2 = { workspace = true, optional = true } +noodles-bgzf = { workspace = true, optional = true } +bitcode = { workspace = true, optional = true } + +[features] + +#################################### Low level features #################################### +# These features are expected to be used by other libraries or in the library code itself +# to fine-tune which algorithms are included. + +# Decompression algorithms +decode-deflate = ["dep:flate2"] +decode-gzip = ["dep:flate2"] +decode-bgzf = ["dep:noodles-bgzf"] + +# Compression algorithms +encode-deflate = ["dep:flate2"] +encode-gzip = ["dep:flate2"] +encode-bgzf = ["dep:noodles-bgzf"] + +#################################### High level features #################################### +# These features are expected to be used by end users. + +# Compression formats +deflate = ["encode-deflate", "decode-deflate"] +gzip = ["encode-gzip", "decode-gzip"] +bgzf = ["encode-bgzf", "decode-bgzf"] + +# All supported compression formats +all-formats = ["deflate", "gzip", "bgzf"] + +# Serialization formats +bitcode = ["dep:bitcode"] + +# All features +all = ["all-formats", "bitcode"] diff --git a/workspace/compress/src/lib.rs b/workspace/compress/src/lib.rs new file mode 100644 index 0000000..1e93a3b --- /dev/null +++ b/workspace/compress/src/lib.rs @@ -0,0 +1,74 @@ +//! # Compress +//! +//! A decoupled compression facade that separates I/O stream handling from the +//! underlying compression algorithms. +//! +//! ## Quick Start: Roundtrip Example +//! +//! This example demonstrates how to infer a format from a file extension, +//! compress data into a memory buffer, and then decompress it back to verify integrity. +//! +//! ```rust +//! use compress::{adapter::BoxedSync, decode::DecodeReadIntoRead, encode::Encode, Format}; +//! use std::io::{Read, Write}; +//! +//! fn main() -> std::io::Result<()> { +//! let data = b"Hello, Beautiful Compression World!"; +//! +//! // ---------------------------------------------------------------------- +//! // 1. Setup & Inference +//! // ---------------------------------------------------------------------- +//! // Detect format from extension (e.g., "gz" -> Format::Gzip) +//! let format = Format::from_extension("gz", &[]); +//! assert_eq!(format, Some(Format::Gzip), "Should infer Gzip format"); +//! let format = format.unwrap(); +//! +//! // ---------------------------------------------------------------------- +//! // 2. Encode (Write) +//! // ---------------------------------------------------------------------- +//! // Get a generic encoder facade based on the inferred format +//! let encoder = format.encoder(); +//! +//! // Encode the data into a memory buffer +//! let mut buffer = Vec::new(); +//! { +//! // We use the `BoxedSync` adapter to type-erase the specific encoder. +//! // This allows `encoder_algo` to return a consistent `Box` +//! // regardless of whether it's Gzip, Deflate, or another format. +//! let mut writer = encoder.encode(&mut buffer, BoxedSync)?; +//! writer.write_all(data)?; +//! +//! // The compression finishes when the writer is dropped or flushed/finished. +//! // In this scope-based block, `writer` is dropped here. +//! } +//! +//! // Verify something was written (Gzip header + body + footer) +//! assert!(buffer.len() > 0); +//! +//! // ---------------------------------------------------------------------- +//! // 3. Decode (Read) +//! // ---------------------------------------------------------------------- +//! // Get a generic decoder facade based on the inferred format +//! let decoder = format.decoder(); +//! +//! // We explicitly ask for a `Read` interface and supply a `Read` source. +//! // Again, we use `BoxedSync` to type-erase the specific decoder. +//! let mut reader = decoder.decode_read_into_read(&buffer[..], BoxedSync)?; +//! let mut decompressed = Vec::new(); +//! reader.read_to_end(&mut decompressed)?; +//! +//! // ---------------------------------------------------------------------- +//! // 4. Verify +//! // ---------------------------------------------------------------------- +//! assert_eq!(decompressed, data); +//! +//! Ok(()) +//! } +//! ``` + +pub mod decode; +pub mod adapter; +pub mod encode; +mod format; + +pub use format::Format; From ac056d30daffad7a1e8e3b5a1e1c21c60499f732 Mon Sep 17 00:00:00 2001 From: Aleksandr Fedorov <18511496+alnfedorov@users.noreply.github.com> Date: Sun, 28 Dec 2025 11:27:24 +0000 Subject: [PATCH 3/7] Update workspace/compress/src/encode/bgzf.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- workspace/compress/src/encode/bgzf.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workspace/compress/src/encode/bgzf.rs b/workspace/compress/src/encode/bgzf.rs index 77b1e4a..2479a6a 100644 --- a/workspace/compress/src/encode/bgzf.rs +++ b/workspace/compress/src/encode/bgzf.rs @@ -48,7 +48,9 @@ where let level = bgzf::io::writer::CompressionLevel::new(self.deflate().level()).unwrap(); if let Some(_threads) = self.threads() { - Err(Error::other("Multithreaded BGZF is not supported")) + Err(Error::other( + "Multithreaded BGZF encoding is not yet implemented. Please use single-threaded mode by leaving threads as None.", + )) // let writer = bgzf::io::multithreaded_writer::Builder::default() // .set_compression_level(level) // .set_worker_count(threads) From b3e839a072c8e3336ea04dcb63946a39716608cc Mon Sep 17 00:00:00 2001 From: Aleksandr Fedorov <18511496+alnfedorov@users.noreply.github.com> Date: Sun, 28 Dec 2025 11:27:36 +0000 Subject: [PATCH 4/7] Update workspace/compress/src/encode/adapter.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- workspace/compress/src/encode/adapter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workspace/compress/src/encode/adapter.rs b/workspace/compress/src/encode/adapter.rs index 7e6d352..a33fea6 100644 --- a/workspace/compress/src/encode/adapter.rs +++ b/workspace/compress/src/encode/adapter.rs @@ -1,7 +1,7 @@ use std::io::{Error, Write}; pub trait AdaptWrite<'a, In>: Adapter { - /// A helper trait to allow introducing flexible constraints on the wrapped reader. + /// A helper trait to allow introducing flexible constraints on the wrapped writer. /// /// This is normally where you would put constraints on `In` and implement the actual wrapping logic. fn wrap(self, internal: In) -> Result, Error>; From 1330af2f7e3c868d6b78cb256a517d456ab54490 Mon Sep 17 00:00:00 2001 From: Aleksandr Fedorov <18511496+alnfedorov@users.noreply.github.com> Date: Sun, 28 Dec 2025 11:29:50 +0000 Subject: [PATCH 5/7] Update workspace/compress/src/lib.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- workspace/compress/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workspace/compress/src/lib.rs b/workspace/compress/src/lib.rs index 1e93a3b..d64d8e0 100644 --- a/workspace/compress/src/lib.rs +++ b/workspace/compress/src/lib.rs @@ -43,7 +43,7 @@ //! } //! //! // Verify something was written (Gzip header + body + footer) -//! assert!(buffer.len() > 0); +//! assert!(!buffer.is_empty()); //! //! // ---------------------------------------------------------------------- //! // 3. Decode (Read) From 702e8f9a7d677948bd3d5cf1895232e7700ce0ca Mon Sep 17 00:00:00 2001 From: Aleksandr Fedorov <18511496+alnfedorov@users.noreply.github.com> Date: Sun, 28 Dec 2025 11:38:18 +0000 Subject: [PATCH 6/7] Update workspace/compress/src/adapter/boxed_sync.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- workspace/compress/src/adapter/boxed_sync.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/workspace/compress/src/adapter/boxed_sync.rs b/workspace/compress/src/adapter/boxed_sync.rs index 8eaa332..5085f40 100644 --- a/workspace/compress/src/adapter/boxed_sync.rs +++ b/workspace/compress/src/adapter/boxed_sync.rs @@ -1,6 +1,12 @@ use crate::{decode, encode}; use std::io::{BufRead, Error, Read}; +/// Adapter that type-erases I/O types into boxed trait objects with `Send + Sync` bounds. +/// +/// `BoxedSync` is used by the compression/decompression adapters to wrap concrete +/// `Read`, `BufRead`, and `Write` implementations into `Box` +/// so they can be passed around as thread-safe, type-erased handles in the public API +/// and examples. pub struct BoxedSync; impl<'a, In> decode::AdaptRead<'a, In> for BoxedSync From 3402746bc773dc29f568f02a80622af68737a923 Mon Sep 17 00:00:00 2001 From: alnfedorov <18511496+alnfedorov@users.noreply.github.com> Date: Sun, 28 Dec 2025 11:44:39 +0000 Subject: [PATCH 7/7] refactoring --- workspace/compress/src/adapter/boxed_sync.rs | 2 +- workspace/compress/src/decode/bgzf.rs | 4 ++- workspace/compress/src/decode/decoder.rs | 25 +++++++++++++++--- workspace/compress/src/decode/deflate.rs | 6 +++-- workspace/compress/src/decode/gzip.rs | 10 +++++--- workspace/compress/src/decode/identity.rs | 7 ++--- workspace/compress/src/decode/mod.rs | 4 ++- workspace/compress/src/encode/bgzf.rs | 4 +-- workspace/compress/src/encode/gzip.rs | 12 ++++----- workspace/compress/src/encode/identity.rs | 1 - workspace/compress/src/format.rs | 27 ++++++++++++-------- workspace/compress/src/lib.rs | 2 +- 12 files changed, 68 insertions(+), 36 deletions(-) diff --git a/workspace/compress/src/adapter/boxed_sync.rs b/workspace/compress/src/adapter/boxed_sync.rs index 5085f40..7f4ca03 100644 --- a/workspace/compress/src/adapter/boxed_sync.rs +++ b/workspace/compress/src/adapter/boxed_sync.rs @@ -43,4 +43,4 @@ where impl encode::Adapter for BoxedSync { type Write<'a> = Box; -} \ No newline at end of file +} diff --git a/workspace/compress/src/decode/bgzf.rs b/workspace/compress/src/decode/bgzf.rs index 27caab9..cd92b16 100644 --- a/workspace/compress/src/decode/bgzf.rs +++ b/workspace/compress/src/decode/bgzf.rs @@ -1,5 +1,7 @@ use super::adapter::{AdaptBufRead, AdaptRead, Adapter}; -use super::decode::{DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead}; +use super::decode::{ + DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead, +}; use std::io::{BufRead, BufReader, Error, Read}; use std::num::NonZeroUsize; diff --git a/workspace/compress/src/decode/decoder.rs b/workspace/compress/src/decode/decoder.rs index 59bbcfd..82c49a8 100644 --- a/workspace/compress/src/decode/decoder.rs +++ b/workspace/compress/src/decode/decoder.rs @@ -1,5 +1,7 @@ use super::adapter::Adapter; -use super::decode::{DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead}; +use super::decode::{ + DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead, +}; use std::io::{BufRead, Error, Read}; #[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] @@ -54,6 +56,21 @@ macro_rules! impl_decompression_trait { } impl_decompression_trait!(Read, Read, DecodeReadIntoRead, decode_read_into_read); -impl_decompression_trait!(Read, BufRead, DecodeReadIntoBufRead, decode_read_into_bufread); -impl_decompression_trait!(BufRead, Read, DecodeBufReadIntoRead, decode_bufread_into_read); -impl_decompression_trait!(BufRead, BufRead, DecodeBufReadIntoBufRead, decode_bufread_into_bufread); +impl_decompression_trait!( + Read, + BufRead, + DecodeReadIntoBufRead, + decode_read_into_bufread +); +impl_decompression_trait!( + BufRead, + Read, + DecodeBufReadIntoRead, + decode_bufread_into_read +); +impl_decompression_trait!( + BufRead, + BufRead, + DecodeBufReadIntoBufRead, + decode_bufread_into_bufread +); diff --git a/workspace/compress/src/decode/deflate.rs b/workspace/compress/src/decode/deflate.rs index dad45b6..62405ad 100644 --- a/workspace/compress/src/decode/deflate.rs +++ b/workspace/compress/src/decode/deflate.rs @@ -1,5 +1,7 @@ use super::adapter::{AdaptBufRead, AdaptRead, Adapter}; -use super::decode::{DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead}; +use super::decode::{ + DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead, +}; use std::io; use std::io::{BufRead, Error, Read}; use std::num::NonZeroUsize; @@ -78,4 +80,4 @@ where let bufreader = io::BufReader::with_capacity(self.out_bufsize.get(), reader); adapter.wrap(bufreader) } -} \ No newline at end of file +} diff --git a/workspace/compress/src/decode/gzip.rs b/workspace/compress/src/decode/gzip.rs index f7feede..751b159 100644 --- a/workspace/compress/src/decode/gzip.rs +++ b/workspace/compress/src/decode/gzip.rs @@ -1,5 +1,7 @@ -use super::decode::{DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead}; use super::adapter::{AdaptBufRead, AdaptRead, Adapter}; +use super::decode::{ + DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead, +}; use std::io::{BufRead, BufReader, Error, Read}; use std::num::NonZeroUsize; @@ -17,7 +19,10 @@ impl Gzip { }; pub fn new(out_bufsize: NonZeroUsize, read_multi: bool) -> Self { - Self { out_bufsize, read_multi } + Self { + out_bufsize, + read_multi, + } } pub fn out_bufsize(&self) -> NonZeroUsize { @@ -35,7 +40,6 @@ impl Default for Gzip { } } - impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoRead<'a, R, A> for Gzip where A: AdaptRead<'a, flate2::read::GzDecoder>, diff --git a/workspace/compress/src/decode/identity.rs b/workspace/compress/src/decode/identity.rs index 4a44375..eb48615 100644 --- a/workspace/compress/src/decode/identity.rs +++ b/workspace/compress/src/decode/identity.rs @@ -1,5 +1,7 @@ use super::adapter::{AdaptBufRead, AdaptRead, Adapter}; -use super::decode::{DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead}; +use super::decode::{ + DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead, +}; use std::io::{BufRead, BufReader, Error, Read}; use std::num::NonZeroUsize; @@ -11,7 +13,7 @@ pub struct Identity { impl Identity { pub const DEFAULT: Identity = Identity { - out_bufsize: NonZeroUsize::new(8 * 1024).unwrap() + out_bufsize: NonZeroUsize::new(8 * 1024).unwrap(), }; pub fn new(out_bufsize: NonZeroUsize) -> Self { @@ -63,4 +65,3 @@ where adapter.wrap(reader) } } - diff --git a/workspace/compress/src/decode/mod.rs b/workspace/compress/src/decode/mod.rs index 97d4d67..462e146 100644 --- a/workspace/compress/src/decode/mod.rs +++ b/workspace/compress/src/decode/mod.rs @@ -3,7 +3,9 @@ pub use adapter::{AdaptBufRead, AdaptRead, Adapter}; #[allow(clippy::module_inception)] mod decode; -pub use decode::{DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead}; +pub use decode::{ + DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead, +}; mod identity; pub use identity::Identity; diff --git a/workspace/compress/src/encode/bgzf.rs b/workspace/compress/src/encode/bgzf.rs index 2479a6a..75dd897 100644 --- a/workspace/compress/src/encode/bgzf.rs +++ b/workspace/compress/src/encode/bgzf.rs @@ -7,7 +7,6 @@ use std::num::NonZeroUsize; use super::deflate::Deflate; - #[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Bgzf { @@ -45,8 +44,7 @@ where A: AdaptWrite<'a, bgzf::io::Writer>, { fn encode(&self, writer: W, adapter: A) -> Result, Error> { - let level = - bgzf::io::writer::CompressionLevel::new(self.deflate().level()).unwrap(); + let level = bgzf::io::writer::CompressionLevel::new(self.deflate().level()).unwrap(); if let Some(_threads) = self.threads() { Err(Error::other( "Multithreaded BGZF encoding is not yet implemented. Please use single-threaded mode by leaving threads as None.", diff --git a/workspace/compress/src/encode/gzip.rs b/workspace/compress/src/encode/gzip.rs index 479dc3d..325eece 100644 --- a/workspace/compress/src/encode/gzip.rs +++ b/workspace/compress/src/encode/gzip.rs @@ -1,10 +1,9 @@ use super::adapter::{AdaptWrite, Adapter}; -use super::encode::Encode; use super::deflate::Deflate; +use super::encode::Encode; use flate2::Compression; use std::io::{Error, Write}; - #[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Gzip { @@ -13,7 +12,7 @@ pub struct Gzip { impl Gzip { pub const DEFAULT: Gzip = Gzip { - deflate: Deflate::DEFAULT + deflate: Deflate::DEFAULT, }; pub fn new(deflate: Deflate) -> Self { @@ -36,8 +35,9 @@ where A: AdaptWrite<'a, flate2::write::GzEncoder>, { fn encode(&self, writer: W, adapter: A) -> Result, Error> { - adapter.wrap( - flate2::write::GzEncoder::new(writer, Compression::new(self.deflate.level() as u32)) - ) + adapter.wrap(flate2::write::GzEncoder::new( + writer, + Compression::new(self.deflate.level() as u32), + )) } } diff --git a/workspace/compress/src/encode/identity.rs b/workspace/compress/src/encode/identity.rs index b50b8d4..d44c820 100644 --- a/workspace/compress/src/encode/identity.rs +++ b/workspace/compress/src/encode/identity.rs @@ -2,7 +2,6 @@ use super::adapter::{AdaptWrite, Adapter}; use super::encode::Encode; use std::io::{Error, Write}; - #[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Identity {} diff --git a/workspace/compress/src/format.rs b/workspace/compress/src/format.rs index 14787f4..4e9832e 100644 --- a/workspace/compress/src/format.rs +++ b/workspace/compress/src/format.rs @@ -5,7 +5,6 @@ const DEFLATE: &[&str] = &[]; const GZIP: &[&str] = &["gz", "gzip"]; const BGZF: &[&str] = &["bgz", "bgzf", "bgzip"]; - #[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Format { @@ -39,7 +38,7 @@ impl Format { #[cfg(any(feature = "decode-bgzf", feature = "encode-bgzf"))] ext if BGZF.contains(&ext) => Some(Self::Bgzf), - _ => None + _ => None, } } @@ -79,20 +78,28 @@ mod tests { #[test] fn test_format_from_extension() { let uncompressed_exts = &["txt", "csv"]; - + // Uncompressed extensions - assert_eq!(Format::from_extension("txt", uncompressed_exts), Some(Format::Uncompressed)); - assert_eq!(Format::from_extension(".txt", uncompressed_exts), None); - + assert_eq!( + Format::from_extension("txt", uncompressed_exts), + Some(Format::Uncompressed) + ); + // Known compressed extensions #[cfg(feature = "decode-gzip")] - assert_eq!(Format::from_extension("gz", uncompressed_exts), Some(Format::Gzip)); + assert_eq!( + Format::from_extension("gz", uncompressed_exts), + Some(Format::Gzip) + ); #[cfg(feature = "decode-bgzf")] - assert_eq!(Format::from_extension("bgzf", uncompressed_exts), Some(Format::Bgzf)); - + assert_eq!( + Format::from_extension("bgzf", uncompressed_exts), + Some(Format::Bgzf) + ); + // Unknown or empty extensions assert_eq!(Format::from_extension("unknown", uncompressed_exts), None); assert_eq!(Format::from_extension("", uncompressed_exts), None); } -} \ No newline at end of file +} diff --git a/workspace/compress/src/lib.rs b/workspace/compress/src/lib.rs index d64d8e0..a92cfe6 100644 --- a/workspace/compress/src/lib.rs +++ b/workspace/compress/src/lib.rs @@ -66,8 +66,8 @@ //! } //! ``` -pub mod decode; pub mod adapter; +pub mod decode; pub mod encode; mod format;