diff --git a/Cargo.toml b/Cargo.toml index efde9ee..114885a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,9 +13,13 @@ categories = ["science", "algorithms"] [workspace] members = [ + "workspace/compress", "workspace/multilock", ] resolver = "3" [workspace.dependencies] lock_api = { version = "0.4.14", default-features = false } +flate2 = { version = "1.1.5", features = ["zlib-rs"], default-features = false } +noodles-bgzf = { version = "0.45.0", default-features = false } +bitcode = { version = "0.6.9", features = ["derive"], default-features = false } diff --git a/workspace/compress/Cargo.toml b/workspace/compress/Cargo.toml new file mode 100644 index 0000000..7b27ff9 --- /dev/null +++ b/workspace/compress/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "compress" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +description.workspace = true +readme.workspace = true +documentation.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +categories.workspace = true + +[dependencies] +flate2 = { workspace = true, optional = true } +noodles-bgzf = { workspace = true, optional = true } +bitcode = { workspace = true, optional = true } + +[features] + +#################################### Low level features #################################### +# These features are expected to be used by other libraries or in the library code itself +# to fine-tune which algorithms are included. + +# Decompression algorithms +decode-deflate = ["dep:flate2"] +decode-gzip = ["dep:flate2"] +decode-bgzf = ["dep:noodles-bgzf"] + +# Compression algorithms +encode-deflate = ["dep:flate2"] +encode-gzip = ["dep:flate2"] +encode-bgzf = ["dep:noodles-bgzf"] + +#################################### High level features #################################### +# These features are expected to be used by end users. + +# Compression formats +deflate = ["encode-deflate", "decode-deflate"] +gzip = ["encode-gzip", "decode-gzip"] +bgzf = ["encode-bgzf", "decode-bgzf"] + +# All supported compression formats +all-formats = ["deflate", "gzip", "bgzf"] + +# Serialization formats +bitcode = ["dep:bitcode"] + +# All features +all = ["all-formats", "bitcode"] diff --git a/workspace/compress/src/adapter/boxed_sync.rs b/workspace/compress/src/adapter/boxed_sync.rs new file mode 100644 index 0000000..7f4ca03 --- /dev/null +++ b/workspace/compress/src/adapter/boxed_sync.rs @@ -0,0 +1,46 @@ +use crate::{decode, encode}; +use std::io::{BufRead, Error, Read}; + +/// Adapter that type-erases I/O types into boxed trait objects with `Send + Sync` bounds. +/// +/// `BoxedSync` is used by the compression/decompression adapters to wrap concrete +/// `Read`, `BufRead`, and `Write` implementations into `Box` +/// so they can be passed around as thread-safe, type-erased handles in the public API +/// and examples. +pub struct BoxedSync; + +impl<'a, In> decode::AdaptRead<'a, In> for BoxedSync +where + In: Read + Send + Sync + 'a, +{ + fn wrap(self, internal: In) -> Result, Error> { + Ok(Box::new(internal)) + } +} + +impl<'a, In> decode::AdaptBufRead<'a, In> for BoxedSync +where + In: BufRead + Send + Sync + 'a, +{ + fn wrap(self, internal: In) -> Result, Error> { + Ok(Box::new(internal)) + } +} + +impl decode::Adapter for BoxedSync { + type Read<'a> = Box; + type BufRead<'a> = Box; +} + +impl<'a, In> encode::AdaptWrite<'a, In> for BoxedSync +where + In: std::io::Write + Send + Sync + 'a, +{ + fn wrap(self, internal: In) -> Result, Error> { + Ok(Box::new(internal)) + } +} + +impl encode::Adapter for BoxedSync { + type Write<'a> = Box; +} diff --git a/workspace/compress/src/adapter/mod.rs b/workspace/compress/src/adapter/mod.rs new file mode 100644 index 0000000..a2e1539 --- /dev/null +++ b/workspace/compress/src/adapter/mod.rs @@ -0,0 +1,2 @@ +mod boxed_sync; +pub use boxed_sync::BoxedSync; diff --git a/workspace/compress/src/decode/adapter.rs b/workspace/compress/src/decode/adapter.rs new file mode 100644 index 0000000..6f21871 --- /dev/null +++ b/workspace/compress/src/decode/adapter.rs @@ -0,0 +1,20 @@ +use std::io::{BufRead, Error, Read}; + +pub trait AdaptRead<'a, In>: Adapter { + /// A helper trait to allow introducing flexible constraints on the wrapped reader. + /// + /// This is normally where you would put constraints on `In` and implement the actual wrapping logic. + fn wrap(self, internal: In) -> Result, Error>; +} + +pub trait AdaptBufRead<'a, In>: Adapter { + /// A helper trait to allow introducing flexible constraints on the wrapped buffered reader. + /// + /// This is normally where you would put constraints on `In` and implement the actual wrapping logic. + fn wrap(self, internal: In) -> Result, Error>; +} + +pub trait Adapter { + type Read<'a>: Read + 'a; + type BufRead<'a>: BufRead + 'a; +} diff --git a/workspace/compress/src/decode/bgzf.rs b/workspace/compress/src/decode/bgzf.rs new file mode 100644 index 0000000..cd92b16 --- /dev/null +++ b/workspace/compress/src/decode/bgzf.rs @@ -0,0 +1,74 @@ +use super::adapter::{AdaptBufRead, AdaptRead, Adapter}; +use super::decode::{ + DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead, +}; +use std::io::{BufRead, BufReader, Error, Read}; +use std::num::NonZeroUsize; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Bgzf { + out_bufsize: NonZeroUsize, +} + +impl Bgzf { + pub const DEFAULT: Bgzf = Bgzf { + out_bufsize: NonZeroUsize::new(8 * 1024).unwrap(), + }; + + pub fn new(out_bufsize: NonZeroUsize) -> Self { + Self { out_bufsize } + } + + pub fn out_bufsize(&self) -> NonZeroUsize { + self.out_bufsize + } +} + +impl Default for Bgzf { + fn default() -> Self { + Self::DEFAULT + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoRead<'a, R, A> for Bgzf +where + A: AdaptRead<'a, flate2::read::MultiGzDecoder>, +{ + fn decode_read_into_read(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::read::MultiGzDecoder::new(reader); + adapter.wrap(reader) + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoBufRead<'a, R, A> for Bgzf +where + A: AdaptBufRead<'a, BufReader>>, +{ + fn decode_read_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::read::MultiGzDecoder::new(reader); + let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoRead<'a, R, A> for Bgzf +where + A: AdaptRead<'a, flate2::bufread::MultiGzDecoder>, +{ + fn decode_bufread_into_read(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::bufread::MultiGzDecoder::new(reader); + adapter.wrap(reader) + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoBufRead<'a, R, A> for Bgzf +where + A: AdaptBufRead<'a, BufReader>>, +{ + fn decode_bufread_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::bufread::MultiGzDecoder::new(reader); + let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } +} diff --git a/workspace/compress/src/decode/decode.rs b/workspace/compress/src/decode/decode.rs new file mode 100644 index 0000000..ca80dca --- /dev/null +++ b/workspace/compress/src/decode/decode.rs @@ -0,0 +1,18 @@ +use super::adapter::Adapter; +use std::io::{BufRead, Error, Read}; + +pub trait DecodeReadIntoRead<'a, R: Read + 'a, A: Adapter> { + fn decode_read_into_read(&self, reader: R, adapter: A) -> Result, Error>; +} + +pub trait DecodeReadIntoBufRead<'a, R: Read + 'a, A: Adapter> { + fn decode_read_into_bufread(&self, reader: R, adapter: A) -> Result, Error>; +} + +pub trait DecodeBufReadIntoRead<'a, R: BufRead + 'a, A: Adapter> { + fn decode_bufread_into_read(&self, reader: R, adapter: A) -> Result, Error>; +} + +pub trait DecodeBufReadIntoBufRead<'a, R: BufRead + 'a, A: Adapter> { + fn decode_bufread_into_bufread(&self, reader: R, adapter: A) -> Result, Error>; +} diff --git a/workspace/compress/src/decode/decoder.rs b/workspace/compress/src/decode/decoder.rs new file mode 100644 index 0000000..82c49a8 --- /dev/null +++ b/workspace/compress/src/decode/decoder.rs @@ -0,0 +1,76 @@ +use super::adapter::Adapter; +use super::decode::{ + DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead, +}; +use std::io::{BufRead, Error, Read}; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Decoder { + /// No compression (identity) decoding algorithm. + /// + /// The option is useful to uniformly handle both compressed and uncompressed data streams. + Identity(crate::decode::Identity), + + /// DEFLATE compression algorithm + #[cfg(feature = "decode-deflate")] + Deflate(crate::decode::Deflate), + + /// GZIP container; Only supports DEFLATE compression. + #[cfg(feature = "decode-gzip")] + Gzip(crate::decode::Gzip), + + /// BGZF container; Only supports DEFLATE compression. + #[cfg(feature = "decode-bgzf")] + Bgzf(crate::decode::Bgzf), +} + +impl Default for Decoder { + fn default() -> Self { + Decoder::Identity(Default::default()) + } +} + +macro_rules! impl_decompression_trait { + ($read:ident, $out:ident, $trait:ident, $method:ident) => { + impl<'a, R: $read + 'a, A: Adapter> $trait<'a, R, A> for Decoder + where + crate::decode::Identity: $trait<'a, R, A>, + crate::decode::Deflate: $trait<'a, R, A>, + crate::decode::Gzip: $trait<'a, R, A>, + crate::decode::Bgzf: $trait<'a, R, A>, + { + fn $method(&self, reader: R, adapter: A) -> Result, Error> { + match self { + Decoder::Identity(codec) => codec.$method(reader, adapter), + #[cfg(feature = "decode-deflate")] + Decoder::Deflate(codec) => codec.$method(reader, adapter), + #[cfg(feature = "decode-gzip")] + Decoder::Gzip(codec) => codec.$method(reader, adapter), + #[cfg(feature = "decode-bgzf")] + Decoder::Bgzf(codec) => codec.$method(reader, adapter), + } + } + } + }; +} + +impl_decompression_trait!(Read, Read, DecodeReadIntoRead, decode_read_into_read); +impl_decompression_trait!( + Read, + BufRead, + DecodeReadIntoBufRead, + decode_read_into_bufread +); +impl_decompression_trait!( + BufRead, + Read, + DecodeBufReadIntoRead, + decode_bufread_into_read +); +impl_decompression_trait!( + BufRead, + BufRead, + DecodeBufReadIntoBufRead, + decode_bufread_into_bufread +); diff --git a/workspace/compress/src/decode/deflate.rs b/workspace/compress/src/decode/deflate.rs new file mode 100644 index 0000000..62405ad --- /dev/null +++ b/workspace/compress/src/decode/deflate.rs @@ -0,0 +1,83 @@ +use super::adapter::{AdaptBufRead, AdaptRead, Adapter}; +use super::decode::{ + DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead, +}; +use std::io; +use std::io::{BufRead, Error, Read}; +use std::num::NonZeroUsize; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Deflate { + in_bufsize: NonZeroUsize, + out_bufsize: NonZeroUsize, +} + +impl Deflate { + pub const DEFAULT: Deflate = Deflate { + in_bufsize: NonZeroUsize::new(32 * 1024).unwrap(), + out_bufsize: NonZeroUsize::new(8 * 1024).unwrap(), + }; + + pub fn new(deflate_bufsize: NonZeroUsize, out_bufsize: NonZeroUsize) -> Self { + Self { + in_bufsize: deflate_bufsize, + out_bufsize, + } + } + + pub fn in_bufsize(&self) -> usize { + self.in_bufsize.get() + } + + pub fn out_bufsize(&self) -> usize { + self.out_bufsize.get() + } +} + +impl Default for Deflate { + fn default() -> Self { + Deflate::DEFAULT + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoRead<'a, R, A> for Deflate +where + A: AdaptRead<'a, flate2::read::DeflateDecoder>, +{ + fn decode_read_into_read(&self, reader: R, adapter: A) -> Result, Error> { + adapter.wrap(flate2::read::DeflateDecoder::new(reader)) + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoBufRead<'a, R, A> for Deflate +where + A: AdaptBufRead<'a, io::BufReader>>, +{ + fn decode_read_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::read::DeflateDecoder::new(reader); + let bufreader = io::BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoRead<'a, R, A> for Deflate +where + A: AdaptRead<'a, flate2::bufread::DeflateDecoder>, +{ + fn decode_bufread_into_read(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::bufread::DeflateDecoder::new(reader); + adapter.wrap(reader) + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoBufRead<'a, R, A> for Deflate +where + A: AdaptBufRead<'a, io::BufReader>>, +{ + fn decode_bufread_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + let reader = flate2::bufread::DeflateDecoder::new(reader); + let bufreader = io::BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } +} diff --git a/workspace/compress/src/decode/gzip.rs b/workspace/compress/src/decode/gzip.rs new file mode 100644 index 0000000..751b159 --- /dev/null +++ b/workspace/compress/src/decode/gzip.rs @@ -0,0 +1,109 @@ +use super::adapter::{AdaptBufRead, AdaptRead, Adapter}; +use super::decode::{ + DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead, +}; +use std::io::{BufRead, BufReader, Error, Read}; +use std::num::NonZeroUsize; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Gzip { + out_bufsize: NonZeroUsize, + read_multi: bool, +} + +impl Gzip { + pub const DEFAULT: Gzip = Gzip { + out_bufsize: NonZeroUsize::new(8 * 1024).unwrap(), + read_multi: true, + }; + + pub fn new(out_bufsize: NonZeroUsize, read_multi: bool) -> Self { + Self { + out_bufsize, + read_multi, + } + } + + pub fn out_bufsize(&self) -> NonZeroUsize { + self.out_bufsize + } + + pub fn read_multi(&self) -> bool { + self.read_multi + } +} + +impl Default for Gzip { + fn default() -> Self { + Self::DEFAULT + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoRead<'a, R, A> for Gzip +where + A: AdaptRead<'a, flate2::read::GzDecoder>, + A: AdaptRead<'a, flate2::read::MultiGzDecoder>, +{ + fn decode_read_into_read(&self, reader: R, adapter: A) -> Result, Error> { + if self.read_multi { + let reader = flate2::read::MultiGzDecoder::new(reader); + adapter.wrap(reader) + } else { + let reader = flate2::read::GzDecoder::new(reader); + adapter.wrap(reader) + } + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoBufRead<'a, R, A> for Gzip +where + A: AdaptBufRead<'a, BufReader>>, + A: AdaptBufRead<'a, BufReader>>, +{ + fn decode_read_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + if self.read_multi { + let reader = flate2::read::MultiGzDecoder::new(reader); + let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } else { + let reader = flate2::read::GzDecoder::new(reader); + let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoRead<'a, R, A> for Gzip +where + A: AdaptRead<'a, flate2::bufread::GzDecoder>, + A: AdaptRead<'a, flate2::bufread::MultiGzDecoder>, +{ + fn decode_bufread_into_read(&self, reader: R, adapter: A) -> Result, Error> { + if self.read_multi { + let reader = flate2::bufread::MultiGzDecoder::new(reader); + adapter.wrap(reader) + } else { + let reader = flate2::bufread::GzDecoder::new(reader); + adapter.wrap(reader) + } + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoBufRead<'a, R, A> for Gzip +where + A: AdaptBufRead<'a, BufReader>>, + A: AdaptBufRead<'a, BufReader>>, +{ + fn decode_bufread_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + if self.read_multi { + let reader = flate2::bufread::MultiGzDecoder::new(reader); + let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } else { + let reader = flate2::bufread::GzDecoder::new(reader); + let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader); + adapter.wrap(bufreader) + } + } +} diff --git a/workspace/compress/src/decode/identity.rs b/workspace/compress/src/decode/identity.rs new file mode 100644 index 0000000..eb48615 --- /dev/null +++ b/workspace/compress/src/decode/identity.rs @@ -0,0 +1,67 @@ +use super::adapter::{AdaptBufRead, AdaptRead, Adapter}; +use super::decode::{ + DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead, +}; +use std::io::{BufRead, BufReader, Error, Read}; +use std::num::NonZeroUsize; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Identity { + out_bufsize: NonZeroUsize, +} + +impl Identity { + pub const DEFAULT: Identity = Identity { + out_bufsize: NonZeroUsize::new(8 * 1024).unwrap(), + }; + + pub fn new(out_bufsize: NonZeroUsize) -> Self { + Self { out_bufsize } + } + + pub fn out_bufsize(&self) -> NonZeroUsize { + self.out_bufsize + } +} +impl Default for Identity { + fn default() -> Self { + Self::DEFAULT + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoRead<'a, R, A> for Identity +where + A: AdaptRead<'a, R>, +{ + fn decode_read_into_read(&self, reader: R, adapter: A) -> Result, Error> { + adapter.wrap(reader) + } +} + +impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoBufRead<'a, R, A> for Identity +where + A: AdaptBufRead<'a, BufReader>, +{ + fn decode_read_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + adapter.wrap(BufReader::with_capacity(self.out_bufsize.get(), reader)) + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoRead<'a, R, A> for Identity +where + A: AdaptRead<'a, R>, +{ + fn decode_bufread_into_read(&self, reader: R, adapter: A) -> Result, Error> { + adapter.wrap(reader) + } +} + +impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoBufRead<'a, R, A> for Identity +where + A: AdaptBufRead<'a, R>, +{ + fn decode_bufread_into_bufread(&self, reader: R, adapter: A) -> Result, Error> { + adapter.wrap(reader) + } +} diff --git a/workspace/compress/src/decode/mod.rs b/workspace/compress/src/decode/mod.rs new file mode 100644 index 0000000..462e146 --- /dev/null +++ b/workspace/compress/src/decode/mod.rs @@ -0,0 +1,29 @@ +mod adapter; +pub use adapter::{AdaptBufRead, AdaptRead, Adapter}; + +#[allow(clippy::module_inception)] +mod decode; +pub use decode::{ + DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead, +}; + +mod identity; +pub use identity::Identity; + +#[cfg(feature = "decode-deflate")] +mod deflate; +#[cfg(feature = "decode-deflate")] +pub use deflate::Deflate; + +#[cfg(feature = "decode-gzip")] +mod gzip; +#[cfg(feature = "decode-gzip")] +pub use gzip::Gzip; + +#[cfg(feature = "decode-bgzf")] +mod bgzf; +#[cfg(feature = "decode-bgzf")] +pub use bgzf::Bgzf; + +mod decoder; +pub use decoder::Decoder; diff --git a/workspace/compress/src/encode/adapter.rs b/workspace/compress/src/encode/adapter.rs new file mode 100644 index 0000000..a33fea6 --- /dev/null +++ b/workspace/compress/src/encode/adapter.rs @@ -0,0 +1,12 @@ +use std::io::{Error, Write}; + +pub trait AdaptWrite<'a, In>: Adapter { + /// A helper trait to allow introducing flexible constraints on the wrapped writer. + /// + /// This is normally where you would put constraints on `In` and implement the actual wrapping logic. + fn wrap(self, internal: In) -> Result, Error>; +} + +pub trait Adapter { + type Write<'a>: Write + 'a; +} diff --git a/workspace/compress/src/encode/bgzf.rs b/workspace/compress/src/encode/bgzf.rs new file mode 100644 index 0000000..75dd897 --- /dev/null +++ b/workspace/compress/src/encode/bgzf.rs @@ -0,0 +1,64 @@ +use super::adapter::{AdaptWrite, Adapter}; +use super::encode::Encode; +use noodles_bgzf as bgzf; +use std::io; +use std::io::{Error, Write}; +use std::num::NonZeroUsize; + +use super::deflate::Deflate; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Bgzf { + deflate: Deflate, + threads: Option, +} + +impl Bgzf { + pub const DEFAULT: Bgzf = Bgzf { + deflate: Deflate::DEFAULT, + threads: None, + }; + + pub fn new(deflate: Deflate, threads: Option) -> io::Result { + Ok(Self { deflate, threads }) + } + + pub fn deflate(&self) -> &Deflate { + &self.deflate + } + + pub fn threads(&self) -> Option { + self.threads + } +} + +impl Default for Bgzf { + fn default() -> Self { + Bgzf::DEFAULT + } +} + +impl<'a, W: Write + 'a, A: Adapter> Encode<'a, W, A> for Bgzf +where + A: AdaptWrite<'a, bgzf::io::Writer>, +{ + fn encode(&self, writer: W, adapter: A) -> Result, Error> { + let level = bgzf::io::writer::CompressionLevel::new(self.deflate().level()).unwrap(); + if let Some(_threads) = self.threads() { + Err(Error::other( + "Multithreaded BGZF encoding is not yet implemented. Please use single-threaded mode by leaving threads as None.", + )) + // let writer = bgzf::io::multithreaded_writer::Builder::default() + // .set_compression_level(level) + // .set_worker_count(threads) + // .build_from_writer(writer); + // adapter.wrap(writer) + } else { + let writer = bgzf::io::writer::Builder::default() + .set_compression_level(level) + .build_from_writer(writer); + adapter.wrap(writer) + } + } +} diff --git a/workspace/compress/src/encode/deflate.rs b/workspace/compress/src/encode/deflate.rs new file mode 100644 index 0000000..ad53c22 --- /dev/null +++ b/workspace/compress/src/encode/deflate.rs @@ -0,0 +1,49 @@ +use super::adapter::{AdaptWrite, Adapter}; +use super::encode::Encode; +use std::io; +use std::io::{Error, Write}; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Deflate { + level: u8, +} + +impl Deflate { + pub const NONE: Deflate = Deflate { level: 0 }; + pub const FAST: Deflate = Deflate { level: 1 }; + pub const DEFAULT: Deflate = Deflate { level: 6 }; + pub const BEST: Deflate = Deflate { level: 9 }; + + pub fn new(level: u8) -> io::Result { + if level > 9 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("Invalid DEFLATE compress level: {}", level), + )); + } + Ok(Self { level }) + } + + pub fn level(&self) -> u8 { + self.level + } +} + +impl Default for Deflate { + fn default() -> Self { + Deflate::DEFAULT + } +} + +impl<'a, W: Write + 'a, A: Adapter> Encode<'a, W, A> for Deflate +where + A: AdaptWrite<'a, flate2::write::DeflateEncoder>, +{ + fn encode(&self, writer: W, adapter: A) -> Result, Error> { + adapter.wrap(flate2::write::DeflateEncoder::new( + writer, + flate2::Compression::new(self.level as u32), + )) + } +} diff --git a/workspace/compress/src/encode/encode.rs b/workspace/compress/src/encode/encode.rs new file mode 100644 index 0000000..fcce8b8 --- /dev/null +++ b/workspace/compress/src/encode/encode.rs @@ -0,0 +1,6 @@ +use super::adapter::Adapter; +use std::io::{Error, Write}; + +pub trait Encode<'a, W: Write + 'a, A: Adapter> { + fn encode(&self, writer: W, adapter: A) -> Result, Error>; +} diff --git a/workspace/compress/src/encode/encoder.rs b/workspace/compress/src/encode/encoder.rs new file mode 100644 index 0000000..40fafe0 --- /dev/null +++ b/workspace/compress/src/encode/encoder.rs @@ -0,0 +1,50 @@ +use super::adapter::Adapter; +use super::encode::Encode; +use std::io::{Error, Write}; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Encoder { + /// No compression (identity) encoding algorithm. + /// + /// The option is useful to uniformly handle both compressed and uncompressed data streams. + Identity(crate::encode::Identity), + + /// DEFLATE compression algorithm + #[cfg(feature = "encode-deflate")] + Deflate(crate::encode::Deflate), + + /// GZIP container; Only supports DEFLATE compression. + #[cfg(feature = "encode-gzip")] + Gzip(crate::encode::Gzip), + + /// BGZF container; Only supports DEFLATE compression. + #[cfg(feature = "encode-bgzf")] + Bgzf(crate::encode::Bgzf), +} + +impl Default for Encoder { + fn default() -> Self { + Encoder::Identity(Default::default()) + } +} + +impl<'a, W: Write + 'a, A: Adapter> Encode<'a, W, A> for Encoder +where + crate::encode::Identity: Encode<'a, W, A>, + crate::encode::Deflate: Encode<'a, W, A>, + crate::encode::Gzip: Encode<'a, W, A>, + crate::encode::Bgzf: Encode<'a, W, A>, +{ + fn encode(&self, writer: W, adapter: A) -> Result, Error> { + match self { + Encoder::Identity(codec) => codec.encode(writer, adapter), + #[cfg(feature = "encode-deflate")] + Encoder::Deflate(codec) => codec.encode(writer, adapter), + #[cfg(feature = "encode-gzip")] + Encoder::Gzip(codec) => codec.encode(writer, adapter), + #[cfg(feature = "encode-bgzf")] + Encoder::Bgzf(codec) => codec.encode(writer, adapter), + } + } +} diff --git a/workspace/compress/src/encode/gzip.rs b/workspace/compress/src/encode/gzip.rs new file mode 100644 index 0000000..325eece --- /dev/null +++ b/workspace/compress/src/encode/gzip.rs @@ -0,0 +1,43 @@ +use super::adapter::{AdaptWrite, Adapter}; +use super::deflate::Deflate; +use super::encode::Encode; +use flate2::Compression; +use std::io::{Error, Write}; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Gzip { + deflate: Deflate, +} + +impl Gzip { + pub const DEFAULT: Gzip = Gzip { + deflate: Deflate::DEFAULT, + }; + + pub fn new(deflate: Deflate) -> Self { + Gzip { deflate } + } + + pub fn deflate(&self) -> &Deflate { + &self.deflate + } +} + +impl Default for Gzip { + fn default() -> Self { + Self::DEFAULT + } +} + +impl<'a, W: Write + 'a, A: Adapter> Encode<'a, W, A> for Gzip +where + A: AdaptWrite<'a, flate2::write::GzEncoder>, +{ + fn encode(&self, writer: W, adapter: A) -> Result, Error> { + adapter.wrap(flate2::write::GzEncoder::new( + writer, + Compression::new(self.deflate.level() as u32), + )) + } +} diff --git a/workspace/compress/src/encode/identity.rs b/workspace/compress/src/encode/identity.rs new file mode 100644 index 0000000..d44c820 --- /dev/null +++ b/workspace/compress/src/encode/identity.rs @@ -0,0 +1,16 @@ +use super::adapter::{AdaptWrite, Adapter}; +use super::encode::Encode; +use std::io::{Error, Write}; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Identity {} + +impl<'a, W: Write + 'a, A: Adapter> Encode<'a, W, A> for Identity +where + A: AdaptWrite<'a, W>, +{ + fn encode(&self, writer: W, adapter: A) -> Result, Error> { + adapter.wrap(writer) + } +} diff --git a/workspace/compress/src/encode/mod.rs b/workspace/compress/src/encode/mod.rs new file mode 100644 index 0000000..40b04e6 --- /dev/null +++ b/workspace/compress/src/encode/mod.rs @@ -0,0 +1,27 @@ +mod adapter; +pub use adapter::{AdaptWrite, Adapter}; + +#[allow(clippy::module_inception)] +mod encode; +pub use encode::Encode; + +mod identity; +pub use identity::Identity; + +#[cfg(feature = "encode-deflate")] +mod deflate; +#[cfg(feature = "encode-deflate")] +pub use deflate::Deflate; + +#[cfg(feature = "encode-gzip")] +mod gzip; +#[cfg(feature = "encode-gzip")] +pub use gzip::Gzip; + +#[cfg(feature = "encode-bgzf")] +mod bgzf; +#[cfg(feature = "encode-bgzf")] +pub use bgzf::Bgzf; + +mod encoder; +pub use encoder::Encoder; diff --git a/workspace/compress/src/format.rs b/workspace/compress/src/format.rs new file mode 100644 index 0000000..4e9832e --- /dev/null +++ b/workspace/compress/src/format.rs @@ -0,0 +1,105 @@ +use crate::{decode, encode}; +use std::path::Path; + +const DEFLATE: &[&str] = &[]; +const GZIP: &[&str] = &["gz", "gzip"]; +const BGZF: &[&str] = &["bgz", "bgzf", "bgzip"]; + +#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Format { + Uncompressed, + #[cfg(any(feature = "decode-deflate", feature = "encode-deflate"))] + Deflate, + #[cfg(any(feature = "decode-gzip", feature = "encode-gzip"))] + Gzip, + #[cfg(any(feature = "decode-bgzf", feature = "encode-bgzf"))] + Bgzf, +} + +impl Format { + pub fn from_extension(ext: &str, uncompressed_exts: &[&str]) -> Option { + match ext { + // Empty extension can't be used to infer format + ext if ext.is_empty() => None, + + // Uncompressed files + ext if uncompressed_exts.contains(&ext) => Some(Self::Uncompressed), + + // DEFLATE compressed files + #[cfg(any(feature = "decode-deflate", feature = "encode-deflate"))] + ext if DEFLATE.contains(&ext) => Some(Self::Deflate), + + // GZIP compressed files + #[cfg(any(feature = "decode-gzip", feature = "encode-gzip"))] + ext if GZIP.contains(&ext) => Some(Self::Gzip), + + // BGZF compressed files + #[cfg(any(feature = "decode-bgzf", feature = "encode-bgzf"))] + ext if BGZF.contains(&ext) => Some(Self::Bgzf), + + _ => None, + } + } + + pub fn from_path(path: impl AsRef, uncompressed_exts: &[&str]) -> Option { + let path = path.as_ref().extension()?.to_str()?; + Self::from_extension(path, uncompressed_exts) + } + + pub fn encoder(&self) -> encode::Encoder { + match self { + Self::Uncompressed => encode::Encoder::Identity(Default::default()), + #[cfg(feature = "encode-deflate")] + Self::Deflate => encode::Encoder::Deflate(Default::default()), + #[cfg(feature = "encode-gzip")] + Self::Gzip => encode::Encoder::Gzip(Default::default()), + #[cfg(feature = "encode-bgzf")] + Self::Bgzf => encode::Encoder::Bgzf(Default::default()), + } + } + + pub fn decoder(&self) -> decode::Decoder { + match self { + Self::Uncompressed => decode::Decoder::Identity(Default::default()), + #[cfg(feature = "decode-deflate")] + Self::Deflate => decode::Decoder::Deflate(Default::default()), + #[cfg(feature = "decode-gzip")] + Self::Gzip => decode::Decoder::Gzip(Default::default()), + #[cfg(feature = "decode-bgzf")] + Self::Bgzf => decode::Decoder::Bgzf(Default::default()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_format_from_extension() { + let uncompressed_exts = &["txt", "csv"]; + + // Uncompressed extensions + assert_eq!( + Format::from_extension("txt", uncompressed_exts), + Some(Format::Uncompressed) + ); + + // Known compressed extensions + #[cfg(feature = "decode-gzip")] + assert_eq!( + Format::from_extension("gz", uncompressed_exts), + Some(Format::Gzip) + ); + + #[cfg(feature = "decode-bgzf")] + assert_eq!( + Format::from_extension("bgzf", uncompressed_exts), + Some(Format::Bgzf) + ); + + // Unknown or empty extensions + assert_eq!(Format::from_extension("unknown", uncompressed_exts), None); + assert_eq!(Format::from_extension("", uncompressed_exts), None); + } +} diff --git a/workspace/compress/src/lib.rs b/workspace/compress/src/lib.rs new file mode 100644 index 0000000..a92cfe6 --- /dev/null +++ b/workspace/compress/src/lib.rs @@ -0,0 +1,74 @@ +//! # Compress +//! +//! A decoupled compression facade that separates I/O stream handling from the +//! underlying compression algorithms. +//! +//! ## Quick Start: Roundtrip Example +//! +//! This example demonstrates how to infer a format from a file extension, +//! compress data into a memory buffer, and then decompress it back to verify integrity. +//! +//! ```rust +//! use compress::{adapter::BoxedSync, decode::DecodeReadIntoRead, encode::Encode, Format}; +//! use std::io::{Read, Write}; +//! +//! fn main() -> std::io::Result<()> { +//! let data = b"Hello, Beautiful Compression World!"; +//! +//! // ---------------------------------------------------------------------- +//! // 1. Setup & Inference +//! // ---------------------------------------------------------------------- +//! // Detect format from extension (e.g., "gz" -> Format::Gzip) +//! let format = Format::from_extension("gz", &[]); +//! assert_eq!(format, Some(Format::Gzip), "Should infer Gzip format"); +//! let format = format.unwrap(); +//! +//! // ---------------------------------------------------------------------- +//! // 2. Encode (Write) +//! // ---------------------------------------------------------------------- +//! // Get a generic encoder facade based on the inferred format +//! let encoder = format.encoder(); +//! +//! // Encode the data into a memory buffer +//! let mut buffer = Vec::new(); +//! { +//! // We use the `BoxedSync` adapter to type-erase the specific encoder. +//! // This allows `encoder_algo` to return a consistent `Box` +//! // regardless of whether it's Gzip, Deflate, or another format. +//! let mut writer = encoder.encode(&mut buffer, BoxedSync)?; +//! writer.write_all(data)?; +//! +//! // The compression finishes when the writer is dropped or flushed/finished. +//! // In this scope-based block, `writer` is dropped here. +//! } +//! +//! // Verify something was written (Gzip header + body + footer) +//! assert!(!buffer.is_empty()); +//! +//! // ---------------------------------------------------------------------- +//! // 3. Decode (Read) +//! // ---------------------------------------------------------------------- +//! // Get a generic decoder facade based on the inferred format +//! let decoder = format.decoder(); +//! +//! // We explicitly ask for a `Read` interface and supply a `Read` source. +//! // Again, we use `BoxedSync` to type-erase the specific decoder. +//! let mut reader = decoder.decode_read_into_read(&buffer[..], BoxedSync)?; +//! let mut decompressed = Vec::new(); +//! reader.read_to_end(&mut decompressed)?; +//! +//! // ---------------------------------------------------------------------- +//! // 4. Verify +//! // ---------------------------------------------------------------------- +//! assert_eq!(decompressed, data); +//! +//! Ok(()) +//! } +//! ``` + +pub mod adapter; +pub mod decode; +pub mod encode; +mod format; + +pub use format::Format; diff --git a/workspace/multilock/src/lock.rs b/workspace/multilock/src/lock.rs index f6544e3..97cba61 100644 --- a/workspace/multilock/src/lock.rs +++ b/workspace/multilock/src/lock.rs @@ -28,7 +28,7 @@ impl<'a> LockId<'a> { pub trait Unlock { /// Unlock a previously acquired lock. /// - /// # Safety: + /// # Safety /// /// Callers must ensure that the lock is currently held by the calling thread. /// Some lock implementations may exhibit undefined behavior if this is not the case. diff --git a/workspace/multilock/src/multi_guard.rs b/workspace/multilock/src/multi_guard.rs index 9c1ca93..d727ec0 100644 --- a/workspace/multilock/src/multi_guard.rs +++ b/workspace/multilock/src/multi_guard.rs @@ -161,7 +161,7 @@ impl<'a> MultiGuard<'a> { .get(&lock) .ok_or(MultiLockError::LockNotHeld) .and_then(|cell| cell.try_borrow_mut().map_err(MultiLockError::from)) - .map(move |x| WriteLockProof::WithBorrow(x)) + .map(WriteLockProof::WithBorrow) } }