Compress module

Cargo.toml

@@ -13,9 +13,13 @@ categories = ["science", "algorithms"]
 
 [workspace]
 members = [
+    "workspace/compress",
     "workspace/multilock",
 ]
 resolver = "3"
 
 [workspace.dependencies]
 lock_api = { version = "0.4.14", default-features = false }
+flate2 = { version = "1.1.5", features = ["zlib-rs"], default-features = false }
+noodles-bgzf = { version = "0.45.0", default-features = false }
+bitcode = { version = "0.6.9", features = ["derive"], default-features = false }

workspace/compress/Cargo.toml

@@ -0,0 +1,51 @@
+[package]
+name = "compress"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+authors.workspace = true
+description.workspace = true
+readme.workspace = true
+documentation.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+categories.workspace = true
+
+[dependencies]
+flate2 = { workspace = true, optional = true }
+noodles-bgzf = { workspace = true, optional = true }
+bitcode = { workspace = true, optional = true }
+
+[features]
+
+#################################### Low level features ####################################
+# These features are expected to be used by other libraries or in the library code itself
+# to fine-tune which algorithms are included.
+
+# Decompression algorithms
+decode-deflate = ["dep:flate2"]
+decode-gzip = ["dep:flate2"]
+decode-bgzf = ["dep:noodles-bgzf"]
+
+# Compression algorithms
+encode-deflate = ["dep:flate2"]
+encode-gzip = ["dep:flate2"]
+encode-bgzf = ["dep:noodles-bgzf"]
+
+#################################### High level features ####################################
+# These features are expected to be used by end users.
+
+# Compression formats
+deflate = ["encode-deflate", "decode-deflate"]
+gzip = ["encode-gzip", "decode-gzip"]
+bgzf = ["encode-bgzf", "decode-bgzf"]
+
+# All supported compression formats
+all-formats = ["deflate", "gzip", "bgzf"]
+
+# Serialization formats
+bitcode = ["dep:bitcode"]
+
+# All features
+all = ["all-formats", "bitcode"]

workspace/compress/src/adapter/boxed_sync.rs

@@ -0,0 +1,46 @@
+use crate::{decode, encode};
+use std::io::{BufRead, Error, Read};
+
+/// Adapter that type-erases I/O types into boxed trait objects with `Send + Sync` bounds.
+///
+/// `BoxedSync` is used by the compression/decompression adapters to wrap concrete
+/// `Read`, `BufRead`, and `Write` implementations into `Box<dyn ... + Send + Sync + 'a>`
+/// so they can be passed around as thread-safe, type-erased handles in the public API
+/// and examples.
+pub struct BoxedSync;
+
+impl<'a, In> decode::AdaptRead<'a, In> for BoxedSync
+where
+    In: Read + Send + Sync + 'a,
+{
+    fn wrap(self, internal: In) -> Result<Box<dyn Read + Send + Sync + 'a>, Error> {
+        Ok(Box::new(internal))
+    }
+}
+
+impl<'a, In> decode::AdaptBufRead<'a, In> for BoxedSync
+where
+    In: BufRead + Send + Sync + 'a,
+{
+    fn wrap(self, internal: In) -> Result<Box<dyn BufRead + Send + Sync + 'a>, Error> {
+        Ok(Box::new(internal))
+    }
+}
+
+impl decode::Adapter for BoxedSync {
+    type Read<'a> = Box<dyn Read + Send + Sync + 'a>;
+    type BufRead<'a> = Box<dyn BufRead + Send + Sync + 'a>;
+}
+
+impl<'a, In> encode::AdaptWrite<'a, In> for BoxedSync
+where
+    In: std::io::Write + Send + Sync + 'a,
+{
+    fn wrap(self, internal: In) -> Result<Box<dyn std::io::Write + Send + Sync + 'a>, Error> {
+        Ok(Box::new(internal))
+    }
+}
+
+impl encode::Adapter for BoxedSync {
+    type Write<'a> = Box<dyn std::io::Write + Send + Sync + 'a>;
+}

workspace/compress/src/adapter/mod.rs

@@ -0,0 +1,2 @@
+mod boxed_sync;
+pub use boxed_sync::BoxedSync;

workspace/compress/src/decode/adapter.rs

@@ -0,0 +1,20 @@
+use std::io::{BufRead, Error, Read};
+
+pub trait AdaptRead<'a, In>: Adapter {
+    /// A helper trait to allow introducing flexible constraints on the wrapped reader.
+    ///
+    /// This is normally where you would put constraints on `In` and implement the actual wrapping logic.
+    fn wrap(self, internal: In) -> Result<Self::Read<'a>, Error>;
+}
+
+pub trait AdaptBufRead<'a, In>: Adapter {
+    /// A helper trait to allow introducing flexible constraints on the wrapped buffered reader.
+    ///
+    /// This is normally where you would put constraints on `In` and implement the actual wrapping logic.
+    fn wrap(self, internal: In) -> Result<Self::BufRead<'a>, Error>;
+}
+
+pub trait Adapter {
+    type Read<'a>: Read + 'a;
+    type BufRead<'a>: BufRead + 'a;
+}

workspace/compress/src/decode/bgzf.rs

@@ -0,0 +1,74 @@
+use super::adapter::{AdaptBufRead, AdaptRead, Adapter};
+use super::decode::{
+    DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead,
+};
+use std::io::{BufRead, BufReader, Error, Read};
+use std::num::NonZeroUsize;
+
+#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Bgzf {
+    out_bufsize: NonZeroUsize,
+}
+
+impl Bgzf {
+    pub const DEFAULT: Bgzf = Bgzf {
+        out_bufsize: NonZeroUsize::new(8 * 1024).unwrap(),
+    };
+
+    pub fn new(out_bufsize: NonZeroUsize) -> Self {
+        Self { out_bufsize }
+    }
+
+    pub fn out_bufsize(&self) -> NonZeroUsize {
+        self.out_bufsize
+    }
+}
+
+impl Default for Bgzf {
+    fn default() -> Self {
+        Self::DEFAULT
+    }
+}
+
+impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoRead<'a, R, A> for Bgzf
+where
+    A: AdaptRead<'a, flate2::read::MultiGzDecoder<R>>,
+{
+    fn decode_read_into_read(&self, reader: R, adapter: A) -> Result<A::Read<'a>, Error> {
+        let reader = flate2::read::MultiGzDecoder::new(reader);
+        adapter.wrap(reader)
+    }
+}
+
+impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoBufRead<'a, R, A> for Bgzf
+where
+    A: AdaptBufRead<'a, BufReader<flate2::read::MultiGzDecoder<R>>>,
+{
+    fn decode_read_into_bufread(&self, reader: R, adapter: A) -> Result<A::BufRead<'a>, Error> {
+        let reader = flate2::read::MultiGzDecoder::new(reader);
+        let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader);
+        adapter.wrap(bufreader)
+    }
+}
+
+impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoRead<'a, R, A> for Bgzf
+where
+    A: AdaptRead<'a, flate2::bufread::MultiGzDecoder<R>>,
+{
+    fn decode_bufread_into_read(&self, reader: R, adapter: A) -> Result<A::Read<'a>, Error> {
+        let reader = flate2::bufread::MultiGzDecoder::new(reader);
+        adapter.wrap(reader)
+    }
+}
+
+impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoBufRead<'a, R, A> for Bgzf
+where
+    A: AdaptBufRead<'a, BufReader<flate2::bufread::MultiGzDecoder<R>>>,
+{
+    fn decode_bufread_into_bufread(&self, reader: R, adapter: A) -> Result<A::BufRead<'a>, Error> {
+        let reader = flate2::bufread::MultiGzDecoder::new(reader);
+        let bufreader = BufReader::with_capacity(self.out_bufsize.get(), reader);
+        adapter.wrap(bufreader)
+    }
+}

workspace/compress/src/decode/decode.rs

@@ -0,0 +1,18 @@
+use super::adapter::Adapter;
+use std::io::{BufRead, Error, Read};
+
+pub trait DecodeReadIntoRead<'a, R: Read + 'a, A: Adapter> {
+    fn decode_read_into_read(&self, reader: R, adapter: A) -> Result<A::Read<'a>, Error>;
+}
+
+pub trait DecodeReadIntoBufRead<'a, R: Read + 'a, A: Adapter> {
+    fn decode_read_into_bufread(&self, reader: R, adapter: A) -> Result<A::BufRead<'a>, Error>;
+}
+
+pub trait DecodeBufReadIntoRead<'a, R: BufRead + 'a, A: Adapter> {
+    fn decode_bufread_into_read(&self, reader: R, adapter: A) -> Result<A::Read<'a>, Error>;
+}
+
+pub trait DecodeBufReadIntoBufRead<'a, R: BufRead + 'a, A: Adapter> {
+    fn decode_bufread_into_bufread(&self, reader: R, adapter: A) -> Result<A::BufRead<'a>, Error>;
+}

workspace/compress/src/decode/decoder.rs

@@ -0,0 +1,76 @@
+use super::adapter::Adapter;
+use super::decode::{
+    DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead,
+};
+use std::io::{BufRead, Error, Read};
+
+#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Decoder {
+    /// No compression (identity) decoding algorithm.
+    ///
+    /// The option is useful to uniformly handle both compressed and uncompressed data streams.
+    Identity(crate::decode::Identity),
+
+    /// DEFLATE compression algorithm
+    #[cfg(feature = "decode-deflate")]
+    Deflate(crate::decode::Deflate),
+
+    /// GZIP container; Only supports DEFLATE compression.
+    #[cfg(feature = "decode-gzip")]
+    Gzip(crate::decode::Gzip),
+
+    /// BGZF container; Only supports DEFLATE compression.
+    #[cfg(feature = "decode-bgzf")]
+    Bgzf(crate::decode::Bgzf),
+}
+
+impl Default for Decoder {
+    fn default() -> Self {
+        Decoder::Identity(Default::default())
+    }
+}
+
+macro_rules! impl_decompression_trait {
+    ($read:ident, $out:ident, $trait:ident, $method:ident) => {
+        impl<'a, R: $read + 'a, A: Adapter> $trait<'a, R, A> for Decoder
+        where
+            crate::decode::Identity: $trait<'a, R, A>,
+            crate::decode::Deflate: $trait<'a, R, A>,
+            crate::decode::Gzip: $trait<'a, R, A>,
+            crate::decode::Bgzf: $trait<'a, R, A>,
+        {
+            fn $method(&self, reader: R, adapter: A) -> Result<A::$out<'a>, Error> {
+                match self {
+                    Decoder::Identity(codec) => codec.$method(reader, adapter),
+                    #[cfg(feature = "decode-deflate")]
+                    Decoder::Deflate(codec) => codec.$method(reader, adapter),
+                    #[cfg(feature = "decode-gzip")]
+                    Decoder::Gzip(codec) => codec.$method(reader, adapter),
+                    #[cfg(feature = "decode-bgzf")]
+                    Decoder::Bgzf(codec) => codec.$method(reader, adapter),
+                }
+            }
+        }
+    };
+}
+
+impl_decompression_trait!(Read, Read, DecodeReadIntoRead, decode_read_into_read);
+impl_decompression_trait!(
+    Read,
+    BufRead,
+    DecodeReadIntoBufRead,
+    decode_read_into_bufread
+);
+impl_decompression_trait!(
+    BufRead,
+    Read,
+    DecodeBufReadIntoRead,
+    decode_bufread_into_read
+);
+impl_decompression_trait!(
+    BufRead,
+    BufRead,
+    DecodeBufReadIntoBufRead,
+    decode_bufread_into_bufread
+);

workspace/compress/src/decode/deflate.rs

@@ -0,0 +1,83 @@
+use super::adapter::{AdaptBufRead, AdaptRead, Adapter};
+use super::decode::{
+    DecodeBufReadIntoBufRead, DecodeBufReadIntoRead, DecodeReadIntoBufRead, DecodeReadIntoRead,
+};
+use std::io;
+use std::io::{BufRead, Error, Read};
+use std::num::NonZeroUsize;
+
+#[cfg_attr(feature = "bitcode", derive(::bitcode::Encode, ::bitcode::Decode))]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Deflate {
+    in_bufsize: NonZeroUsize,
+    out_bufsize: NonZeroUsize,
+}
+
+impl Deflate {
+    pub const DEFAULT: Deflate = Deflate {
+        in_bufsize: NonZeroUsize::new(32 * 1024).unwrap(),
+        out_bufsize: NonZeroUsize::new(8 * 1024).unwrap(),
+    };
+
+    pub fn new(deflate_bufsize: NonZeroUsize, out_bufsize: NonZeroUsize) -> Self {
+        Self {
+            in_bufsize: deflate_bufsize,
+            out_bufsize,
+        }
+    }
+
+    pub fn in_bufsize(&self) -> usize {
+        self.in_bufsize.get()
+    }
+
+    pub fn out_bufsize(&self) -> usize {
+        self.out_bufsize.get()
+    }
+}
+
+impl Default for Deflate {
+    fn default() -> Self {
+        Deflate::DEFAULT
+    }
+}
+
+impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoRead<'a, R, A> for Deflate
+where
+    A: AdaptRead<'a, flate2::read::DeflateDecoder<R>>,
+{
+    fn decode_read_into_read(&self, reader: R, adapter: A) -> Result<A::Read<'a>, Error> {
+        adapter.wrap(flate2::read::DeflateDecoder::new(reader))
+    }
+}
+
+impl<'a, R: Read + 'a, A: Adapter> DecodeReadIntoBufRead<'a, R, A> for Deflate
+where
+    A: AdaptBufRead<'a, io::BufReader<flate2::read::DeflateDecoder<R>>>,
+{
+    fn decode_read_into_bufread(&self, reader: R, adapter: A) -> Result<A::BufRead<'a>, Error> {
+        let reader = flate2::read::DeflateDecoder::new(reader);
+        let bufreader = io::BufReader::with_capacity(self.out_bufsize.get(), reader);
+        adapter.wrap(bufreader)
+    }
+}
+
+impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoRead<'a, R, A> for Deflate
+where
+    A: AdaptRead<'a, flate2::bufread::DeflateDecoder<R>>,
+{
+    fn decode_bufread_into_read(&self, reader: R, adapter: A) -> Result<A::Read<'a>, Error> {
+        let reader = flate2::bufread::DeflateDecoder::new(reader);
+        adapter.wrap(reader)
+    }
+}
+
+impl<'a, R: BufRead + 'a, A: Adapter> DecodeBufReadIntoBufRead<'a, R, A> for Deflate
+where
+    A: AdaptBufRead<'a, io::BufReader<flate2::bufread::DeflateDecoder<R>>>,
+{
+    fn decode_bufread_into_bufread(&self, reader: R, adapter: A) -> Result<A::BufRead<'a>, Error> {
+        let reader = flate2::bufread::DeflateDecoder::new(reader);
+        let bufreader = io::BufReader::with_capacity(self.out_bufsize.get(), reader);
+        adapter.wrap(bufreader)
+    }
+}