diff --git a/minarrow-py/Cargo.lock b/minarrow-py/Cargo.lock index c362336..4a28354 100644 --- a/minarrow-py/Cargo.lock +++ b/minarrow-py/Cargo.lock @@ -30,7 +30,6 @@ checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" name = "minarrow" version = "0.16.2" dependencies = [ - "libc", "log", "num-traits", "vec64", diff --git a/minarrow-py/Cargo.toml b/minarrow-py/Cargo.toml index 95174ef..e237a1d 100644 --- a/minarrow-py/Cargo.toml +++ b/minarrow-py/Cargo.toml @@ -53,7 +53,7 @@ large_string = ["minarrow/large_string"] scalar_type = ["minarrow/scalar_type"] value_type = ["minarrow/value_type"] cube = ["minarrow/cube"] -default_categorical_8 = ["minarrow/default_categorical_8"] +default_categorical_8 = ["minarrow/default_categorical_8", "minarrow-pyo3?/default_categorical_8"] # Mirrors the core crate, where `extended_categorical` implies `default_categorical_8`. # `minarrow-pyo3` needs `extended_numeric_types` too, as its dictionary-index conversion # gates the 8 and 16-bit key arms on both features. diff --git a/minarrow-py/src/lib.rs b/minarrow-py/src/lib.rs index 3572f01..4aef44a 100644 --- a/minarrow-py/src/lib.rs +++ b/minarrow-py/src/lib.rs @@ -37,8 +37,8 @@ mod xarray; use pyo3::prelude::*; -use array::PyArray; -use table::PyTable; +pub use array::PyArray; +pub use table::PyTable; pub use array::PyArrayInner; pub use arrow_type::{PyArrowType, PyCategoricalIndexType}; diff --git a/pyo3/Cargo.lock b/pyo3/Cargo.lock index 8a24d58..6a33543 100644 --- a/pyo3/Cargo.lock +++ b/pyo3/Cargo.lock @@ -30,7 +30,6 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" name = "minarrow" version = "0.16.2" dependencies = [ - "libc", "log", "num-traits", "vec64", diff --git a/pyo3/Cargo.toml b/pyo3/Cargo.toml index c63d6e5..b71b84d 100644 --- a/pyo3/Cargo.toml +++ b/pyo3/Cargo.toml @@ -33,6 +33,11 @@ default = ["datetime", "extended_numeric_types"] extension-module = ["pyo3/extension-module"] datetime = ["minarrow/datetime"] extended_numeric_types = ["minarrow/extended_numeric_types"] +# Mirrors the core crate, using 8-bit key (256 max unique categorical values) instead of 32-bit +# for efficiency +default_categorical_8 = ["minarrow/default_categorical_8"] +# Enables 8, 16 and 64 byte categorical bit widths. When combined with default_categorical_8, +# the extra one added is the 32 byte (making the other feature unused). extended_categorical = ["minarrow/extended_categorical"] table_metadata = ["minarrow/table_metadata"] # N-dimensional tensor bridging with DLPack capsule interchange. diff --git a/pyo3/src/ffi/to_py.rs b/pyo3/src/ffi/to_py.rs index 204f7df..d6b25c1 100644 --- a/pyo3/src/ffi/to_py.rs +++ b/pyo3/src/ffi/to_py.rs @@ -131,10 +131,14 @@ fn arrow_type_to_pyarrow<'py>( ArrowType::Dictionary(key_type) => { let index_ty = match key_type { - #[cfg(all(feature = "extended_categorical", feature = "extended_numeric_types"))] + #[cfg(feature = "default_categorical_8")] CategoricalIndexType::UInt8 => pa.call_method0("uint8")?, - #[cfg(all(feature = "extended_categorical", feature = "extended_numeric_types"))] + #[cfg(feature = "extended_categorical")] CategoricalIndexType::UInt16 => pa.call_method0("uint16")?, + #[cfg(any( + not(feature = "default_categorical_8"), + feature = "extended_categorical" + ))] CategoricalIndexType::UInt32 => pa.call_method0("uint32")?, #[cfg(feature = "extended_categorical")] CategoricalIndexType::UInt64 => pa.call_method0("uint64")?, diff --git a/src/ffi/arrow_dtype.rs b/src/ffi/arrow_dtype.rs index d185be5..f2a20ef 100644 --- a/src/ffi/arrow_dtype.rs +++ b/src/ffi/arrow_dtype.rs @@ -17,7 +17,7 @@ //! Unified Minarrow representations of supported *Apache Arrow* data types. //! //! ## Overview -//! - Covers integer, floating-point, boolean, string, dictionary-encoded, and optional temporal types +//! - Covers integer, floating-point, boolean, string, dictionary-encoded, and optional temporal types //! (date, time, duration, timestamp, interval). //! - Each Minarrow array type implements `arrow_type()` to return its matching `ArrowType`. //! - Enables consistent Arrow FFI compatibility without requiring the full Arrow type system. @@ -37,7 +37,7 @@ //! ## Copyright Notice //! - The `Minarrow` crate is not affiliated with the `Apache Arrow` project. //! - The term `Apache Arrow` is a trademark of the *Apache Software Foundation*. -//! - The term `Arrow` is used here under fair use to implement the public FFI compatibility standard, +//! - The term `Arrow` is used here under fair use to implement the public FFI compatibility standard, //! in accordance with the official guidance: . //! //! See `./LICENSE` for more information. @@ -58,7 +58,7 @@ use crate::{BooleanArray, CategoricalArray, Float, FloatArray, Integer, StringAr /// ## Purpose /// - Encodes the physical type and, for temporal variants, associated unit information for all supported Minarrow arrays. /// - Provides a single discriminant used across the crate for schema definitions, type matching, and Arrow FFI export. -/// - Implements a focused subset of the official Arrow type specification: +/// - Implements a focused subset of the official Arrow type specification: /// . /// /// ## Coverage @@ -74,7 +74,7 @@ use crate::{BooleanArray, CategoricalArray, Float, FloatArray, Integer, StringAr /// - Simplifies Minarrow’s type system *(e.g., one `DatetimeArray` type)* while tagging `ArrowType` on `Field` for ecosystem compatibility. /// /// ## Notes -/// - For `DatetimeArray` types, `ArrowType` reflects only the physical encoding. +/// - For `DatetimeArray` types, `ArrowType` reflects only the physical encoding. /// Logical distinctions (e.g., interpreting a `Date64` as a timestamp vs. a duration) are stored in `Field` metadata. /// - Dictionary key widths are defined by the associated `CategoricalIndexType`. #[derive(PartialEq, Eq, Hash, Clone, Debug)] @@ -123,6 +123,40 @@ pub enum ArrowType { } impl ArrowType { + /// High-level typing categories: Numeric, Text, Boolean, + /// Datetime, or `Null`. + /// + /// These are not part of the Apache Arrow specification. + pub fn minarrow_category_label(&self) -> &'static str { + match self { + ArrowType::Null => "Null", + ArrowType::Boolean => "Boolean", + #[cfg(feature = "extended_numeric_types")] + ArrowType::Int8 | ArrowType::Int16 | ArrowType::UInt8 | ArrowType::UInt16 => { + "Numeric" + } + ArrowType::Int32 + | ArrowType::Int64 + | ArrowType::UInt32 + | ArrowType::UInt64 + | ArrowType::Float32 + | ArrowType::Float64 => "Numeric", + #[cfg(feature = "datetime")] + ArrowType::Date32 + | ArrowType::Date64 + | ArrowType::Time32(_) + | ArrowType::Time64(_) + | ArrowType::Duration32(_) + | ArrowType::Duration64(_) + | ArrowType::Timestamp(_, _) + | ArrowType::Interval(_) => "Datetime", + ArrowType::String | ArrowType::Utf8View => "Text", + #[cfg(feature = "large_string")] + ArrowType::LargeString => "Text", + ArrowType::Dictionary(_) => "Text", + } + } + /// Upcast target for a binary operation over a pair of input types. /// /// Returns the element type that carries the result of a binary