diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ec8667..78c4eba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,7 @@ where any bugs or reasonable feature requests may be requested anonymously / tre ### Added **NdArray, XArray and DLPack Feature**: -- `NdArray` n-dimensional dense array (`ndarray` feature) over `Buffer`, +- `NdArray` n-dimensional contiguous array (`ndarray` feature) over `Buffer`, generic over f32/f64, with NaN null semantics, `NdArrayV` zero-copy views, transpose and axis permutation, and Table/Matrix/Array interop. - `SuperNdArray` chunked n-dimensional array with `SuperNdArrayV` diff --git a/README.md b/README.md index 780bb69..1ab88ed 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ Six typed arrays back standard workloads: | `IntegerArray` | i8 through u64 | | `FloatArray` | f32, f64 | | `StringArray` | UTF-8 with u32 or u64 offsets | -| `BooleanArray` | Bit-packed with validity mask | +| `BooleanArray` | Bit-packed with null mask | | `CategoricalArray` | Dictionary-encoded | | `DatetimeArray` | Timestamps, dates, durations | diff --git a/pyo3/README.md b/pyo3/README.md index 323af70..8d4de1c 100644 --- a/pyo3/README.md +++ b/pyo3/README.md @@ -107,7 +107,7 @@ Unless you have a very large unique category count, this should not cause perfor ### Nullability -All array types support null values via MinArrow's `MaskedArray` wrapper. The validity bitmap is transferred through the Arrow C Data Interface and PyArrow reconstructs the same null positions on import. +All array types support null values via MinArrow's `MaskedArray` wrapper. The null mask is transferred through the Arrow C Data Interface and PyArrow reconstructs the same null positions on import. ## Conversion Path diff --git a/pyo3/src/ffi/mod.rs b/pyo3/src/ffi/mod.rs index 85792a4..e5854df 100644 --- a/pyo3/src/ffi/mod.rs +++ b/pyo3/src/ffi/mod.rs @@ -166,7 +166,7 @@ //! ## Nullability //! //! All array types support null values via MinArrow's `MaskedArray` wrapper. When an -//! array contains nulls, the validity bitmap is transferred through the Arrow C Data +//! array contains nulls, the null mask is transferred through the Arrow C Data //! Interface. PyArrow reconstructs the same null positions on import. //! //! ## Ownership Model diff --git a/src/conversions.rs b/src/conversions.rs index c10be72..9ad77af 100644 --- a/src/conversions.rs +++ b/src/conversions.rs @@ -58,6 +58,7 @@ use std::convert::{From, TryFrom}; use std::sync::Arc; use crate::enums::error::MinarrowError; +use crate::traits::concatenate::Concatenate; #[cfg(feature = "views")] use crate::traits::view::View; use crate::{ @@ -1295,6 +1296,51 @@ impl View for BooleanArray<()> { // From Scalar for Array // -------------------------------- +impl TryFrom> for Array { + type Error = MinarrowError; + + + /// Joins a sequence of arrays end to end. + /// + /// The pieces must share an element type, which the join itself + /// enforces. An empty sequence yields the default array. + fn try_from(value: Vec) -> Result { + let mut pieces = value.into_iter(); + let Some(mut joined) = pieces.next() else { + return Ok(Array::default()); + }; + for next in pieces { + joined = joined.concat(next)?; + } + Ok(joined) + } +} + +#[cfg(feature = "scalar_type")] +impl TryFrom for Scalar { + type Error = MinarrowError; + + /// Reads a single-element array as that element. + /// + /// Rejects lengths > 1. + fn try_from(value: Array) -> Result { + match value.len() { + 1 => value.get_scalar(0).ok_or_else(|| MinarrowError::TypeError { + from: "Array", + to: "Scalar", + message: Some("the element could not be read as a scalar".to_owned()), + }), + n => Err(MinarrowError::TypeError { + from: "Array", + to: "Scalar", + message: Some(format!( + "a single value is needed, the array held {n} elements" + )), + }), + } + } +} + #[cfg(feature = "scalar_type")] impl From for Array { fn from(scalar: Scalar) -> Self { diff --git a/src/enums/array.rs b/src/enums/array.rs index 255a323..396b1f5 100644 --- a/src/enums/array.rs +++ b/src/enums/array.rs @@ -26,6 +26,8 @@ use std::any::TypeId; use std::fmt::{Display, Formatter}; +#[cfg(feature = "scalar_type")] +use std::ops::Range; use std::sync::Arc; #[cfg(any(feature = "cast_arrow", feature = "cast_polars"))] @@ -974,6 +976,8 @@ impl Array { } /// Sets the element at `idx` to `value`, converting it to the element type. + /// A `Scalar::Null` value masks the element null and leaves the buffer + /// contents in place. /// /// Returns an error when the value cannot be represented as the array's type, /// or the array is `Null`. Mutation is copy-on-write. @@ -985,6 +989,18 @@ impl Array { to: "array element", message: Some(format!("cannot set a value in a {dtype:?} array")), }; + if matches!(value, crate::Scalar::Null) { + if matches!(self, Array::Null) { + return Err(unsupported()); + } + let mut mask = self + .null_mask() + .cloned() + .unwrap_or_else(|| Bitmask::new_set_all(self.len(), true)); + mask.set(idx, false); + self.set_null_mask(mask); + return Ok(()); + } match self { Array::NumericArray(inner) => match inner { #[cfg(feature = "extended_numeric_types")] @@ -1070,6 +1086,241 @@ impl Array { Ok(()) } + /// Sets every element in `range` to `value`, converting it to the element type. + /// + /// The scalar converts once and each variant then writes through its own + /// typed buffer, so the per-element work carries no dispatch. A + /// `Scalar::Null` value masks the whole range null and leaves the buffer + /// contents in place. Categorical arrays intern the value into the + /// dictionary once and repeat its code across the range. + /// + /// Returns an error when the value cannot be represented as the array's + /// type, when the range reaches past the array's length, or when the + /// array is `Null`. Mutation is copy-on-write. + #[cfg(feature = "scalar_type")] + pub fn set_range( + &mut self, + range: Range, + value: crate::Scalar, + ) -> Result<(), MinarrowError> { + use crate::Scalar; + let dtype = self.arrow_type(); + let unsupported = || MinarrowError::TypeError { + from: "scalar", + to: "array element", + message: Some(format!("cannot set a value in a {dtype:?} array")), + }; + if range.end > self.len() { + return Err(MinarrowError::IndexError(format!( + "set_range: range {}..{} exceeds array length {}", + range.start, + range.end, + self.len() + ))); + } + if range.is_empty() { + return Ok(()); + } + if matches!(value, Scalar::Null) { + if matches!(self, Array::Null) { + return Err(unsupported()); + } + let mut mask = self + .null_mask() + .cloned() + .unwrap_or_else(|| Bitmask::new_set_all(self.len(), true)); + mask.set_range(range.start, range.end, false); + self.set_null_mask(mask); + return Ok(()); + } + match self { + Array::NumericArray(inner) => match inner { + #[cfg(feature = "extended_numeric_types")] + NumericArray::Int8(a) => { + let v = value.try_i8().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.data.as_mut_slice()[range.clone()].fill(v); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + #[cfg(feature = "extended_numeric_types")] + NumericArray::Int16(a) => { + let v = value.try_i16().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.data.as_mut_slice()[range.clone()].fill(v); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + NumericArray::Int32(a) => { + let v = value.try_i32().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.data.as_mut_slice()[range.clone()].fill(v); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + NumericArray::Int64(a) => { + let v = value.try_i64().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.data.as_mut_slice()[range.clone()].fill(v); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + #[cfg(feature = "extended_numeric_types")] + NumericArray::UInt8(a) => { + let v = value.try_u8().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.data.as_mut_slice()[range.clone()].fill(v); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + #[cfg(feature = "extended_numeric_types")] + NumericArray::UInt16(a) => { + let v = value.try_u16().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.data.as_mut_slice()[range.clone()].fill(v); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + NumericArray::UInt32(a) => { + let v = value.try_u32().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.data.as_mut_slice()[range.clone()].fill(v); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + NumericArray::UInt64(a) => { + let v = value.try_u64().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.data.as_mut_slice()[range.clone()].fill(v); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + NumericArray::Float32(a) => { + let v = value.try_f32().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.data.as_mut_slice()[range.clone()].fill(v); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + NumericArray::Float64(a) => { + let v = value.try_f64().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.data.as_mut_slice()[range.clone()].fill(v); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + NumericArray::Null => return Err(unsupported()), + }, + Array::BooleanArray(a) => { + let v = value.try_bool().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.data.set_range(range.start, range.end, v); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + Array::TextArray(inner) => match inner { + TextArray::String32(a) => { + let v = value.try_str().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + for idx in range.clone() { + arr.set_str(idx, &v); + } + } + #[cfg(feature = "large_string")] + TextArray::String64(a) => { + let v = value.try_str().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + for idx in range.clone() { + arr.set_str(idx, &v); + } + } + #[cfg(feature = "default_categorical_8")] + TextArray::Categorical8(a) => { + let v = value.try_str().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + // The first element goes through set_str so the dictionary + // interning happens once. The rest repeat its code. + arr.set_str(range.start, &v); + let code = arr.data[range.start]; + arr.data.as_mut_slice()[range.start + 1..range.end].fill(code); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + #[cfg(feature = "extended_categorical")] + TextArray::Categorical16(a) => { + let v = value.try_str().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.set_str(range.start, &v); + let code = arr.data[range.start]; + arr.data.as_mut_slice()[range.start + 1..range.end].fill(code); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + #[cfg(any( + not(feature = "default_categorical_8"), + feature = "extended_categorical" + ))] + TextArray::Categorical32(a) => { + let v = value.try_str().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.set_str(range.start, &v); + let code = arr.data[range.start]; + arr.data.as_mut_slice()[range.start + 1..range.end].fill(code); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + #[cfg(feature = "extended_categorical")] + TextArray::Categorical64(a) => { + let v = value.try_str().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.set_str(range.start, &v); + let code = arr.data[range.start]; + arr.data.as_mut_slice()[range.start + 1..range.end].fill(code); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + TextArray::Null => return Err(unsupported()), + }, + #[cfg(feature = "datetime")] + Array::TemporalArray(inner) => match inner { + TemporalArray::Datetime32(a) => { + let v = value.try_i32().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.data.as_mut_slice()[range.clone()].fill(v); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + TemporalArray::Datetime64(a) => { + let v = value.try_i64().ok_or_else(unsupported)?; + let arr = Arc::make_mut(a); + arr.data.as_mut_slice()[range.clone()].fill(v); + if let Some(mask) = &mut arr.null_mask { + mask.set_range(range.start, range.end, true); + } + } + TemporalArray::Null => return Err(unsupported()), + }, + Array::Null => return Err(unsupported()), + } + Ok(()) + } + /// Returns a metadata view and reference over the specified window of this array. /// /// Does not slice the object (yet). @@ -4887,6 +5138,93 @@ mod tests { } } + #[cfg(feature = "scalar_type")] + #[test] + fn test_array_set_range_numeric() { + let mut arr = Array::from_int64(IntegerArray::::from_slice(&[1, 2, 3, 4, 5])); + arr.set_range(1..4, crate::Scalar::Int64(9)).unwrap(); + if let Array::NumericArray(NumericArray::Int64(a)) = &arr { + assert_eq!(a.data.as_slice(), &[1, 9, 9, 9, 5]); + assert!(a.null_mask.is_none(), "a value write must not create a mask"); + } else { + panic!("wrong variant"); + } + } + + #[cfg(feature = "scalar_type")] + #[test] + fn test_array_set_range_null_scalar_masks_the_range() { + let mut arr = Array::from_int64(IntegerArray::::from_slice(&[1, 2, 3])); + arr.set_range(0..2, crate::Scalar::Null).unwrap(); + let mask = arr.null_mask().expect("the null write creates the mask"); + assert!(!mask.get(0)); + assert!(!mask.get(1)); + assert!(mask.get(2)); + } + + #[cfg(feature = "scalar_type")] + #[test] + fn test_array_set_range_revalidates_masked_entries() { + let mut arr = Array::from_int64(IntegerArray::::from_slice(&[1, 2, 3])); + arr.set_range(0..3, crate::Scalar::Null).unwrap(); + arr.set_range(1..3, crate::Scalar::Int64(7)).unwrap(); + let mask = arr.null_mask().expect("the mask persists once created"); + assert!(!mask.get(0)); + assert!(mask.get(1)); + assert!(mask.get(2)); + if let Array::NumericArray(NumericArray::Int64(a)) = &arr { + assert_eq!(a.data.as_slice()[1..], [7, 7]); + } else { + panic!("wrong variant"); + } + } + + #[cfg(feature = "scalar_type")] + #[test] + fn test_array_set_range_string() { + let mut arr = + Array::from_string32(StringArray::::from_slice(&["a", "bb", "ccc", "d"])); + arr.set_range(1..3, crate::Scalar::String32("zz".into())).unwrap(); + if let Array::TextArray(TextArray::String32(a)) = &arr { + assert_eq!(a.get(0), Some("a")); + assert_eq!(a.get(1), Some("zz")); + assert_eq!(a.get(2), Some("zz")); + assert_eq!(a.get(3), Some("d")); + assert!(a.null_mask.is_none(), "a value write must not create a mask"); + } else { + panic!("wrong variant"); + } + } + + #[cfg(all(feature = "scalar_type", feature = "default_categorical_8"))] + #[test] + fn test_array_set_range_categorical_interns_once() { + let mut arr = Array::from_categorical8(CategoricalArray::::from_slices( + &[0, 1, 0], + &["red".to_string(), "blue".to_string()], + )); + arr.set_range(0..3, crate::Scalar::String32("green".into())).unwrap(); + if let Array::TextArray(TextArray::Categorical8(a)) = &arr { + assert_eq!(a.unique_values(), &["red", "blue", "green"]); + assert_eq!(a.data.as_slice(), &[2, 2, 2]); + } else { + panic!("wrong variant"); + } + } + + #[cfg(feature = "scalar_type")] + #[test] + fn test_array_set_range_rejects_unrepresentable_and_oversize() { + let mut arr = Array::from_int64(IntegerArray::::from_slice(&[1, 2, 3])); + assert!(arr.set_range(0..2, crate::Scalar::String32("pear".into())).is_err()); + assert!(arr.set_range(1..4, crate::Scalar::Int64(9)).is_err()); + if let Array::NumericArray(NumericArray::Int64(a)) = &arr { + assert_eq!(a.data.as_slice(), &[1, 2, 3], "a failed write leaves the data alone"); + } else { + panic!("wrong variant"); + } + } + #[test] fn test_array_is_nullable() { assert!(Array::Null.is_nullable()); @@ -5536,7 +5874,7 @@ mod macro_tests { // ===== numeric types ===== #[test] - fn arr_i32_vec64_dense() { + fn arr_i32_vec64_contiguous() { let v = vec64![1i32, 2, 3]; let arr = arr_i32!(v); if let Array::NumericArray(NumericArray::Int32(a)) = arr { @@ -5560,7 +5898,7 @@ mod macro_tests { } #[test] - fn arr_f64_vec64_dense() { + fn arr_f64_vec64_contiguous() { let v = vec64![1.1f64, 2.2, 3.3]; let arr = arr_f64!(v); if let Array::NumericArray(NumericArray::Float64(a)) = arr { @@ -5586,7 +5924,7 @@ mod macro_tests { // ===== bool ===== #[test] - fn arr_bool_vec64_dense() { + fn arr_bool_vec64_contiguous() { let v = vec64![true, false, true]; let arr = arr_bool!(v); if let Array::BooleanArray(a) = arr { @@ -5616,7 +5954,7 @@ mod macro_tests { // ===== string ===== #[test] - fn arr_str32_vec64_dense() { + fn arr_str32_vec64_contiguous() { let v = vec64!["a", "b", "c"]; let arr = arr_str32!(v); if let Array::TextArray(TextArray::String32(a)) = arr { @@ -5650,7 +5988,7 @@ mod macro_tests { feature = "extended_categorical" ))] #[test] - fn arr_cat32_vec64_dense() { + fn arr_cat32_vec64_contiguous() { let v = vec64!["red", "green", "red"]; let arr = arr_cat32!(v); if let Array::TextArray(TextArray::Categorical32(a)) = arr { @@ -6294,7 +6632,7 @@ mod arr_macro_extensions_tests { } #[test] - fn arr_f64_accepts_slice_dense() { + fn arr_f64_accepts_slice_contiguous() { let s: &[f64] = &[1.0, 2.0, 3.0]; let (data, mask) = expect_f64_mask(arr_f64!(s)); assert_eq!(data, vec![1.0, 2.0, 3.0]); @@ -6302,7 +6640,7 @@ mod arr_macro_extensions_tests { } #[test] - fn arr_f64_accepts_vec64_dense() { + fn arr_f64_accepts_vec64_contiguous() { let v: Vec64 = vec64![1.0, 2.0, 3.0]; let (data, mask) = expect_f64_mask(arr_f64!(v)); assert_eq!(data, vec![1.0, 2.0, 3.0]); diff --git a/src/enums/operators.rs b/src/enums/operators.rs index da6b3ae..cffb126 100644 --- a/src/enums/operators.rs +++ b/src/enums/operators.rs @@ -25,7 +25,7 @@ pub enum ArithmeticOperator { Multiply, /// Division (`lhs / rhs`) /// - /// For integers, division by zero panics in dense arrays and nullifies in masked arrays. + /// For integers, division by zero panics in unmasked arrays and nullifies in masked arrays. /// For floating-point, follows IEEE 754 (yields ±Inf or NaN). Divide, /// Modulus/remainder operation (`lhs % rhs`) diff --git a/src/enums/value/conversions.rs b/src/enums/value/conversions.rs index 566be54..c392372 100644 --- a/src/enums/value/conversions.rs +++ b/src/enums/value/conversions.rs @@ -34,8 +34,6 @@ use crate::SuperNdArray; use crate::SuperNdArrayV; #[cfg(feature = "xarray")] use crate::XArray; -use crate::traits::concatenate::Concatenate; -use crate::traits::masked_array::MaskedArray; use crate::{Array, FieldArray, Table, enums::error::MinarrowError}; use std::convert::TryFrom; use std::sync::Arc; @@ -624,31 +622,34 @@ macro_rules! impl_value_from { }; } -/// Macro to implement `TryFrom` for `Value` variants. -macro_rules! impl_tryfrom_value { - ($variant:ident: $t:ty) => { - impl TryFrom for $t { - type Error = MinarrowError; - fn try_from(v: Value) -> Result { - match v { - Value::$variant(inner) => Ok(inner), - _ => Err(MinarrowError::TypeError { - from: "Value", - to: stringify!($t), - message: Some("Value type mismatch".to_owned()), - }), - } - } - } - }; -} - // Scalar Conversions #[cfg(feature = "scalar_type")] impl_value_from!(Scalar: Scalar); #[cfg(feature = "scalar_type")] -impl_tryfrom_value!(Scalar: Scalar); +impl TryFrom for Scalar { + type Error = MinarrowError; + + /// Takes the scalar carried by `Value::Scalar`, unwrapping the recursive + /// `BoxValue` and `ArcValue` wrappers first. + fn try_from(v: Value) -> Result { + match v { + Value::Scalar(s) => Ok(s), + Value::BoxValue(inner) => Scalar::try_from(*inner), + Value::ArcValue(inner) => { + Scalar::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + other => Err(MinarrowError::TypeError { + from: value_variant_name(&other), + to: "Scalar", + message: Some(format!( + "Value::{} does not convert to Scalar", + value_variant_name(&other) + )), + }), + } + } +} // Primitive numerics -> Value (wrap as Value::Scalar). @@ -752,20 +753,13 @@ macro_rules! impl_tryfrom_value_numeric { type Error = MinarrowError; #[inline] fn try_from(v: Value) -> Result { - match v { - Value::Scalar(s) => match s { - Scalar::Null => Err(MinarrowError::TypeError { - from: "Value::Scalar(Null)", - to: stringify!($t), - message: Some("Cannot convert Null to numeric type".to_owned()), - }), - _ => Ok(s.$method()), - }, - _ => Err(MinarrowError::TypeError { - from: "Value", + match Scalar::try_from(v)? { + Scalar::Null => Err(MinarrowError::TypeError { + from: "Value::Scalar(Null)", to: stringify!($t), - message: Some("Expected Value::Scalar variant".to_owned()), + message: Some("a null value has no numeric reading".to_owned()), }), + s => Ok(s.$method()), } } } @@ -795,16 +789,9 @@ macro_rules! impl_tryfrom_value_option { type Error = MinarrowError; #[inline] fn try_from(v: Value) -> Result { - match v { - Value::Scalar(s) => match s { - Scalar::Null => Ok(None), - _ => Ok(Some(s.$method())), - }, - _ => Err(MinarrowError::TypeError { - from: "Value", - to: concat!("Option<", stringify!($t), ">"), - message: Some("Expected Value::Scalar variant".to_owned()), - }), + match Scalar::try_from(v)? { + Scalar::Null => Ok(None), + s => Ok(Some(s.$method())), } } } @@ -983,7 +970,7 @@ impl From for Value { /// Wrap a `BitmaskV` as `Value::ArrayView` of a `BooleanArray`. The bitmask /// data is taken as the BooleanArray's data; `null_mask` is `None` since -/// `BitmaskV` carries no separate validity. +/// `BitmaskV` carries no separate null mask. #[cfg(feature = "views")] impl<'a> From> for Value { #[inline] @@ -1015,10 +1002,13 @@ impl From for Value { impl TryFrom for Array { type Error = MinarrowError; fn try_from(v: Value) -> Result { - let err = || MinarrowError::TypeError { - from: "Value", + let err = |v: &Value| MinarrowError::TypeError { + from: value_variant_name(v), to: "Array", - message: Some("Value type mismatch".to_owned()), + message: Some(format!( + "Value::{} does not convert to Array", + value_variant_name(v) + )), }; match v { Value::Array(inner) => Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())), @@ -1033,58 +1023,67 @@ impl TryFrom for Array { } #[cfg(feature = "scalar_type")] Value::Scalar(s) => Ok(Array::from(s)), - Value::Table(_) => Err(err()), + Value::BoxValue(inner) => Array::try_from(*inner), + Value::ArcValue(inner) => { + Array::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + Value::Table(inner) => { + Array::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } #[cfg(feature = "views")] - Value::TableView(_) => Err(err()), + Value::TableView(inner) => Array::try_from(Table::from( + Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()), + )), #[cfg(feature = "chunked")] - Value::SuperArray(_) => Err(err()), + Value::SuperArray(inner) => { + Array::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } #[cfg(all(feature = "chunked", feature = "views"))] - Value::SuperArrayView(_) => Err(err()), + Value::SuperArrayView(inner) => { + let view = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); + Array::try_from(SuperArray::from(view)) + } #[cfg(feature = "chunked")] - Value::SuperTable(_) => Err(err()), + Value::SuperTable(_) => Array::try_from(Table::try_from(v)?), #[cfg(all(feature = "chunked", feature = "views"))] - Value::SuperTableView(_) => Err(err()), + Value::SuperTableView(_) => Array::try_from(Table::try_from(v)?), #[cfg(feature = "matrix")] - Value::Matrix(_) => Err(err()), + Value::Matrix(_) => Array::try_from(Table::try_from(v)?), #[cfg(feature = "ndarray")] Value::NdArray(inner) => { let nd = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); nd.to_array() } #[cfg(all(feature = "ndarray", feature = "views"))] - Value::NdArrayView(_) => Err(err()), - #[cfg(all(feature = "ndarray", feature = "chunked"))] - Value::SuperNdArray(_) => Err(err()), - #[cfg(all(feature = "ndarray", feature = "chunked", feature = "views"))] - Value::SuperNdArrayView(_) => Err(err()), + Value::NdArrayView(inner) => Array::try_from(NdArray::::from( + Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()), + )), #[cfg(feature = "xarray")] - Value::XArray(_) => Err(err()), + Value::XArray(_) => Array::try_from(Table::try_from(v)?), #[cfg(feature = "cube")] - Value::Cube(_) => Err(err()), + Value::Cube(_) => Array::try_from(Table::try_from(v)?), Value::VecValue(inner) => { let values = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - match values.len() { - 0 => Ok(Array::default()), - 1 => Array::try_from(values.into_iter().next().unwrap()), - _ => { - let mut iter = values.into_iter(); - let mut acc = Array::try_from(iter.next().unwrap())?; - for v in iter { - let next = Array::try_from(v)?; - acc = acc.concat(next)?; - } - Ok(acc) - } - } + let pieces: Result, _> = + values.into_iter().map(Array::try_from).collect(); + Array::try_from(pieces?) } - Value::BoxValue(_) => Err(err()), - Value::ArcValue(_) => Err(err()), - Value::Tuple2(_) => Err(err()), - Value::Tuple3(_) => Err(err()), - Value::Tuple4(_) => Err(err()), - Value::Tuple5(_) => Err(err()), - Value::Tuple6(_) => Err(err()), - Value::Custom(_) => Err(err()), + #[cfg(all(feature = "ndarray", feature = "chunked"))] + Value::SuperNdArray(inner) => Array::try_from(NdArray::::try_from( + Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()), + )?), + #[cfg(all(feature = "ndarray", feature = "chunked", feature = "views"))] + Value::SuperNdArrayView(inner) => Array::try_from(NdArray::::try_from( + SuperNdArray::::from( + Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()), + ), + )?), + Value::Tuple2(_) + | Value::Tuple3(_) + | Value::Tuple4(_) + | Value::Tuple5(_) + | Value::Tuple6(_) + | Value::Custom(_) => Err(err(&v)), } } } @@ -1092,20 +1091,17 @@ impl TryFrom for Array { #[cfg(feature = "views")] impl TryFrom for ArrayV { type Error = MinarrowError; + + /// Accepts any `Value` that resolves to an `Array`, which includes a + /// `FieldArray`, a `Scalar`, chunked arrays and the recursive wrappers. + /// An incoming view passes through with its window intact rather than + /// being materialised. fn try_from(v: Value) -> Result { match v { Value::ArrayView(inner) => { Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) } - Value::Array(inner) => { - let array = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - Ok(ArrayV::from(array)) - } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "ArrayV", - message: Some("Value type mismatch".to_owned()), - }), + other => Ok(ArrayV::from(Array::try_from(other)?)), } } } @@ -1113,122 +1109,72 @@ impl TryFrom for ArrayV { #[cfg(feature = "views")] impl TryFrom for NumericArrayV { type Error = MinarrowError; + + /// Accepts any `Value` that resolves to an `Array` carrying a + /// `NumericArray` payload. The conversion routes through `ArrayV`, so an + /// incoming view keeps its window instead of materialising it, and every + /// input `Array` accepts is accepted here too. fn try_from(v: Value) -> Result { - match v { - Value::ArrayView(inner) => { - let view = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - let (array, offset, len) = view.as_tuple(); - match array { - Array::NumericArray(num_arr) => Ok(NumericArrayV::new(num_arr, offset, len)), - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "NumericArrayV", - message: Some("ArrayV is not a NumericArray".to_owned()), - }), - } - } - Value::Array(inner) => { - let arr = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - match arr { - Array::NumericArray(num_arr) => { - let len = num_arr.len(); - Ok(NumericArrayV::new(num_arr, 0, len)) - } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "NumericArrayV", - message: Some("Array is not a NumericArray".to_owned()), - }), - } - } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "NumericArrayV", - message: Some("Value type mismatch".to_owned()), - }), + let view = ArrayV::try_from(v)?; + if matches!(view.array, Array::NumericArray(_)) { + return Ok(NumericArrayV::from(view)); } + Err(MinarrowError::TypeError { + from: "Array", + to: "NumericArrayV", + message: Some(format!( + "the array holds {:?}, which is not a NumericArray", + view.array.arrow_type() + )), + }) } } #[cfg(feature = "views")] impl TryFrom for TextArrayV { type Error = MinarrowError; + + /// Accepts any `Value` that resolves to an `Array` carrying a + /// `TextArray` payload. The conversion routes through `ArrayV`, so an + /// incoming view keeps its window instead of materialising it, and every + /// input `Array` accepts is accepted here too. fn try_from(v: Value) -> Result { - match v { - Value::ArrayView(inner) => { - let view = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - let (array, offset, len) = view.as_tuple(); - match array { - Array::TextArray(text_arr) => Ok(TextArrayV::new(text_arr, offset, len)), - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "TextArrayV", - message: Some("ArrayV is not a TextArray".to_owned()), - }), - } - } - Value::Array(inner) => { - let array = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - match array { - Array::TextArray(text_arr) => { - let len = text_arr.len(); - Ok(TextArrayV::new(text_arr, 0, len)) - } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "TextArrayV", - message: Some("Array is not a TextArray".to_owned()), - }), - } - } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "TextArrayV", - message: Some("Value type mismatch".to_owned()), - }), + let view = ArrayV::try_from(v)?; + if matches!(view.array, Array::TextArray(_)) { + return Ok(TextArrayV::from(view)); } + Err(MinarrowError::TypeError { + from: "Array", + to: "TextArrayV", + message: Some(format!( + "the array holds {:?}, which is not a TextArray", + view.array.arrow_type() + )), + }) } } #[cfg(all(feature = "views", feature = "datetime"))] impl TryFrom for TemporalArrayV { type Error = MinarrowError; + + /// Accepts any `Value` that resolves to an `Array` carrying a + /// `TemporalArray` payload. The conversion routes through `ArrayV`, so an + /// incoming view keeps its window instead of materialising it, and every + /// input `Array` accepts is accepted here too. fn try_from(v: Value) -> Result { - match v { - Value::ArrayView(inner) => { - let view = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - let (array, offset, len) = view.as_tuple(); - match array { - Array::TemporalArray(temp_arr) => { - Ok(TemporalArrayV::new(temp_arr, offset, len)) - } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "TemporalArrayV", - message: Some("ArrayV is not a TemporalArray".to_owned()), - }), - } - } - Value::Array(inner) => { - let array = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - match array { - Array::TemporalArray(temp_arr) => { - let len = temp_arr.len(); - Ok(TemporalArrayV::new(temp_arr, 0, len)) - } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "TemporalArrayV", - message: Some("Array is not a TemporalArray".to_owned()), - }), - } - } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "TemporalArrayV", - message: Some("Value type mismatch".to_owned()), - }), + let view = ArrayV::try_from(v)?; + if matches!(view.array, Array::TemporalArray(_)) { + return Ok(TemporalArrayV::from(view)); } + Err(MinarrowError::TypeError { + from: "Array", + to: "TemporalArrayV", + message: Some(format!( + "the array holds {:?}, which is not a TemporalArray", + view.array.arrow_type() + )), + }) } } @@ -1241,6 +1187,14 @@ impl TryFrom for TemporalArrayV { impl<'a> TryFrom<&'a Value> for BitmaskV<'a> { type Error = MinarrowError; fn try_from(v: &'a Value) -> Result { + let err = |v: &Value| MinarrowError::TypeError { + from: value_variant_name(v), + to: "BitmaskV", + message: Some(format!( + "Value::{} does not convert to BitmaskV", + value_variant_name(v) + )), + }; match v { Value::ArrayView(inner) => { let (array, offset, len) = inner.as_tuple_ref(); @@ -1264,11 +1218,43 @@ impl<'a> TryFrom<&'a Value> for BitmaskV<'a> { message: Some("Array is not a BooleanArray".to_owned()), }), }, - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "BitmaskV", - message: Some("Value type mismatch".to_owned()), - }), + #[cfg(feature = "scalar_type")] + Value::Scalar(_) => Err(err(v)), + #[cfg(feature = "views")] + Value::TableView(_) => Err(err(v)), + #[cfg(feature = "chunked")] + Value::SuperArray(_) => Err(err(v)), + #[cfg(all(feature = "chunked", feature = "views"))] + Value::SuperArrayView(_) => Err(err(v)), + #[cfg(feature = "chunked")] + Value::SuperTable(_) => Err(err(v)), + #[cfg(all(feature = "chunked", feature = "views"))] + Value::SuperTableView(_) => Err(err(v)), + #[cfg(feature = "matrix")] + Value::Matrix(_) => Err(err(v)), + #[cfg(feature = "ndarray")] + Value::NdArray(_) => Err(err(v)), + #[cfg(all(feature = "ndarray", feature = "views"))] + Value::NdArrayView(_) => Err(err(v)), + #[cfg(all(feature = "ndarray", feature = "chunked"))] + Value::SuperNdArray(_) => Err(err(v)), + #[cfg(all(feature = "ndarray", feature = "chunked", feature = "views"))] + Value::SuperNdArrayView(_) => Err(err(v)), + #[cfg(feature = "xarray")] + Value::XArray(_) => Err(err(v)), + #[cfg(feature = "cube")] + Value::Cube(_) => Err(err(v)), + Value::FieldArray(_) + | Value::Table(_) + | Value::VecValue(_) + | Value::BoxValue(_) + | Value::ArcValue(_) + | Value::Tuple2(_) + | Value::Tuple3(_) + | Value::Tuple4(_) + | Value::Tuple5(_) + | Value::Tuple6(_) + | Value::Custom(_) => Err(err(v)), } } } @@ -1276,50 +1262,37 @@ impl<'a> TryFrom<&'a Value> for BitmaskV<'a> { #[cfg(feature = "views")] impl TryFrom for BooleanArrayV { type Error = MinarrowError; + + /// Accepts any `Value` that resolves to an `Array` carrying a + /// `BooleanArray` payload. The conversion routes through `ArrayV`, so an + /// incoming view keeps its window instead of materialising it, and every + /// input `Array` accepts is accepted here too. fn try_from(v: Value) -> Result { - match v { - Value::ArrayView(inner) => { - let view = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - let (array, offset, len) = view.as_tuple(); - match array { - Array::BooleanArray(bool_arr) => Ok(BooleanArrayV::new(bool_arr, offset, len)), - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "BooleanArrayV", - message: Some("ArrayV is not a BooleanArray".to_owned()), - }), - } - } - Value::Array(inner) => { - let array = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - match array { - Array::BooleanArray(bool_arr) => { - let len = bool_arr.len(); - Ok(BooleanArrayV::new(bool_arr, 0, len)) - } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "BooleanArrayV", - message: Some("Array is not a BooleanArray".to_owned()), - }), - } - } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "BooleanArrayV", - message: Some("Value type mismatch".to_owned()), - }), + let view = ArrayV::try_from(v)?; + if matches!(view.array, Array::BooleanArray(_)) { + return Ok(BooleanArrayV::from(view)); } + Err(MinarrowError::TypeError { + from: "Array", + to: "BooleanArrayV", + message: Some(format!( + "the array holds {:?}, which is not a BooleanArray", + view.array.arrow_type() + )), + }) } } impl TryFrom for Table { type Error = MinarrowError; fn try_from(v: Value) -> Result { - let err = || MinarrowError::TypeError { - from: "Value", + let err = |v: &Value| MinarrowError::TypeError { + from: value_variant_name(v), to: "Table", - message: Some("Value type mismatch".to_owned()), + message: Some(format!( + "Value::{} does not convert to Table", + value_variant_name(v) + )), }; match v { Value::Table(inner) => Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())), @@ -1328,88 +1301,79 @@ impl TryFrom for Table { let view = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); Ok(view.to_table()) } - Value::Array(inner) => { - let array = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - Ok(Table::new( - "array".to_string(), - Some(vec![FieldArray::from_arr("column_0", array)]), - )) - } + Value::Array(inner) => Ok(Table::from( + Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()), + )), #[cfg(feature = "views")] - Value::ArrayView(inner) => { - let view = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - Ok(Table::new( - "array".to_string(), - Some(vec![FieldArray::from_arr("column_0", view.to_array())]), - )) - } + Value::ArrayView(_) => Ok(Table::from(Array::try_from(v)?)), Value::FieldArray(inner) => { let fa = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); Ok(Table::new("array".to_string(), Some(vec![fa]))) } #[cfg(feature = "scalar_type")] - Value::Scalar(s) => { - let array = Array::from(s); - Ok(Table::new( - "scalar".to_string(), - Some(vec![FieldArray::from_arr("column_0", array)]), - )) - } + Value::Scalar(s) => Ok(Table::from(Array::from(s))), #[cfg(feature = "chunked")] - Value::SuperArray(_) => Err(err()), + Value::SuperArray(_) => Ok(Table::from(Array::try_from(v)?)), #[cfg(all(feature = "chunked", feature = "views"))] - Value::SuperArrayView(_) => Err(err()), + Value::SuperArrayView(_) => Ok(Table::from(Array::try_from(v)?)), #[cfg(feature = "chunked")] - Value::SuperTable(_) => Err(err()), + Value::SuperTable(inner) => { + Table::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } #[cfg(all(feature = "chunked", feature = "views"))] - Value::SuperTableView(_) => Err(err()), + Value::SuperTableView(inner) => Table::try_from(SuperTable::from( + Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()), + )), #[cfg(feature = "matrix")] - Value::Matrix(_) => Err(err()), + Value::Matrix(inner) => Ok(Table::from( + Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()), + )), #[cfg(feature = "ndarray")] Value::NdArray(inner) => { let nd = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); nd.to_table(None) } #[cfg(all(feature = "ndarray", feature = "views"))] - Value::NdArrayView(_) => Err(err()), - #[cfg(all(feature = "ndarray", feature = "chunked"))] - Value::SuperNdArray(_) => Err(err()), - #[cfg(all(feature = "ndarray", feature = "chunked", feature = "views"))] - Value::SuperNdArrayView(_) => Err(err()), + Value::NdArrayView(inner) => Table::try_from(NdArray::::from( + Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()), + )), #[cfg(feature = "xarray")] Value::XArray(inner) => { let xa = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); xa.to_table() } #[cfg(feature = "cube")] - Value::Cube(_) => Err(err()), - // VecValue terminal coercion: engine wire produced by `Long` fan-out gather. - // Empty -> Table::new_empty (typed-empty); single -> recurse on inner; - // many -> concat-fold across inner Tables. + Value::Cube(inner) => { + Table::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + // A sequence of tables is a chunked table, which the chunked + // layer rejoins. Value::VecValue(inner) => { let values = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - match values.len() { - 0 => Ok(Table::new_empty()), - 1 => Table::try_from(values.into_iter().next().unwrap()), - _ => { - let mut iter = values.into_iter(); - let mut acc = Table::try_from(iter.next().unwrap())?; - for v in iter { - let next = Table::try_from(v)?; - acc = acc.concat(next)?; - } - Ok(acc) - } - } + let pieces: Result, _> = + values.into_iter().map(Table::try_from).collect(); + Table::try_from(pieces?) } - Value::BoxValue(_) => Err(err()), - Value::ArcValue(_) => Err(err()), - Value::Tuple2(_) => Err(err()), - Value::Tuple3(_) => Err(err()), - Value::Tuple4(_) => Err(err()), - Value::Tuple5(_) => Err(err()), - Value::Tuple6(_) => Err(err()), - Value::Custom(_) => Err(err()), + Value::BoxValue(inner) => Table::try_from(*inner), + Value::ArcValue(inner) => { + Table::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + #[cfg(all(feature = "ndarray", feature = "chunked"))] + Value::SuperNdArray(inner) => Table::try_from(NdArray::::try_from( + Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()), + )?), + #[cfg(all(feature = "ndarray", feature = "chunked", feature = "views"))] + Value::SuperNdArrayView(inner) => Table::try_from(NdArray::::try_from( + SuperNdArray::::from( + Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()), + ), + )?), + Value::Tuple2(_) + | Value::Tuple3(_) + | Value::Tuple4(_) + | Value::Tuple5(_) + | Value::Tuple6(_) + | Value::Custom(_) => Err(err(&v)), } } } @@ -1417,35 +1381,42 @@ impl TryFrom for Table { #[cfg(feature = "views")] impl TryFrom for TableV { type Error = MinarrowError; + + /// Accepts any `Value` that resolves to a `Table`, which includes a + /// single array or field array as a one-column table, chunked tables and + /// the recursive wrappers. An incoming view keeps its window and column + /// projection. fn try_from(v: Value) -> Result { match v { Value::TableView(inner) => { Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) } - Value::Table(inner) => { - let table = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - Ok(TableV::from(table)) - } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "TableV", - message: Some("Value type mismatch".to_owned()), - }), + other => Ok(TableV::from(Table::try_from(other)?)), } } } impl TryFrom for FieldArray { type Error = MinarrowError; + + /// Takes the field array carried by `Value::FieldArray`, unwrapping the + /// recursive `BoxValue` and `ArcValue` wrappers first. fn try_from(v: Value) -> Result { match v { Value::FieldArray(inner) => { Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) } - _ => Err(MinarrowError::TypeError { - from: "Value", + Value::BoxValue(inner) => FieldArray::try_from(*inner), + Value::ArcValue(inner) => { + FieldArray::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + other => Err(MinarrowError::TypeError { + from: value_variant_name(&other), to: "FieldArray", - message: Some("Value type mismatch".to_owned()), + message: Some(format!( + "Value::{} does not convert to FieldArray", + value_variant_name(&other) + )), }), } } @@ -1454,60 +1425,31 @@ impl TryFrom for FieldArray { #[cfg(feature = "chunked")] impl TryFrom for SuperTable { type Error = MinarrowError; + + /// Accepts any `Value` that resolves to a `Table`, which becomes a + /// chunked table of one batch. A `VecValue` keeps its elements as + /// separate batches. fn try_from(v: Value) -> Result { match v { Value::SuperTable(inner) => { Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) } - // VecValue terminal coercion: wrap inner Tables as chunks. Mirrors the - // `SuperArray::try_from(VecValue)` precedent. No row-data reallocation; - // each chunk holds an Arc. `None` lets SuperTable inherit its - // name from the first batch rather than imposing a fresh one. - // - // `SuperTable::from_batches` panics on schema mismatch between - // chunks; validate up-front and surface mismatches as a typed - // MinarrowError so this conversion never aborts the caller. + #[cfg(feature = "views")] + Value::SuperTableView(inner) => Ok(SuperTable::from( + Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()), + )), Value::VecValue(inner) => { let values = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - let chunks: Result>, _> = values - .into_iter() - .map(|v| Table::try_from(v).map(Arc::new)) - .collect(); - let chunks = chunks?; - if let Some(first) = chunks.first() { - let n_cols = first.n_cols(); - for (idx, chunk) in chunks.iter().enumerate().skip(1) { - if chunk.n_cols() != n_cols { - return Err(MinarrowError::TypeError { - from: "Value::VecValue", - to: "SuperTable", - message: Some(format!( - "batch {idx} column-count mismatch: {} vs {}", - chunk.n_cols(), - n_cols - )), - }); - } - for (col_idx, fa) in chunk.cols.iter().enumerate() { - if fa.field != first.cols[col_idx].field { - return Err(MinarrowError::TypeError { - from: "Value::VecValue", - to: "SuperTable", - message: Some(format!( - "batch {idx} col {col_idx} schema mismatch" - )), - }); - } - } - } - } - Ok(SuperTable::from_batches(chunks, None)) + let pieces: Result, _> = + values.into_iter().map(Table::try_from).collect(); + Ok(SuperTable::from(pieces?)) } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "SuperTable", - message: Some("Value type mismatch".to_owned()), - }), + // A cube stacks tables that become the chunked table's batches. + #[cfg(feature = "cube")] + Value::Cube(inner) => Ok(SuperTable::from( + Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()), + )), + other => Ok(SuperTable::from(Table::try_from(other)?)), } } } @@ -1515,13 +1457,23 @@ impl TryFrom for SuperTable { #[cfg(feature = "cube")] impl TryFrom for Cube { type Error = MinarrowError; + + /// Takes the `Cube` carried by `Value::Cube`, unwrapping the recursive + /// `BoxValue` and `ArcValue` wrappers first. fn try_from(v: Value) -> Result { match v { Value::Cube(inner) => Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())), - _ => Err(MinarrowError::TypeError { - from: "Value", + Value::BoxValue(inner) => Cube::try_from(*inner), + Value::ArcValue(inner) => { + Cube::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + other => Err(MinarrowError::TypeError { + from: value_variant_name(&other), to: "Cube", - message: Some("Value type mismatch".to_owned()), + message: Some(format!( + "Value::{} does not convert to Cube", + value_variant_name(&other) + )), }), } } @@ -1530,6 +1482,10 @@ impl TryFrom for Cube { #[cfg(feature = "chunked")] impl TryFrom for SuperArray { type Error = MinarrowError; + + /// Accepts any `Value` that resolves to an `Array`, which becomes a + /// chunked array of one chunk. A `VecValue` keeps its elements as + /// separate chunks rather than concatenating them. fn try_from(v: Value) -> Result { match v { Value::SuperArray(inner) => { @@ -1537,15 +1493,11 @@ impl TryFrom for SuperArray { } Value::VecValue(inner) => { let values = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); - let chunks: Result, _> = + let pieces: Result, _> = values.into_iter().map(Array::try_from).collect(); - Ok(SuperArray::from_arrays(chunks?)) + Ok(SuperArray::from(pieces?)) } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "SuperArray", - message: Some("Value type mismatch".to_owned()), - }), + other => Ok(SuperArray::from(Array::try_from(other)?)), } } } @@ -1553,16 +1505,15 @@ impl TryFrom for SuperArray { #[cfg(all(feature = "chunked", feature = "views"))] impl TryFrom for SuperArrayV { type Error = MinarrowError; + + /// Accepts any `Value` that resolves to a chunked array, viewed over its + /// full extent. An incoming view passes through with its window intact. fn try_from(v: Value) -> Result { match v { Value::SuperArrayView(inner) => { Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "SuperArrayV", - message: Some("Value type mismatch".to_owned()), - }), + other => Ok(SuperArrayV::from(SuperArray::try_from(other)?)), } } } @@ -1570,14 +1521,16 @@ impl TryFrom for SuperArrayV { #[cfg(feature = "matrix")] impl TryFrom for Matrix { type Error = MinarrowError; + + /// Accepts any `Value` that resolves to a `Table` whose columns are all + /// float columns of equal length, which is the shape a contiguous matrix + /// holds. fn try_from(v: Value) -> Result { match v { - Value::Matrix(inner) => Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())), - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "Matrix", - message: Some("Value type mismatch".to_owned()), - }), + Value::Matrix(inner) => { + Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + other => Matrix::try_from(Table::try_from(other)?), } } } @@ -1585,16 +1538,15 @@ impl TryFrom for Matrix { #[cfg(all(feature = "chunked", feature = "views"))] impl TryFrom for SuperTableV { type Error = MinarrowError; + + /// Accepts any `Value` that resolves to a chunked table, viewed over its + /// full extent. An incoming view passes through with its window intact. fn try_from(v: Value) -> Result { match v { Value::SuperTableView(inner) => { Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "SuperTableV", - message: Some("Value type mismatch".to_owned()), - }), + other => Ok(SuperTableV::from(SuperTable::try_from(other)?)), } } } @@ -1602,16 +1554,16 @@ impl TryFrom for SuperTableV { #[cfg(feature = "ndarray")] impl TryFrom for NdArray { type Error = MinarrowError; + + /// Accepts any `Value` that resolves to a `Table` of float columns of + /// equal length, which is the shape a contiguous n-dimensional array holds. + /// The column-type requirement is enforced by the `Table` conversion. fn try_from(v: Value) -> Result { match v { Value::NdArray(inner) => { Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "NdArray", - message: Some("Value type mismatch".to_owned()), - }), + other => NdArray::::try_from(Table::try_from(other)?), } } } @@ -1619,15 +1571,23 @@ impl TryFrom for NdArray { #[cfg(all(feature = "ndarray", feature = "views"))] impl TryFrom for NdArrayV { type Error = MinarrowError; + + /// Takes the `NdArrayV` carried by `Value::NdArrayView`, unwrapping the + /// recursive `BoxValue` and `ArcValue` wrappers first. fn try_from(v: Value) -> Result { match v { - Value::NdArrayView(inner) => { - Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + Value::NdArrayView(inner) => Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())), + Value::BoxValue(inner) => NdArrayV::::try_from(*inner), + Value::ArcValue(inner) => { + NdArrayV::::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "NdArrayV", - message: Some("Value type mismatch".to_owned()), + other => Err(MinarrowError::TypeError { + from: value_variant_name(&other), + to: "NdArrayV", + message: Some(format!( + "Value::{} does not convert to NdArrayV", + value_variant_name(&other) + )), }), } } @@ -1636,15 +1596,23 @@ impl TryFrom for NdArrayV { #[cfg(all(feature = "ndarray", feature = "chunked"))] impl TryFrom for SuperNdArray { type Error = MinarrowError; + + /// Takes the `SuperNdArray` carried by `Value::SuperNdArray`, unwrapping + /// the recursive `BoxValue` and `ArcValue` wrappers first. fn try_from(v: Value) -> Result { match v { - Value::SuperNdArray(inner) => { - Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + Value::SuperNdArray(inner) => Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())), + Value::BoxValue(inner) => SuperNdArray::::try_from(*inner), + Value::ArcValue(inner) => { + SuperNdArray::::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "SuperNdArray", - message: Some("Value type mismatch".to_owned()), + other => Err(MinarrowError::TypeError { + from: value_variant_name(&other), + to: "SuperNdArray", + message: Some(format!( + "Value::{} does not convert to SuperNdArray", + value_variant_name(&other) + )), }), } } @@ -1653,15 +1621,23 @@ impl TryFrom for SuperNdArray { #[cfg(all(feature = "ndarray", feature = "chunked", feature = "views"))] impl TryFrom for SuperNdArrayV { type Error = MinarrowError; + + /// Takes the `SuperNdArrayV` carried by `Value::SuperNdArrayView`, + /// unwrapping the recursive `BoxValue` and `ArcValue` wrappers first. fn try_from(v: Value) -> Result { match v { - Value::SuperNdArrayView(inner) => { - Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + Value::SuperNdArrayView(inner) => Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())), + Value::BoxValue(inner) => SuperNdArrayV::::try_from(*inner), + Value::ArcValue(inner) => { + SuperNdArrayV::::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "SuperNdArrayV", - message: Some("Value type mismatch".to_owned()), + other => Err(MinarrowError::TypeError { + from: value_variant_name(&other), + to: "SuperNdArrayV", + message: Some(format!( + "Value::{} does not convert to SuperNdArrayV", + value_variant_name(&other) + )), }), } } @@ -1670,15 +1646,23 @@ impl TryFrom for SuperNdArrayV { #[cfg(feature = "xarray")] impl TryFrom for XArray { type Error = MinarrowError; + + /// Takes the `XArray` carried by `Value::XArray`, unwrapping the + /// recursive `BoxValue` and `ArcValue` wrappers first. fn try_from(v: Value) -> Result { match v { - Value::XArray(inner) => { - Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + Value::XArray(inner) => Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())), + Value::BoxValue(inner) => XArray::::try_from(*inner), + Value::ArcValue(inner) => { + XArray::::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "XArray", - message: Some("Value type mismatch".to_owned()), + other => Err(MinarrowError::TypeError { + from: value_variant_name(&other), + to: "XArray", + message: Some(format!( + "Value::{} does not convert to XArray", + value_variant_name(&other) + )), }), } } @@ -1725,29 +1709,82 @@ impl From<(Value, Value, Value, Value, Value, Value)> for Value { // TryFrom for recursive containers impl TryFrom for Vec { type Error = MinarrowError; + + /// Accepts a `VecValue` as its elements and a tuple as its members. Any + /// other `Value` is a single-element sequence, so it converts rather + /// than rejecting. fn try_from(v: Value) -> Result { match v { Value::VecValue(inner) => { Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) } - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "Vec", - message: Some("Expected VecValue variant".to_owned()), - }), + Value::BoxValue(inner) => Vec::::try_from(*inner), + Value::ArcValue(inner) => { + Vec::::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + Value::Tuple2(inner) => { + let (a, b) = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); + Ok(vec![a, b]) + } + Value::Tuple3(inner) => { + let (a, b, c) = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); + Ok(vec![a, b, c]) + } + Value::Tuple4(inner) => { + let (a, b, c, d) = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); + Ok(vec![a, b, c, d]) + } + Value::Tuple5(inner) => { + let (a, b, c, d, e) = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); + Ok(vec![a, b, c, d, e]) + } + Value::Tuple6(inner) => { + let (a, b, c, d, e, f) = + Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); + Ok(vec![a, b, c, d, e, f]) + } + other => Ok(vec![other]), } } } impl TryFrom for (Value, Value) { type Error = MinarrowError; + + /// Accepts the matching tuple variant, and a `VecValue` holding exactly + /// 2 elements. The recursive wrappers unwrap first, so a boxed or + /// shared tuple converts as its payload. fn try_from(v: Value) -> Result { match v { - Value::Tuple2(tuple) => Ok(Arc::try_unwrap(tuple).unwrap_or_else(|arc| (*arc).clone())), - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "(Value, Value)", - message: Some("Expected Tuple2 variant".to_owned()), + Value::Tuple2(inner) => { + Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + Value::BoxValue(inner) => Self::try_from(*inner), + Value::ArcValue(inner) => { + Self::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + Value::VecValue(inner) => { + let values = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); + let got = values.len(); + let Ok(members): Result<[Value; 2], _> = values.try_into() else { + return Err(MinarrowError::TypeError { + from: "VecValue", + to: "Tuple2", + message: Some(format!( + "a Tuple2 needs 2 elements, the sequence held {got}" + )), + }); + }; + let [a, b] = members; + Ok((a, b)) + } + other => Err(MinarrowError::TypeError { + from: value_variant_name(&other), + to: "Tuple2", + message: Some(format!( + "Value::{} does not convert to a 2-member tuple", + value_variant_name(&other) + )), }), } } @@ -1755,13 +1792,41 @@ impl TryFrom for (Value, Value) { impl TryFrom for (Value, Value, Value) { type Error = MinarrowError; + + /// Accepts the matching tuple variant, and a `VecValue` holding exactly + /// 3 elements. The recursive wrappers unwrap first, so a boxed or + /// shared tuple converts as its payload. fn try_from(v: Value) -> Result { match v { - Value::Tuple3(tuple) => Ok(Arc::try_unwrap(tuple).unwrap_or_else(|arc| (*arc).clone())), - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "(Value, Value, Value)", - message: Some("Expected Tuple3 variant".to_owned()), + Value::Tuple3(inner) => { + Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + Value::BoxValue(inner) => Self::try_from(*inner), + Value::ArcValue(inner) => { + Self::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + Value::VecValue(inner) => { + let values = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); + let got = values.len(); + let Ok(members): Result<[Value; 3], _> = values.try_into() else { + return Err(MinarrowError::TypeError { + from: "VecValue", + to: "Tuple3", + message: Some(format!( + "a Tuple3 needs 3 elements, the sequence held {got}" + )), + }); + }; + let [a, b, c] = members; + Ok((a, b, c)) + } + other => Err(MinarrowError::TypeError { + from: value_variant_name(&other), + to: "Tuple3", + message: Some(format!( + "Value::{} does not convert to a 3-member tuple", + value_variant_name(&other) + )), }), } } @@ -1769,13 +1834,41 @@ impl TryFrom for (Value, Value, Value) { impl TryFrom for (Value, Value, Value, Value) { type Error = MinarrowError; + + /// Accepts the matching tuple variant, and a `VecValue` holding exactly + /// 4 elements. The recursive wrappers unwrap first, so a boxed or + /// shared tuple converts as its payload. fn try_from(v: Value) -> Result { match v { - Value::Tuple4(tuple) => Ok(Arc::try_unwrap(tuple).unwrap_or_else(|arc| (*arc).clone())), - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "(Value, Value, Value, Value)", - message: Some("Expected Tuple4 variant".to_owned()), + Value::Tuple4(inner) => { + Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + Value::BoxValue(inner) => Self::try_from(*inner), + Value::ArcValue(inner) => { + Self::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + Value::VecValue(inner) => { + let values = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); + let got = values.len(); + let Ok(members): Result<[Value; 4], _> = values.try_into() else { + return Err(MinarrowError::TypeError { + from: "VecValue", + to: "Tuple4", + message: Some(format!( + "a Tuple4 needs 4 elements, the sequence held {got}" + )), + }); + }; + let [a, b, c, d] = members; + Ok((a, b, c, d)) + } + other => Err(MinarrowError::TypeError { + from: value_variant_name(&other), + to: "Tuple4", + message: Some(format!( + "Value::{} does not convert to a 4-member tuple", + value_variant_name(&other) + )), }), } } @@ -1783,13 +1876,41 @@ impl TryFrom for (Value, Value, Value, Value) { impl TryFrom for (Value, Value, Value, Value, Value) { type Error = MinarrowError; + + /// Accepts the matching tuple variant, and a `VecValue` holding exactly + /// 5 elements. The recursive wrappers unwrap first, so a boxed or + /// shared tuple converts as its payload. fn try_from(v: Value) -> Result { match v { - Value::Tuple5(tuple) => Ok(Arc::try_unwrap(tuple).unwrap_or_else(|arc| (*arc).clone())), - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "(Value, Value, Value, Value, Value)", - message: Some("Expected Tuple5 variant".to_owned()), + Value::Tuple5(inner) => { + Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + Value::BoxValue(inner) => Self::try_from(*inner), + Value::ArcValue(inner) => { + Self::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + Value::VecValue(inner) => { + let values = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); + let got = values.len(); + let Ok(members): Result<[Value; 5], _> = values.try_into() else { + return Err(MinarrowError::TypeError { + from: "VecValue", + to: "Tuple5", + message: Some(format!( + "a Tuple5 needs 5 elements, the sequence held {got}" + )), + }); + }; + let [a, b, c, d, e] = members; + Ok((a, b, c, d, e)) + } + other => Err(MinarrowError::TypeError { + from: value_variant_name(&other), + to: "Tuple5", + message: Some(format!( + "Value::{} does not convert to a 5-member tuple", + value_variant_name(&other) + )), }), } } @@ -1797,13 +1918,41 @@ impl TryFrom for (Value, Value, Value, Value, Value) { impl TryFrom for (Value, Value, Value, Value, Value, Value) { type Error = MinarrowError; + + /// Accepts the matching tuple variant, and a `VecValue` holding exactly + /// 6 elements. The recursive wrappers unwrap first, so a boxed or + /// shared tuple converts as its payload. fn try_from(v: Value) -> Result { match v { - Value::Tuple6(tuple) => Ok(Arc::try_unwrap(tuple).unwrap_or_else(|arc| (*arc).clone())), - _ => Err(MinarrowError::TypeError { - from: "Value", - to: "(Value, Value, Value, Value, Value, Value)", - message: Some("Expected Tuple6 variant".to_owned()), + Value::Tuple6(inner) => { + Ok(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + Value::BoxValue(inner) => Self::try_from(*inner), + Value::ArcValue(inner) => { + Self::try_from(Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone())) + } + Value::VecValue(inner) => { + let values = Arc::try_unwrap(inner).unwrap_or_else(|arc| (*arc).clone()); + let got = values.len(); + let Ok(members): Result<[Value; 6], _> = values.try_into() else { + return Err(MinarrowError::TypeError { + from: "VecValue", + to: "Tuple6", + message: Some(format!( + "a Tuple6 needs 6 elements, the sequence held {got}" + )), + }); + }; + let [a, b, c, d, e, f] = members; + Ok((a, b, c, d, e, f)) + } + other => Err(MinarrowError::TypeError { + from: value_variant_name(&other), + to: "Tuple6", + message: Some(format!( + "Value::{} does not convert to a 6-member tuple", + value_variant_name(&other) + )), }), } } @@ -2272,7 +2421,14 @@ mod accessor_tests { let back = NdArray::::try_from(Value::from(nd.clone())).unwrap(); assert_eq!(back, nd); - assert!(NdArray::::try_from(Value::Table(Arc::new(Table::new_empty()))).is_err()); + // A tuple carries no single dense array, so it does not convert. + assert!( + NdArray::::try_from(Value::Tuple2(Arc::new(( + Value::from(1.0_f64), + Value::from(2.0_f64), + )))) + .is_err() + ); #[cfg(feature = "views")] { @@ -2437,13 +2593,11 @@ mod vecvalue_terminal_coercion_tests { #[test] fn try_from_vecvalue_array_rejects_unconvertible_inner() { - // SuperArray inside VecValue cannot become an Array; first conversion errors. - #[cfg(feature = "chunked")] - { - let inner = Value::SuperArray(Arc::new(SuperArray::new())); - let v = Value::VecValue(Arc::new(vec![inner])); - let res = Array::try_from(v); - assert!(res.is_err()); - } + // A tuple carries no single array, so the element conversion errors + // and the sequence conversion fails with it. + let inner = Value::Tuple2(Arc::new((Value::from(1.0_f64), Value::from(2.0_f64)))); + let v = Value::VecValue(Arc::new(vec![inner])); + let res = Array::try_from(v); + assert!(res.is_err()); } } diff --git a/src/ffi/arrow_c_ffi.rs b/src/ffi/arrow_c_ffi.rs index d1c6a62..ec5a544 100644 --- a/src/ffi/arrow_c_ffi.rs +++ b/src/ffi/arrow_c_ffi.rs @@ -447,7 +447,7 @@ pub fn export_to_c(array: Arc, schema: Schema) -> (*mut ArrowArray, *mut /// `ArrowArray.offset` and `ArrowArray.length`. Buffer pointers always /// point at the start of the backing buffers; consumers read /// `length` elements starting at `offset` from each buffer -/// (validity bitmap, primitive data or codes, string offsets/values). +/// (null mask, primitive data or codes, string offsets/values). /// /// The full backing [`Array`] is kept alive in the FFI holder, so the /// pointers remain valid for the lifetime of the exported `ArrowArray`. @@ -1143,7 +1143,7 @@ unsafe fn import_array_zero_copy( /// Imports a null bitmap honouring an `ArrowArray.offset` bit window. /// -/// The Arrow C Data Interface stores the validity bitmap at the start of the +/// The Arrow C Data Interface stores the null mask at the start of the /// parent buffer; consumers read `length` bits starting at bit `offset`. /// When `offset == 0` this is a straight copy; otherwise the source bits are /// realigned into a fresh `Bitmask`. @@ -1465,7 +1465,7 @@ unsafe fn import_utf8( /// - Long string (length > 12): `[i32 length][4 bytes prefix][i32 buf_index][i32 offset]` /// /// In the Arrow C Data Interface, Utf8View arrays have buffers: -/// `[validity, views, variadic_buf_0, ..., variadic_buf_n, variadic_sizes]` +/// `[null mask, views, variadic_buf_0, ..., variadic_buf_n, variadic_sizes]` /// /// # Safety /// `arr` must contain a valid Utf8View ArrowArray. @@ -1620,7 +1620,7 @@ unsafe fn import_categorical( .collect(), _ => panic!("Expected String32 dictionary"), }; - // Honours bit-level offset for the codes' validity bitmap. + // Honours bit-level offset for the codes' null mask. let null_mask = if !null_ptr.is_null() { Some(unsafe { import_null_mask_offset(null_ptr, offset, len) }) } else { @@ -4478,7 +4478,7 @@ mod tests { } /// Exercises `import_null_mask_offset`: a windowed view that spans - /// across byte boundaries in the validity bitmap (offset=3, len=7), + /// across byte boundaries in the null mask (offset=3, len=7), /// with nulls at non-aligned positions. #[cfg(feature = "views")] #[test] diff --git a/src/ffi/dlpack.rs b/src/ffi/dlpack.rs index 1573520..1a636a4 100644 --- a/src/ffi/dlpack.rs +++ b/src/ffi/dlpack.rs @@ -33,7 +33,7 @@ //! ## Layout compatibility //! NdArray's compact column-major layout is expressed through DLPack's //! strides field, and the buffer is fully contiguous, so consumers receive -//! a dense tensor with no dead bytes. Consumers such as PyTorch, NumPy, JAX, +//! a packed tensor with no dead bytes. Consumers such as PyTorch, NumPy, JAX, //! and TensorFlow decide whether to operate on those strides directly or //! re-lay the tensor for a particular operation. //! diff --git a/src/ffi/polars.rs b/src/ffi/polars.rs index 78adb73..1af486d 100644 --- a/src/ffi/polars.rs +++ b/src/ffi/polars.rs @@ -58,7 +58,7 @@ //! ## Null masks //! //! Polars preserves the null mask in every direction we test, including -//! across the type promotions above. Validity bits sit on the underlying +//! across the type promotions above. Null mask bits sit on the underlying //! Arrow array and travel through `Series::to_arrow` / `Series::from_arrow` //! along with the data buffer. diff --git a/src/kernels/arithmetic/string.rs b/src/kernels/arithmetic/string.rs index c167e30..c633f59 100644 --- a/src/kernels/arithmetic/string.rs +++ b/src/kernels/arithmetic/string.rs @@ -141,7 +141,7 @@ where } } - // Push offset regardless of validity to keep offsets aligned + // Push offset regardless of the null mask to keep offsets aligned // This ensures we can still slice [a..b] intuitively. let new_offset = O::from(data.len()).expect("offset conversion overflow"); offsets.push(new_offset); @@ -687,7 +687,7 @@ where let lmask_ref = lmask_slice.as_ref(); let rmask_ref = rmask_slice.as_ref(); - // build per-position validity + // build per-position null mask let lmask = lmask_ref; let rmask = rmask_ref; let mut out_mask = Bitmask::new_set_all(llen, false); diff --git a/src/kernels/bitmask/dispatch.rs b/src/kernels/bitmask/dispatch.rs index e9085e6..89a72bb 100644 --- a/src/kernels/bitmask/dispatch.rs +++ b/src/kernels/bitmask/dispatch.rs @@ -85,8 +85,8 @@ pub fn bitmask_unop(src: BitmaskVT<'_>, op: UnaryOperator) -> Bitmask { /// /// # Usage /// ```rust,ignore -/// // Combine validity masks from two nullable arrays -/// let combined_validity = and_masks( +/// // Combine null masks from two nullable arrays +/// let combined_null_mask = and_masks( /// (&array_a.null_mask, 0, array_a.len()), /// (&array_b.null_mask, 0, array_b.len()) /// ); @@ -307,7 +307,7 @@ pub fn all_ne(a: BitmaskVT<'_>, b: BitmaskVT<'_>) -> bool { /// # Usage /// ```rust,ignore /// // Determine if computation is worthwhile -/// let valid_count = popcount_mask((&validity_mask, 0, array.len())); +/// let valid_count = popcount_mask((&null_mask, 0, array.len())); /// if valid_count == 0 { /// // Skip expensive operations on all-null data /// return Array::new_null(array.len()); diff --git a/src/kernels/bitwise.rs b/src/kernels/bitwise.rs index 5e2e9d9..f0c2957 100644 --- a/src/kernels/bitwise.rs +++ b/src/kernels/bitwise.rs @@ -16,7 +16,7 @@ //! //! Element-wise bitwise AND, OR, XOR over two integer arrays, plus unary bitwise //! complement, with null-aware semantics. Bitwise operations never fail, so a -//! result carries the same validity as its input. +//! result carries the same null mask as its input. //! //! ## Overview //! - **SIMD path**: `std::simd` vectorisation with a scalar tail, selected for @@ -82,8 +82,8 @@ pub fn int_bitwise_body_std( } /// Scalar integer bitwise kernel with null mask support. -/// Bitwise operations never nullify, so the output validity equals the input -/// validity. Invalid lanes carry a zero value. +/// Bitwise operations never nullify, so the output null mask equals the input +/// null mask. Invalid lanes carry a zero value. #[inline(always)] pub fn int_bitwise_masked_body_std( op: BitwiseOperator, @@ -188,7 +188,7 @@ pub fn int_bitwise_body_simd( } /// SIMD integer bitwise kernel with null mask support, with a scalar tail. -/// Validity is preserved exactly: the output mask equals the input mask. +/// The null mask is preserved exactly: the output mask equals the input mask. #[cfg(feature = "simd")] #[inline(always)] pub fn int_bitwise_masked_body_simd( @@ -511,7 +511,7 @@ mod tests { } #[test] - fn bitwise_masked_preserves_validity() { + fn bitwise_masked_preserves_the_null_mask() { let lhs: Vec = (0..20).collect(); let rhs: Vec = (0..20).map(|x| x + 5).collect(); let mut mask = Bitmask::new_set_all(20, true); diff --git a/src/kernels/routing/broadcast.rs b/src/kernels/routing/broadcast.rs index 3091481..6e74e7e 100644 --- a/src/kernels/routing/broadcast.rs +++ b/src/kernels/routing/broadcast.rs @@ -48,7 +48,7 @@ pub fn broadcast_length_1_array(av: ArrayV, len: usize) -> Result Err(KernelError::UnsupportedType( - "broadcasting null boolean values not supported in dense mode".into(), + "broadcasting null boolean values not supported when no mask is supplied".into(), )), }, Array::TextArray(TextArray::String32(a)) => { diff --git a/src/kernels/string.rs b/src/kernels/string.rs index dedbf5e..647d4bd 100644 --- a/src/kernels/string.rs +++ b/src/kernels/string.rs @@ -410,7 +410,7 @@ macro_rules! str_predicate { for i in 0..len { unsafe { - // Check null‐mask validity without bounds checks + // Check the null mask without bounds checks let lv = lmask.map_or(true, |m| m.get_unchecked(loff + i)); let rv = rmask.map_or(true, |m| m.get_unchecked(roff + i)); if !lv || !rv { diff --git a/src/macros.rs b/src/macros.rs index d72161b..e4f8f5b 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -82,7 +82,7 @@ macro_rules! impl_numeric_array_constructors { } } - /// Constructs a dense typed Array from a slice. + /// Constructs a contiguous typed Array from a slice. /// /// This is a streamlined constructor - if the Array has nulls the mask /// must be applied after construction. diff --git a/src/structs/bitmask.rs b/src/structs/bitmask.rs index 007d191..4aa9f8d 100644 --- a/src/structs/bitmask.rs +++ b/src/structs/bitmask.rs @@ -14,10 +14,10 @@ //! # **Bitmask Module** - *Fast Bitpacked Byte Bitmask* //! -//! Arrow-compatible, packed validity/boolean bitmask with 64-byte alignment. +//! Arrow-compatible, packed null mask and boolean bitmask with 64-byte alignment. //! //! ## Purpose -//! - Validity (null) masks for all array types (1 = valid, 0 = null). +//! - Null masks for all array types (1 = valid, 0 = null). //! - Backing storage for `BooleanArray`. //! //! ## Behaviour @@ -48,7 +48,7 @@ use vec64::Vec64; /// 64-byte–aligned packed bitmask. /// /// ### Description -/// - Used for `BooleanArray` data and as the validity/null mask for all datatypes. +/// - Used for `BooleanArray` data and as the null mask for all datatypes. /// - Arrow-compatible: LSB = first element, 1 = set/valid, 0 = cleared/null. /// - Automatically enforced alignment enables efficient bitwise filtering on SIMD targets. /// @@ -191,7 +191,7 @@ impl Bitmask { mask } - /// Wrap a validity view from an [`crate::LBuffer`], sharing its bytes and + /// Wrap a null mask view from an [`crate::LBuffer`], sharing its bytes and /// trailing partial byte. Length and bits are read through the view; the /// stored `len` is the owned bit count, zero here. #[cfg(feature = "lbuffer")] diff --git a/src/structs/chunked/super_array.rs b/src/structs/chunked/super_array.rs index 15af5e5..f5a3d69 100644 --- a/src/structs/chunked/super_array.rs +++ b/src/structs/chunked/super_array.rs @@ -1424,6 +1424,23 @@ impl Display for SuperArray { } } +impl TryFrom for Array { + type Error = MinarrowError; + + /// Rejoins the chunks into one array. + fn try_from(mut value: SuperArray) -> Result { + let len = value.len(); + if len == 0 { + return Ok(Array::default()); + } + value.rechunk(RechunkStrategy::Count(len))?; + match value.into_chunks().into_iter().next() { + Some(array) => Ok(array), + None => Ok(Array::default()), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/structs/chunked/super_ndarray.rs b/src/structs/chunked/super_ndarray.rs index d2afd34..d5d0caa 100644 --- a/src/structs/chunked/super_ndarray.rs +++ b/src/structs/chunked/super_ndarray.rs @@ -898,6 +898,23 @@ impl fmt::Debug for SuperNdArray { // Tests // **************************************************************** +impl TryFrom> for NdArray { + type Error = MinarrowError; + + /// Rejoins the batches into one contiguous array. + fn try_from(mut value: SuperNdArray) -> Result { + let len = value.len(); + if len == 0 { + return Ok(NdArray::new(&[0])); + } + value.rechunk(RechunkStrategy::Count(len))?; + match value.batches.into_iter().next() { + Some(array) => Ok(array), + None => Ok(NdArray::new(&[0])), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/structs/chunked/super_table.rs b/src/structs/chunked/super_table.rs index 69b8058..474dc53 100644 --- a/src/structs/chunked/super_table.rs +++ b/src/structs/chunked/super_table.rs @@ -1245,6 +1245,37 @@ macro_rules! st { }; } +impl From
for SuperTable { + /// Holds a single table as one batch. + fn from(value: Table) -> Self { + SuperTable::from_batches(vec![Arc::new(value)], None) + } +} + +impl From> for SuperTable { + /// Holds a sequence of tables as its batches, in the order given. + fn from(value: Vec
) -> Self { + SuperTable::from_batches(value.into_iter().map(Arc::new).collect(), None) + } +} + +impl TryFrom for Table { + type Error = MinarrowError; + + /// Rejoins the batches into one table. + fn try_from(mut value: SuperTable) -> Result { + let rows = value.n_rows(); + if rows == 0 { + return Ok(Table::default()); + } + value.rechunk_to(rows, RechunkStrategy::Count(rows))?; + match value.batches().first() { + Some(batch) => Ok((**batch).clone()), + None => Ok(Table::default()), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/structs/cube.rs b/src/structs/cube.rs index 8e79f3d..1709390 100644 --- a/src/structs/cube.rs +++ b/src/structs/cube.rs @@ -750,6 +750,33 @@ impl crate::traits::selection::ColumnSelection for Cube { } } +impl TryFrom for Table { + type Error = MinarrowError; + + /// Joins the cube's tables end to end into one table. + /// + /// A cube stacks tables that share a schema, so the flattened form is + /// those tables concatenated by rows. An empty cube yields the + /// typed-empty table. + fn try_from(value: Cube) -> Result { + let tables: Vec
= value + .tables + .into_iter() + .map(|t| Arc::try_unwrap(t).unwrap_or_else(|arc| (*arc).clone())) + .collect(); + Table::try_from(tables) + } +} + +#[cfg(feature = "chunked")] +impl From for crate::SuperTable { + /// Presents the cube's tables as the batches of a chunked table, + /// keeping the cube's name. + fn from(value: Cube) -> Self { + crate::SuperTable::from_batches(value.tables, Some(value.name)) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/structs/matrix.rs b/src/structs/matrix.rs index f25435a..e02a3ec 100644 --- a/src/structs/matrix.rs +++ b/src/structs/matrix.rs @@ -14,7 +14,7 @@ //! # **Matrix Module** - *De-facto Matrix Memory Layout for BLAS/LAPACK ecosystem compatibility* //! -//! Dense column-major matrix type for high-performance linear algebra. +//! Contiguous column-major matrix type for high-performance linear algebra. //! BLAS/LAPACK compatible with built-inconversions from `Table` data. use std::fmt; @@ -30,7 +30,7 @@ use crate::{SharedBuffer, TableV}; /// # Matrix /// -/// Column-major dense matrix. +/// Column-major contiguous matrix. /// /// ### Description /// This struct is compatible with Arrow, LAPACK, BLAS, and all @@ -49,7 +49,7 @@ use crate::{SharedBuffer, TableV}; /// - `name`: Optional matrix name for diagnostics. /// /// ### Null handling -/// - It is dense - nulls can be represented through `f64::NAN` +/// - It is contiguous - nulls can be represented through `f64::NAN` /// - However this is not always reliable, as a single *NaN* can affect vectorised /// calculations when integrating with various frameworks. /// @@ -79,7 +79,7 @@ pub struct Matrix { /// Backing storage. `Buffer` carries either an owned `Vec64` or a /// shared view into an existing allocation. The shared variant enables /// zero-copy construction from upstream `TableV` arenas via - /// `TableV::try_as_matrix_zc` without sacrificing the dense column-major + /// `TableV::try_as_matrix_zc` without sacrificing the contiguous column-major /// stride layout that BLAS/LAPACK expects. pub data: Buffer, pub name: Option, @@ -95,7 +95,7 @@ pub(crate) const fn aligned_stride(n_rows: usize) -> usize { } impl Matrix { - /// Constructs a new dense Matrix with shape and optional name. + /// Constructs a new contiguous Matrix with shape and optional name. /// Data buffer is zeroed. Columns are padded to 64-byte alignment. pub fn new(n_rows: usize, n_cols: usize, name: Option>) -> Self { let stride = aligned_stride(n_rows); @@ -181,7 +181,7 @@ impl Matrix { /// Constructs a Matrix from a slice of `FloatArray` columns. Each /// column becomes one Matrix column; null masks are rejected so Matrix - /// stays dense. + /// stays contiguous. /// /// `impl AsRef<[FloatArray]>` accepts any contiguous layout at the /// call-site: `&[FloatArray]`, `&Vec>`, @@ -218,7 +218,7 @@ impl Matrix { if col.null_mask.as_ref().map_or(false, |m| m.has_nulls()) { return Err(MinarrowError::NullError { message: Some(format!( - "Matrix::try_from_cols: column {i} contains null values; Matrix requires dense data" + "Matrix::try_from_cols: column {i} contains null values; Matrix requires contiguous data" )), }); } @@ -229,7 +229,7 @@ impl Matrix { let pad = stride - n_rows; let mut vec = Vec64::with_capacity(stride * n_cols); - // Each column is a guaranteed-dense slice, so + // Each column is a guaranteed-contiguous slice, so // extend_from_slice is a straight memcpy into the column-major buffer. for col in columns { let col_slice: &[f64] = col.data.as_slice(); @@ -526,7 +526,7 @@ impl TableV { /// Attempt a zero-copy construction of a `Matrix` from this view. /// /// Succeeds when the columns are already laid out in memory the way a - /// Matrix expects: every active column is a dense, null-free + /// Matrix expects: every active column is a contiguous, null-free /// `FloatArray` whose `Buffer` is a `Shared` view into one common /// `SharedBuffer`, with column `i` starting at `base + i * stride` elements /// (where `stride = aligned_stride(n_rows)`). When that holds, the Matrix @@ -563,7 +563,7 @@ impl TableV { let mut base_offset: usize = 0; let mut owner_elem_len: usize = 0; - // Every column must be a dense null-free f64 view into + // Every column must be a contiguous null-free f64 view into // the same SharedBuffer, with column_i starting at base + i * stride. // ArrayV::new enforces offset+len <= array.len at construction so the // window bound is already invariant; ArrayV::null_count() covers the @@ -1279,6 +1279,14 @@ mod try_as_matrix_zc_tests { } } +impl From for Table { + /// Presents the matrix columns as table columns, naming them by + /// position where the matrix carries no field names of its own. + fn from(value: Matrix) -> Self { + value.to_table_gen() + } +} + #[cfg(test)] mod mat_macro_tests { #[test] diff --git a/src/structs/ndarray.rs b/src/structs/ndarray.rs index d8cd603..89806f9 100644 --- a/src/structs/ndarray.rs +++ b/src/structs/ndarray.rs @@ -90,7 +90,7 @@ use crate::ffi::dlpack::{ // NdArray // **************************************************************** -/// N-dimensional dense array of float values. +/// N-dimensional contiguous array of float values. /// /// Backed by [`Buffer`] for zero-copy interop with external memory. /// Compact column-major layout with a 64-byte aligned allocation start. @@ -1131,7 +1131,7 @@ macro_rules! nd { /// Compute compact column-major strides with no inter-dimension padding. /// /// For a shape `[a, b, c]`, the strides are `[1, a, a * b]`. The buffer is -/// fully contiguous, so DLPack consumers receive a dense tensor and range +/// fully contiguous, so DLPack consumers receive a contiguous tensor and range /// indexing over the outermost axis reads logical data with no gaps. The /// backing allocation start remains 64-byte aligned through `Vec64`. pub(crate) fn col_major_strides(shape: &[usize]) -> Vec { @@ -1986,6 +1986,36 @@ impl fmt::Debug for NdArray { } } +impl TryFrom> for Table { + type Error = MinarrowError; + + /// Presents the leading axis as rows and the trailing axis as columns, + /// naming the columns by position. Shapes that do not lay out as a + /// table report the reason. + fn try_from(value: NdArray) -> Result { + value.to_table(None) + } +} + +impl TryFrom> for Array { + type Error = MinarrowError; + + /// Reads a one-dimensional array as a single column. + fn try_from(value: NdArray) -> Result { + value.to_array() + } +} + +#[cfg(feature = "matrix")] +impl TryFrom> for Matrix { + type Error = MinarrowError; + + /// Reads a two-dimensional array as a contiguous matrix. + fn try_from(value: NdArray) -> Result { + value.to_matrix() + } +} + #[cfg(test)] mod tests { use super::*; @@ -2834,7 +2864,7 @@ mod tests { fn try_from_table_coerces_unparseable_text_to_nan() { // TryFrom<&Table> uses the library's lenient numeric cast. Text that // does not parse as a number coerces to nulls, which surface as NaN - // in the dense NdArray rather than failing the conversion. + // in the contiguous NdArray rather than failing the conversion. let c0 = FieldArray::from_arr("name", Array::from_string32( StringArray::from_slice(&["a", "b"]) )); diff --git a/src/structs/table.rs b/src/structs/table.rs index 6f94185..e2d0c6b 100644 --- a/src/structs/table.rs +++ b/src/structs/table.rs @@ -46,6 +46,7 @@ use rayon::iter::{IntoParallelRefIterator, IntoParallelRefMutIterator}; use super::field_array::FieldArray; #[cfg(all(feature = "views", feature = "select"))] use crate::ArrayV; +use crate::Array; use crate::Field; #[cfg(feature = "chunked")] use crate::SuperTable; @@ -64,7 +65,7 @@ use crate::Bitmask; #[cfg(all(feature = "views", feature = "chunked", feature = "size"))] use crate::{ByteSize, SuperTableV}; #[cfg(feature = "views")] -use crate::{Array, BitmaskV, NumericArrayV, TableV, TextArrayV}; +use crate::{BitmaskV, NumericArrayV, TableV, TextArrayV}; // Global counter for unnamed table instances static UNNAMED_COUNTER: AtomicUsize = AtomicUsize::new(1); @@ -1177,7 +1178,7 @@ impl Display for Table { // row-index column (“idx”) let idx_width = usize::max( 3, // “idx” - ((self.n_rows - 1) as f64).log10().floor() as usize + 1, + ((self.n_rows.saturating_sub(1)) as f64).log10().floor() as usize + 1, ); // Render header @@ -1335,6 +1336,55 @@ macro_rules! tbl { }; } +impl From for Table { + /// Presents a single array as a one-column table, naming the column by + /// position since a standalone array carries no field of its own. + fn from(value: Array) -> Self { + Table::new("array".to_string(), Some(vec![FieldArray::from_arr("column_0", value)])) + } +} + +impl TryFrom> for Table { + type Error = MinarrowError; + + /// Joins a sequence of tables end to end by rows. + /// + /// The pieces must share a schema, which the join itself enforces. An + /// empty sequence yields the typed-empty table. + fn try_from(value: Vec
) -> Result { + let mut pieces = value.into_iter(); + let Some(mut joined) = pieces.next() else { + return Ok(Table::new_empty()); + }; + for next in pieces { + joined = joined.concat(next)?; + } + Ok(joined) + } +} + +impl TryFrom
for Array { + type Error = MinarrowError; + + /// Takes the array of a one-column table. + /// + /// A single-column table is that column, which is the shape a + /// one-column projection arrives in. A wider table has no single + /// reading, so it reports the column count instead of taking the first. + fn try_from(value: Table) -> Result { + match value.n_cols() { + 1 => Ok(value.cols()[0].array.clone()), + n => Err(MinarrowError::TypeError { + from: "Table", + to: "Array", + message: Some(format!( + "a single column is needed, the table held {n} columns" + )), + }), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/structs/variants/boolean.rs b/src/structs/variants/boolean.rs index e8b7f2a..399ebb1 100644 --- a/src/structs/variants/boolean.rs +++ b/src/structs/variants/boolean.rs @@ -19,7 +19,7 @@ //! //! ## Overview //! - Logical type: boolean values (`true` / `false`) with optional `null` state. -//! - Physical storage: bit-packed `Bitmask` for values, optional `Bitmask` for validity. +//! - Physical storage: bit-packed `Bitmask` for values, optional `Bitmask` for the null mask. //! - Backed by `Vec64`-aligned buffers for CPU-friendly operations. //! - Integrates with Arrow FFI and MaskedArray trait for schema-agnostic use. //! @@ -27,11 +27,11 @@ //! - **Construction** from raw bitmasks, byte buffers, or boolean slices. //! - **Mutation**: bulk bit writes, per-element set, push, null insertion. //! - **Iteration**: sequential and parallel (with `parallel_proc` feature). -//! - **Null handling**: optional validity mask with auto-growth and validation. +//! - **Null handling**: optional null mask with auto-growth and validation. //! - **Slicing**: zero-copy tuple views or cloned logical slices. //! //! ## Intended Use Cases -//! - High-performance analytical pipelines requiring dense boolean representation. +//! - High-performance analytical pipelines requiring contiguous boolean representation. //! - Interop with Apache Arrow or other columnar formats. //! - Streaming or batch ingestion with incremental append. //! @@ -77,14 +77,14 @@ use crate::{Length, Offset, Vec64, impl_arc_masked_array}; /// ## Description /// - Stores boolean values in a compact `Bitmask` for memory efficiency. /// The first value is stored in the least significant bit (LSB). -/// - Optional `null_mask` stores validity bits (`1 = valid`, `0 = null`). +/// - Optional `null_mask` stores the null mask bits (`1 = valid`, `0 = null`). /// - The `len` field tracks the number of logical elements, not the byte length of the backing buffer. /// - Provides both safe (`get`) and unsafe (`get_unchecked`) element access. /// - Implements [`MaskedArray`] for consistent inner array behaviour. /// /// ### Fields /// - `data`: bit-packed boolean values. -/// - `null_mask`: optional bit-packed validity bitmap. +/// - `null_mask`: optional bit-packed bitmap. /// - `len`: number of logical elements. /// - `_phantom`: marker for generic type `T` (unused at runtime). /// @@ -156,7 +156,7 @@ impl BooleanArray<()> { Self { data: Bitmask::with_capacity(cap), null_mask: if null_mask { - // All-valid (1) default - reserved validity slots default to + // All-valid (1) default - reserved null mask slots default to // valid under Arrow's 1=valid, 0=null convention. Some(Bitmask::new_set_all(cap, true)) } else { @@ -167,7 +167,7 @@ impl BooleanArray<()> { } } - /// Constructs a dense BoolArray from a slice of `bool` values (no nulls). + /// Constructs a contiguous BoolArray from a slice of `bool` values (no nulls). #[inline] pub fn from_slice(slice: &[bool]) -> Self { let n = slice.len(); @@ -1470,7 +1470,7 @@ mod tests_parallel { #[test] fn par_iter_range_opt_with_nulls() { - // validity bits: 1,1,0,1,1 (index 2 is null) + // null mask bits: 1,1,0,1,1 (index 2 is null) let mask = Bitmask::from_bools(&[true, true, false, true, true]); let mut arr = BooleanArray::from_slice(&[true, false, true, false, true]); arr.null_mask = Some(mask); @@ -1490,7 +1490,7 @@ mod tests_parallel { #[test] fn par_iter_opt_unchecked_with_nulls() { - // validity bits: 1,0,0,1,1,0 (only 0,3,4 valid) + // null mask bits: 1,0,0,1,1,0 (only 0,3,4 valid) let mask = Bitmask::from_bools(&[true, false, false, true, true, false]); let mut arr = BooleanArray::from_slice(&[true, false, true, false, true, false]); arr.null_mask = Some(mask); diff --git a/src/structs/variants/categorical.rs b/src/structs/variants/categorical.rs index e3ee454..148d988 100644 --- a/src/structs/variants/categorical.rs +++ b/src/structs/variants/categorical.rs @@ -28,7 +28,7 @@ //! - Optional `null_mask`: bit-packed, where `1 = valid`, `0 = null` //! - Builders from raw values (`from_values`, `from_vec64`) and from raw parts. //! - Iterators over indices and over resolved strings (nullable and non-nullable). -//! - Convert to a dense `StringArray` via `to_string_array()` when needed. +//! - Convert to a contiguous `StringArray` via `to_string_array()` when needed. //! - Parallel helpers behind `parallel_proc` feature. //! //! ## When to use @@ -95,7 +95,7 @@ fn add_category(unique_values: &mut Vec64, value: &str) -> T /// ### Fields: /// - `data`: indices buffer referencing entries in `unique_values`. /// - `unique_values`: dictionary of unique string values. -/// - `null_mask`: optional bit-packed validity bitmap (1=valid, 0=null). +/// - `null_mask`: optional bit-packed bitmap (1=valid, 0=null). /// /// ## Purpose /// Consider this when you have a common set of unique string values, and want to @@ -219,7 +219,7 @@ impl CategoricalArray { #[cfg(feature = "shared_dict")] dictionary: unique_values.map(Dictionary::from).unwrap_or_default(), null_mask: if null_mask { - // All-valid (1) default - reserved validity slots default to + // All-valid (1) default - reserved null mask slots default to // valid under Arrow's 1=valid, 0=null convention. Some(Bitmask::new_set_all(cap, true)) } else { @@ -310,7 +310,7 @@ impl CategoricalArray { } } - /// Constructs a dense DictionaryArray from index and value slices (no nulls). + /// Constructs a contiguous DictionaryArray from index and value slices (no nulls). #[inline] pub fn from_slices(indices: &[T], unique_values: &[String]) -> Self { assert!( @@ -604,7 +604,7 @@ impl CategoricalArray { } } - /// Materialise the categorical as a dense StringArray. + /// Materialise the categorical as a contiguous StringArray. #[inline] pub fn to_string_array(&self) -> StringArray { let len = self.data.len(); diff --git a/src/structs/variants/datetime/mod.rs b/src/structs/variants/datetime/mod.rs index 5769c94..4659fa8 100644 --- a/src/structs/variants/datetime/mod.rs +++ b/src/structs/variants/datetime/mod.rs @@ -29,7 +29,7 @@ //! - Logical type: temporal values with a defined [`TimeUnit`] (seconds, milliseconds, //! microseconds, nanoseconds, days). //! - Physical storage: i64 integers representing raw time offsets from -//! the UNIX epoch or a base date, plus an optional bit-packed validity mask. +//! the UNIX epoch or a base date, plus an optional bit-packed null mask. //! - Single generic type supports all Arrow datetime/timestamp variants via `Field` //! metadata, avoiding multiple specialised array structs. //! - Integrates with Arrow FFI for zero-copy interop. @@ -41,7 +41,7 @@ //! ## Features //! - **Construction** from slices, `Vec64` or plain `Vec` buffers, with optional null mask. //! - **Mutation**: push, set, and bulk null insertion. -//! - **Null handling**: optional validity mask with length validation. +//! - **Null handling**: optional null mask with length validation. //! - **Conversion**: when `datetime_ops` feature is enabled, convert to native date/time //! values via the `time` crate, plus component extraction and datetime operations. //! - **Datetime operations**: full suite of standard datetime operations under ./datetime_ops @@ -102,13 +102,13 @@ pub const UNIX_EPOCH_JULIAN_DAY: i64 = 2_440_588; /// with units defined by [`TimeUnit`]. /// - A single struct supports all Arrow datetime/timestamp variants, with the exact logical /// type determined by an associated `Field`'s `ArrowType`. -/// - `null_mask` is an optional bit-packed validity bitmap (`1 = valid`, `0 = null`). +/// - `null_mask` is an optional bit-packed null mask (`1 = valid`, `0 = null`). /// - Implements [`MaskedArray`] for consistent nullable array behaviour. /// - When `datetime_ops` is enabled, provides conversion to native date/time via the `time` crate. /// /// ### Fields /// - `data`: backing buffer storing raw temporal values. -/// - `null_mask`: optional bit-packed validity bitmap. +/// - `null_mask`: optional bit-packed bitmap. /// - `time_unit`: time units associated with the stored values. /// /// ## Example @@ -170,7 +170,7 @@ impl DatetimeArray { Self { data: Vec64::with_capacity(cap).into(), null_mask: if null_mask { - // All-valid (1) default - reserved validity slots default to + // All-valid (1) default - reserved null mask slots default to // valid under Arrow's 1=valid, 0=null convention. Some(Bitmask::new_set_all(cap, true)) } else { @@ -190,7 +190,7 @@ impl DatetimeArray { } } - /// Constructs an DateTimeArray from a slice (dense, no nulls). + /// Constructs an DateTimeArray from a slice (contiguous, no nulls). #[inline] pub fn from_slice(slice: &[T], time_unit: Option) -> Self { Self { diff --git a/src/structs/variants/float.rs b/src/structs/variants/float.rs index f1efdf2..1aaea09 100644 --- a/src/structs/variants/float.rs +++ b/src/structs/variants/float.rs @@ -19,7 +19,7 @@ //! ## Overview //! - Logical type: fixed-width floats (`T: Float`). //! - Physical storage: `Buffer` (backed by `Vec64` for 64-byte alignment) plus optional -//! Arrow-style validity mask (`Bitmask`). +//! Arrow-style null mask (`Bitmask`). //! - Usable standalone or as the floating-point arm of higher-level enums (`NumericArray`, `Array`). //! - Supports both explicit null masks and NaN-as-null semantics (at the caller’s discretion). //! - Zero-copy compatible with standard `Vec` and slice types. @@ -79,7 +79,7 @@ use vec64::Vec64; /// /// ## Description /// - Stores floating-point values in a contiguous `Buffer` (`Vec64` under the hood). -/// - Optional Arrow-style validity bitmap (`1 = valid`, `0 = null`) via `Bitmask`. +/// - Optional Arrow-style null mask (`1 = valid`, `0 = null`) via `Bitmask`. /// - Can omit null mask entirely and represent missing values with `NaN` where acceptable. /// - Implements [`MaskedArray`] for consistent nullable array behaviour and interop with /// higher-level containers/enums. @@ -87,13 +87,13 @@ use vec64::Vec64; /// /// ### Fields /// - `data`: backing buffer of float values (`Buffer`). -/// - `null_mask`: optional bit-packed validity bitmap. +/// - `null_mask`: optional bit-packed bitmap. /// /// ## Example /// ```rust /// use minarrow::{FloatArray, MaskedArray}; /// -/// // Dense, no nulls +/// // Contiguous, no nulls /// let arr = FloatArray::::from_slice(&[1.1, 2.2, 3.3]); /// assert_eq!(arr.len(), 3); /// assert_eq!(arr.get(1), Some(2.2)); diff --git a/src/structs/variants/integer.rs b/src/structs/variants/integer.rs index b2a7ae1..cb4c677 100644 --- a/src/structs/variants/integer.rs +++ b/src/structs/variants/integer.rs @@ -19,7 +19,7 @@ //! ## Overview //! - Logical type: fixed-width signed/unsigned integers (`T: Integer`). //! - Physical storage: `Buffer` (backed by `Vec64` for 64-byte alignment) plus -//! optional bit-packed validity mask (`Bitmask`). +//! optional bit-packed null mask (`Bitmask`). //! - Usable standalone or as the numeric arm of higher-level enums (`NumericArray`, `Array`). //! - Zero-copy friendly and integrates with Arrow FFI. //! @@ -76,20 +76,20 @@ use vec64::Vec64; /// /// ## Description /// - Stores fixed-width integer values in a contiguous `Buffer` (`Vec64` under the hood). -/// - Optional Arrow-style validity bitmap (`1 = valid`, `0 = null`) via `Bitmask`. +/// - Optional Arrow-style null mask (`1 = valid`, `0 = null`) via `Bitmask`. /// - Implements [`MaskedArray`] for consistent nullable array behavior and interop with /// higher-level containers/enums. /// - Can be used as a standalone numeric buffer or as the numeric arm of `NumericArray` / `Array`. /// /// ### Fields /// - `data`: backing buffer of integer values (`Buffer`). -/// - `null_mask`: optional bit-packed validity bitmap. +/// - `null_mask`: optional bit-packed bitmap. /// /// ## Example /// ```rust /// use minarrow::{IntegerArray, MaskedArray}; /// -/// // Dense, no nulls +/// // Contiguous, no nulls /// let arr = IntegerArray::::from_slice(&[1, 2, 3, 4]); /// assert_eq!(arr.len(), 4); /// assert_eq!(arr.get(2), Some(3)); diff --git a/src/structs/variants/string.rs b/src/structs/variants/string.rs index e50ebfe..3ce48d0 100644 --- a/src/structs/variants/string.rs +++ b/src/structs/variants/string.rs @@ -78,7 +78,7 @@ use vec64::Vec64; /// ## Fields /// - **Offsets**: indices into the `data` buffer. The i-th string is at `data[offsets[i]..offsets[i+1]]`. /// - **Data**: concatenated UTF-8 encoded bytes for all strings. -/// - **Null mask**: optional bit-packed validity bitmap (1=valid, 0=null). +/// - **Null mask**: optional bit-packed bitmap (1=valid, 0=null). /// /// ## Arrow compatibility /// The `Apache Arrow` framework defines two string types: @@ -135,7 +135,7 @@ impl StringArray { } } - /// Constructs a dense StringArray from a slice of string slices (no nulls). + /// Constructs a contiguous StringArray from a slice of string slices (no nulls). #[inline] pub fn from_slice(slice: &[&str]) -> Self { let n = slice.len(); @@ -162,7 +162,7 @@ impl StringArray { offsets: offsets.into(), data: Vec64::with_capacity(values_cap).into(), null_mask: if null_mask { - // All-valid (1) default - reserved validity slots default to + // All-valid (1) default - reserved null mask slots default to // valid under Arrow's 1=valid, 0=null convention. Some(Bitmask::new_set_all(n_strings, true)) } else { @@ -750,7 +750,7 @@ impl MaskedArray for StringArray { /// If `n` is greater than the current length, the `value` is appended repeatedly. /// If `n` is smaller, the array is truncated at the correct byte and offset boundary. /// - /// The `null_mask` is left untouched. Callers are responsible for managing validity if needed. + /// The `null_mask` is left untouched. Callers are responsible for managing the null mask if needed. /// /// # Panics /// Panics if `offsets` are invalid or not of length `self.len() + 1`. diff --git a/src/structs/views/chunked/super_array_view.rs b/src/structs/views/chunked/super_array_view.rs index f6cbbd2..3ff434c 100644 --- a/src/structs/views/chunked/super_array_view.rs +++ b/src/structs/views/chunked/super_array_view.rs @@ -293,6 +293,15 @@ impl Concatenate for SuperArrayV { } } +impl From for SuperArray { + /// Materialises the viewed slices as the chunks of an owned chunked + /// array, preserving the chunk boundaries the view carries. + fn from(value: SuperArrayV) -> Self { + let chunks: Vec = value.chunks().map(|slice| slice.to_array()).collect(); + SuperArray::from_arrays_with_field(chunks, value.field) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/structs/views/chunked/super_ndarray_view.rs b/src/structs/views/chunked/super_ndarray_view.rs index 610b504..fabf4ab 100644 --- a/src/structs/views/chunked/super_ndarray_view.rs +++ b/src/structs/views/chunked/super_ndarray_view.rs @@ -453,6 +453,15 @@ impl From> for SuperNdArrayV { } } +impl From> for SuperNdArray { + /// Materialises the viewed windows as the batches of an owned chunked + /// array, preserving the chunk boundaries the view carries. + fn from(value: SuperNdArrayV) -> Self { + let batches: Vec> = value.chunks().map(|slice| slice.to_ndarray()).collect(); + SuperNdArray::from_batches(batches, value.name.clone()) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/structs/views/ndarray_view.rs b/src/structs/views/ndarray_view.rs index 3e58435..662c2fe 100644 --- a/src/structs/views/ndarray_view.rs +++ b/src/structs/views/ndarray_view.rs @@ -616,6 +616,13 @@ impl fmt::Debug for NdArrayV { } } +impl From> for NdArray { + /// Materialises the viewed window as an owned contiguous array. + fn from(value: NdArrayV) -> Self { + value.to_ndarray() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/structs/xarray.rs b/src/structs/xarray.rs index b8af9e1..b601192 100644 --- a/src/structs/xarray.rs +++ b/src/structs/xarray.rs @@ -1226,6 +1226,22 @@ impl TryFrom
for XArray { // Tests // **************************************************************** +impl TryFrom> for Table { + type Error = MinarrowError; + + /// Presents the labelled axes as table columns. + fn try_from(value: XArray) -> Result { + value.to_table() + } +} + +impl From> for NdArray { + /// Drops the axis labels and keeps the contiguous payload. + fn from(value: XArray) -> Self { + value.into_ndarray() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/utils.rs b/src/utils.rs index 02b4042..41a9831 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -205,12 +205,12 @@ pub fn is_simd_aligned(slice: &[T]) -> bool { /// - `N`: Number of SIMD lanes, must match the SIMD vector width for the target operation /// /// # Parameters -/// - `mask`: Source bitmask containing validity information +/// - `mask`: Source bitmask containing null mask information /// - `offset`: Starting bit offset within the bitmask /// - `len`: Maximum number of bits to consider (bounds checking) /// /// # Returns -/// A `Mask` where each lane corresponds to the validity of the corresponding input element. +/// A `Mask` where each lane corresponds to the null mask bit of the corresponding input element. /// Lanes beyond `len` are set to false for safety. /// /// # Usage Example diff --git a/tests/lbuffer_stress.rs b/tests/lbuffer_stress.rs index 3b16585..c8ac61c 100644 --- a/tests/lbuffer_stress.rs +++ b/tests/lbuffer_stress.rs @@ -307,12 +307,12 @@ fn is_null_row(i: usize) -> bool { } #[test] -fn masked_concurrent_validity_reads() { +fn masked_concurrent_null_mask_reads() { let secs = env_or("STRESS_SECS", 2u64); let cap = env_or("STRESS_WINDOW", 8_000_000usize); let readers = env_or("STRESS_READERS", 4usize); - // One masked window. Readers share the value buffer and the validity + // One masked window. Readers share the value buffer and the null // mask, both LBuffer-backed views over the same writer. let mut buf = LBuffer::::with_capacity_masked(cap); let mask = buf.as_bitmask(); @@ -337,12 +337,12 @@ fn masked_concurrent_validity_reads() { assert_eq!( mask.get(last), !is_null_row(last), - "reader {reader_id}: torn validity at {last}" + "reader {reader_id}: torn null mask at {last}" ); assert_eq!( mask.get(0), !is_null_row(0), - "reader {reader_id}: head validity" + "reader {reader_id}: head null mask" ); if !is_null_row(last) { assert_eq!( @@ -389,7 +389,7 @@ fn masked_concurrent_validity_reads() { let slice = data.as_slice(); for i in 0..produced { let want_null = is_null_row(i); - assert_eq!(mask.get(i), !want_null, "final validity at {i}"); + assert_eq!(mask.get(i), !want_null, "final null mask at {i}"); if want_null { nulls += 1; } else { @@ -399,5 +399,5 @@ fn masked_concurrent_validity_reads() { assert_eq!(mask.count_zeros(), nulls, "null count mismatch"); assert!(produced > 0, "stress produced no rows"); - println!("verified {produced} rows ({nulls} nulls) with {readers} readers, no torn validity"); + println!("verified {produced} rows ({nulls} nulls) with {readers} readers, no torn null mask"); }