diff --git a/benchmarks/compress-bench/src/lib.rs b/benchmarks/compress-bench/src/lib.rs index b971f7af6ed..3705343fe5a 100644 --- a/benchmarks/compress-bench/src/lib.rs +++ b/benchmarks/compress-bench/src/lib.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use ::vortex::array::arrays::ChunkedArray; +use ::vortex::array::arrays::chunked::ChunkedArrayExt; use ::vortex::array::arrays::listview::recursive_list_from_list_view; use arrow_array::RecordBatch; use arrow_schema::Schema; diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index e8907fac04e..c6cce9377eb 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -32,15 +32,11 @@ pub type vortex_alp::ALP::OperationsVTable = vortex_alp::ALP pub type vortex_alp::ALP::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild -pub fn vortex_alp::ALP::array_eq(array: &vortex_alp::ALPData, other: &vortex_alp::ALPData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_alp::ALP::array_hash(array: &vortex_alp::ALPData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_alp::ALP::buffer(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_alp::ALP::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_alp::ALP::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_alp::ALP::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_alp::ALP::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -56,11 +52,7 @@ pub fn vortex_alp::ALP::serialize(array: vortex_array::array::view::ArrayView<'_ pub fn vortex_alp::ALP::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_alp::ALP::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_alp::ALP::validate(&self, data: &vortex_alp::ALPData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_alp::ALP::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_alp::ALP::validate(&self, data: &vortex_alp::ALPData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_alp::ALP @@ -68,7 +60,7 @@ pub fn vortex_alp::ALP::scalar_at(array: vortex_array::array::view::ArrayView<'_ impl vortex_array::array::vtable::validity::ValidityChild for vortex_alp::ALP -pub fn vortex_alp::ALP::validity_child(array: &vortex_alp::ALPData) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_alp::ALP::validity_child(array: vortex_array::array::view::ArrayView<'_, vortex_alp::ALP>) -> vortex_array::array::erased::ArrayRef impl vortex_array::arrays::dict::take::TakeExecute for vortex_alp::ALP @@ -106,19 +98,11 @@ pub struct vortex_alp::ALPData impl vortex_alp::ALPData -pub fn vortex_alp::ALPData::encoded(&self) -> &vortex_array::array::erased::ArrayRef - pub fn vortex_alp::ALPData::exponents(&self) -> vortex_alp::Exponents -pub fn vortex_alp::ALPData::into_parts(self) -> (vortex_array::array::erased::ArrayRef, vortex_alp::Exponents, core::option::Option) - -pub fn vortex_alp::ALPData::patches(&self) -> core::option::Option - impl vortex_alp::ALPData -pub fn vortex_alp::ALPData::new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> Self - -pub fn vortex_alp::ALPData::try_new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> vortex_error::VortexResult +pub fn vortex_alp::ALPData::new(exponents: vortex_alp::Exponents, patches: core::option::Option) -> Self impl core::clone::Clone for vortex_alp::ALPData @@ -128,6 +112,14 @@ impl core::fmt::Debug for vortex_alp::ALPData pub fn vortex_alp::ALPData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_alp::ALPData + +pub fn vortex_alp::ALPData::array_eq(&self, other: &Self, _precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_alp::ALPData + +pub fn vortex_alp::ALPData::array_hash(&self, state: &mut H, _precision: vortex_array::hash::Precision) + pub struct vortex_alp::ALPMetadata impl core::clone::Clone for vortex_alp::ALPMetadata @@ -174,15 +166,11 @@ pub type vortex_alp::ALPRD::OperationsVTable = vortex_alp::ALPRD pub type vortex_alp::ALPRD::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild -pub fn vortex_alp::ALPRD::array_eq(array: &vortex_alp::ALPRDData, other: &vortex_alp::ALPRDData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_alp::ALPRD::array_hash(array: &vortex_alp::ALPRDData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_alp::ALPRD::buffer(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_alp::ALPRD::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_alp::ALPRD::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_alp::ALPRD::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_alp::ALPRD::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -198,11 +186,7 @@ pub fn vortex_alp::ALPRD::serialize(array: vortex_array::array::view::ArrayView< pub fn vortex_alp::ALPRD::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_alp::ALPRD::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_alp::ALPRD::validate(&self, data: &vortex_alp::ALPRDData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_alp::ALPRD::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_alp::ALPRD::validate(&self, data: &vortex_alp::ALPRDData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_alp::ALPRD @@ -210,7 +194,7 @@ pub fn vortex_alp::ALPRD::scalar_at(array: vortex_array::array::view::ArrayView< impl vortex_array::array::vtable::validity::ValidityChild for vortex_alp::ALPRD -pub fn vortex_alp::ALPRD::validity_child(array: &vortex_alp::ALPRDData) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_alp::ALPRD::validity_child(array: vortex_array::array::view::ArrayView<'_, vortex_alp::ALPRD>) -> vortex_array::array::erased::ArrayRef impl vortex_array::arrays::dict::take::TakeExecute for vortex_alp::ALPRD @@ -236,22 +220,14 @@ pub struct vortex_alp::ALPRDData impl vortex_alp::ALPRDData -pub fn vortex_alp::ALPRDData::into_parts(self) -> vortex_alp::ALPRDDataParts - -pub fn vortex_alp::ALPRDData::left_parts(&self) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_alp::ALPRDData::into_parts(self, left_parts: vortex_array::array::erased::ArrayRef, right_parts: vortex_array::array::erased::ArrayRef) -> vortex_alp::ALPRDDataParts pub fn vortex_alp::ALPRDData::left_parts_dictionary(&self) -> &vortex_buffer::buffer::Buffer -pub fn vortex_alp::ALPRDData::left_parts_patches(&self) -> core::option::Option - -pub fn vortex_alp::ALPRDData::replace_left_parts_patches(&mut self, patches: core::option::Option) +pub fn vortex_alp::ALPRDData::new(left_parts_dictionary: vortex_buffer::buffer::Buffer, right_bit_width: u8, left_parts_patches: core::option::Option) -> Self pub fn vortex_alp::ALPRDData::right_bit_width(&self) -> u8 -pub fn vortex_alp::ALPRDData::right_parts(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn vortex_alp::ALPRDData::try_new(dtype: vortex_array::dtype::DType, left_parts: vortex_array::array::erased::ArrayRef, left_parts_dictionary: vortex_buffer::buffer::Buffer, right_parts: vortex_array::array::erased::ArrayRef, right_bit_width: u8, left_parts_patches: core::option::Option) -> vortex_error::VortexResult - impl core::clone::Clone for vortex_alp::ALPRDData pub fn vortex_alp::ALPRDData::clone(&self) -> vortex_alp::ALPRDData @@ -260,6 +236,14 @@ impl core::fmt::Debug for vortex_alp::ALPRDData pub fn vortex_alp::ALPRDData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_alp::ALPRDData + +pub fn vortex_alp::ALPRDData::array_eq(&self, other: &Self, precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_alp::ALPRDData + +pub fn vortex_alp::ALPRDData::array_hash(&self, state: &mut H, precision: vortex_array::hash::Precision) + pub struct vortex_alp::ALPRDDataParts pub vortex_alp::ALPRDDataParts::left_parts: vortex_array::array::erased::ArrayRef @@ -346,6 +330,30 @@ pub fn vortex_alp::RDEncoder::from_parts(right_bit_width: u8, codes: alloc::vec: pub fn vortex_alp::RDEncoder::new(sample: &[T]) -> Self where T: vortex_alp::ALPRDFloat + vortex_array::dtype::ptype::NativePType, ::UINT: vortex_array::dtype::ptype::NativePType +pub trait vortex_alp::ALPArrayExt: vortex_array::array::typed::TypedArrayRef + +pub fn vortex_alp::ALPArrayExt::encoded(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn vortex_alp::ALPArrayExt::exponents(&self) -> vortex_alp::Exponents + +pub fn vortex_alp::ALPArrayExt::patches(&self) -> core::option::Option + +impl> vortex_alp::ALPArrayExt for T + +pub fn T::encoded(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn T::exponents(&self) -> vortex_alp::Exponents + +pub fn T::patches(&self) -> core::option::Option + +pub trait vortex_alp::ALPArrayOwnedExt + +pub fn vortex_alp::ALPArrayOwnedExt::into_parts(self) -> (vortex_array::array::erased::ArrayRef, vortex_alp::Exponents, core::option::Option) + +impl vortex_alp::ALPArrayOwnedExt for vortex_array::array::typed::Array + +pub fn vortex_array::array::typed::Array::into_parts(self) -> (vortex_array::array::erased::ArrayRef, vortex_alp::Exponents, core::option::Option) + pub trait vortex_alp::ALPFloat: vortex_alp::alp::private::Sealed + num_traits::float::Float + core::fmt::Display + vortex_array::dtype::ptype::NativePType pub type vortex_alp::ALPFloat::ALPInt: num_traits::int::PrimInt + core::fmt::Display + num_traits::cast::ToPrimitive + core::marker::Copy + vortex_array::dtype::ptype::NativePType + core::convert::Into @@ -478,6 +486,38 @@ pub fn f64::find_best_exponents(values: &[Self]) -> vortex_alp::Exponents pub fn f64::from_int(n: Self::ALPInt) -> Self +pub trait vortex_alp::ALPRDArrayExt: vortex_array::array::typed::TypedArrayRef + +pub fn vortex_alp::ALPRDArrayExt::left_parts(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn vortex_alp::ALPRDArrayExt::left_parts_dictionary(&self) -> &vortex_buffer::buffer::Buffer + +pub fn vortex_alp::ALPRDArrayExt::left_parts_patches(&self) -> core::option::Option + +pub fn vortex_alp::ALPRDArrayExt::right_bit_width(&self) -> u8 + +pub fn vortex_alp::ALPRDArrayExt::right_parts(&self) -> &vortex_array::array::erased::ArrayRef + +impl> vortex_alp::ALPRDArrayExt for T + +pub fn T::left_parts(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn T::left_parts_dictionary(&self) -> &vortex_buffer::buffer::Buffer + +pub fn T::left_parts_patches(&self) -> core::option::Option + +pub fn T::right_bit_width(&self) -> u8 + +pub fn T::right_parts(&self) -> &vortex_array::array::erased::ArrayRef + +pub trait vortex_alp::ALPRDArrayOwnedExt + +pub fn vortex_alp::ALPRDArrayOwnedExt::into_data_parts(self) -> vortex_alp::ALPRDDataParts + +impl vortex_alp::ALPRDArrayOwnedExt for vortex_array::array::typed::Array + +pub fn vortex_array::array::typed::Array::into_data_parts(self) -> vortex_alp::ALPRDDataParts + pub trait vortex_alp::ALPRDFloat: vortex_alp::alp_rd::private::Sealed + num_traits::float::Float + core::marker::Copy + vortex_array::dtype::ptype::NativePType pub type vortex_alp::ALPRDFloat::UINT: vortex_array::dtype::ptype::NativePType + num_traits::int::PrimInt + num_traits::identities::One + core::marker::Copy diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index feff33c6269..aae281d3be4 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -3,6 +3,7 @@ use std::fmt::Debug; use std::hash::Hash; +use std::hash::Hasher; use prost::Message; use vortex_array::Array; @@ -16,6 +17,7 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::TypedArrayRef; use vortex_array::arrays::Primitive; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; @@ -44,6 +46,22 @@ use crate::alp::rules::RULES; vtable!(ALP, ALP, ALPData); +impl ArrayHash for ALPData { + fn array_hash(&self, state: &mut H, _precision: Precision) { + self.exponents.hash(state); + self.patch_offset.hash(state); + self.patch_offset_within_chunk.hash(state); + } +} + +impl ArrayEq for ALPData { + fn array_eq(&self, other: &Self, _precision: Precision) -> bool { + self.exponents == other.exponents + && self.patch_offset == other.patch_offset + && self.patch_offset_within_chunk == other.patch_offset_within_chunk + } +} + impl VTable for ALP { type ArrayData = ALPData; @@ -54,20 +72,27 @@ impl VTable for ALP { Self::ID } - fn validate(&self, data: &ALPData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate_against_outer(dtype, len) - } - - fn array_hash(array: &ALPData, state: &mut H, precision: Precision) { - array.encoded().array_hash(state, precision); - array.exponents.hash(state); - array.patches().array_hash(state, precision); - } - - fn array_eq(array: &ALPData, other: &ALPData, precision: Precision) -> bool { - array.encoded().array_eq(other.encoded(), precision) - && array.exponents == other.exponents - && array.patches().array_eq(&other.patches(), precision) + fn validate( + &self, + data: &ALPData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + validate_parts( + dtype, + len, + data.exponents, + slots[ENCODED_SLOT] + .as_ref() + .vortex_expect("ALPArray encoded slot"), + patches_from_slots( + slots, + data.patch_offset, + data.patch_offset_within_chunk, + len, + ), + ) } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -105,7 +130,7 @@ impl VTable for ALP { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = ALPMetadata::decode(metadata)?; let encoded_ptype = match &dtype { DType::Primitive(PType::F32, n) => DType::Primitive(PType::I32, *n), @@ -128,42 +153,21 @@ impl VTable for ALP { }) .transpose()?; - ALPData::try_new( - encoded, + let slots = ALPData::make_slots(&encoded, &patches); + let data = ALPData::new( Exponents { e: u8::try_from(metadata.exp_e)?, f: u8::try_from(metadata.exp_f)?, }, patches, - ) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + ); + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "ALPArray expects {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - - // If patch slots are being cleared, clear the metadata too - if slots[PATCH_INDICES_SLOT].is_none() || slots[PATCH_VALUES_SLOT].is_none() { - array.patch_offset = None; - array.patch_offset_within_chunk = None; - }; - - array.slots = slots; - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { let array = require_child!(array, array.encoded(), ENCODED_SLOT => Primitive); require_patches!( @@ -215,7 +219,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = [ #[derive(Clone, Debug)] pub struct ALPData { - pub(super) slots: Vec>, patch_offset: Option, patch_offset_within_chunk: Option, exponents: Exponents, @@ -302,26 +305,6 @@ impl ALPData { } } - fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { - let patches = self.patches(); - let logical_dtype = Self::logical_dtype(self.encoded())?; - Self::validate_components(self.encoded(), self.exponents, patches.as_ref())?; - - vortex_ensure!( - self.encoded().len() == len, - "ALP encoded len {} != outer len {len}", - self.encoded().len(), - ); - vortex_ensure!( - &logical_dtype == dtype, - "ALP dtype {} does not match encoded logical dtype {}", - dtype, - logical_dtype, - ); - - Ok(()) - } - /// Validate that any patches provided are valid for the ALPArray. fn validate_patches(patches: &Patches, encoded: &ArrayRef) -> VortexResult<()> { vortex_ensure!( @@ -345,105 +328,33 @@ impl ALPData { impl ALPData { /// Build a new `ALPArray` from components, panicking on validation failure. /// - /// See [`ALPData::try_new`] for reference on preconditions that must pass before + /// See [`ALP::try_new`] for reference on preconditions that must pass before /// calling this method. - pub fn new(encoded: ArrayRef, exponents: Exponents, patches: Option) -> Self { - Self::try_new(encoded, exponents, patches).vortex_expect("ALPArray new") - } - - /// Build a new `ALPArray` from components: - /// - /// * `encoded` contains the ALP-encoded ints. Any null values are replaced with placeholders - /// * `exponents` are the ALP exponents, valid range depends on the data type - /// * `patches` are any patch values that don't cleanly encode using the ALP conversion function - /// - /// This method validates the inputs and will return an error if any validation fails. - /// - /// # Validation - /// - /// * The `encoded` array must be either `i32` or `i64` - /// * If `i32`, any `patches` must have DType `f32` with same nullability - /// * If `i64`, then `patches`must have DType `f64` with same nullability - /// * `exponents` must be in the valid range depending on if the ALPArray is of type `f32` or - /// `f64`. - /// * `patches` must have an `array_len` equal to the length of `encoded` - /// - /// Any failure of these preconditions will result in an error being returned. - /// - /// # Examples - /// - /// ``` - /// # use vortex_alp::{ALP, ALPData, Exponents}; - /// # use vortex_array::IntoArray; - /// # use vortex_buffer::buffer; - /// - /// // Returns error because buffer has wrong PType. - /// let result = ALPData::try_new( - /// buffer![1i8].into_array(), - /// Exponents { e: 1, f: 1 }, - /// None - /// ); - /// assert!(result.is_err()); - /// - /// // Returns error because Exponents are out of bounds for f32 - /// let result = ALPData::try_new( - /// buffer![1i32, 2i32].into_array(), - /// Exponents { e: 100, f: 100 }, - /// None - /// ); - /// assert!(result.is_err()); - /// - /// // Success! - /// let value = ALP::try_new( - /// buffer![0i32].into_array(), - /// Exponents { e: 1, f: 1 }, - /// None - /// ).unwrap(); - /// - /// assert_eq!(value.scalar_at(0).unwrap(), 0f32.into()); - /// ``` - pub fn try_new( - encoded: ArrayRef, - exponents: Exponents, - patches: Option, - ) -> VortexResult { - Self::validate_components(&encoded, exponents, patches.as_ref())?; - - let slots = Self::make_slots(&encoded, &patches); + pub fn new(exponents: Exponents, patches: Option) -> Self { let (patch_offset, patch_offset_within_chunk) = match &patches { Some(p) => (Some(p.offset()), p.offset_within_chunk()), None => (None, None), }; - Ok(Self { - slots, + Self { patch_offset, patch_offset_within_chunk, exponents, - }) + } } + /// Build a new `ALPArray` from components: + /// + /// * `encoded` contains the ALP-encoded ints. Any null values are replaced with placeholders + /// * `exponents` are the ALP exponents, valid range depends on the data type + /// * `patches` are any patch values that don't cleanly encode using the ALP conversion function + /// /// Build a new `ALPArray` from components without validation. /// - /// See [`ALPData::try_new`] for information about the preconditions that should be checked + /// See [`ALP::try_new`] for information about the preconditions that should be checked /// **before** calling this method. - pub(crate) unsafe fn new_unchecked( - encoded: ArrayRef, - exponents: Exponents, - patches: Option, - ) -> Self { - let slots = Self::make_slots(&encoded, &patches); - let (patch_offset, patch_offset_within_chunk) = match &patches { - Some(p) => (Some(p.offset()), p.offset_within_chunk()), - None => (None, None), - }; - - Self { - slots, - patch_offset, - patch_offset_within_chunk, - exponents, - } + pub(crate) unsafe fn new_unchecked(exponents: Exponents, patches: Option) -> Self { + Self::new(exponents, patches) } } @@ -452,13 +363,12 @@ impl ALP { pub fn new(encoded: ArrayRef, exponents: Exponents, patches: Option) -> ALPArray { let dtype = ALPData::logical_dtype(&encoded).vortex_expect("ALP encoded dtype"); let len = encoded.len(); + let slots = ALPData::make_slots(&encoded, &patches); unsafe { - Array::from_parts_unchecked(ArrayParts::new( - ALP, - dtype, - len, - ALPData::new(encoded, exponents, patches), - )) + Array::from_parts_unchecked( + ArrayParts::new(ALP, dtype, len, ALPData::new(exponents, patches)) + .with_slots(slots), + ) } } @@ -469,12 +379,13 @@ impl ALP { ) -> VortexResult { let dtype = ALPData::logical_dtype(&encoded)?; let len = encoded.len(); - let data = ALPData::try_new(encoded, exponents, patches)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(ALP, dtype, len, data)) }) + let slots = ALPData::make_slots(&encoded, &patches); + let data = ALPData::new(exponents, patches); + Array::try_from_parts(ArrayParts::new(ALP, dtype, len, data).with_slots(slots)) } /// # Safety - /// See [`ALPData::try_new`] for preconditions. + /// See [`ALP::try_new`] for preconditions. pub unsafe fn new_unchecked( encoded: ArrayRef, exponents: Exponents, @@ -482,8 +393,11 @@ impl ALP { ) -> ALPArray { let dtype = ALPData::logical_dtype(&encoded).vortex_expect("ALP encoded dtype"); let len = encoded.len(); - let data = unsafe { ALPData::new_unchecked(encoded, exponents, patches) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(ALP, dtype, len, data)) } + let slots = ALPData::make_slots(&encoded, &patches); + let data = unsafe { ALPData::new_unchecked(exponents, patches) }; + unsafe { + Array::from_parts_unchecked(ArrayParts::new(ALP, dtype, len, data).with_slots(slots)) + } } } @@ -505,55 +419,99 @@ impl ALPData { ] } - pub fn encoded(&self) -> &ArrayRef { - self.slots[ENCODED_SLOT] + #[inline] + pub fn exponents(&self) -> Exponents { + self.exponents + } +} + +pub trait ALPArrayExt: TypedArrayRef { + fn encoded(&self) -> &ArrayRef { + self.as_ref().slots()[ENCODED_SLOT] .as_ref() .vortex_expect("ALPArray encoded slot") } - #[inline] - pub fn exponents(&self) -> Exponents { + fn exponents(&self) -> Exponents { self.exponents } - pub fn patches(&self) -> Option { - match ( - &self.slots[PATCH_INDICES_SLOT], - &self.slots[PATCH_VALUES_SLOT], - ) { - (Some(indices), Some(values)) => { - let patch_offset = self - .patch_offset - .vortex_expect("has patch slots but no patch_offset"); - Some(unsafe { - Patches::new_unchecked( - self.encoded().len(), - patch_offset, - indices.clone(), - values.clone(), - self.slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), - self.patch_offset_within_chunk, - ) - }) - } - _ => None, + fn patches(&self) -> Option { + patches_from_slots( + self.as_ref().slots(), + self.patch_offset, + self.patch_offset_within_chunk, + self.as_ref().len(), + ) + } +} + +fn patches_from_slots( + slots: &[Option], + patch_offset: Option, + patch_offset_within_chunk: Option, + len: usize, +) -> Option { + match (&slots[PATCH_INDICES_SLOT], &slots[PATCH_VALUES_SLOT]) { + (Some(indices), Some(values)) => { + let patch_offset = patch_offset.vortex_expect("has patch slots but no patch_offset"); + Some(unsafe { + Patches::new_unchecked( + len, + patch_offset, + indices.clone(), + values.clone(), + slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), + patch_offset_within_chunk, + ) + }) } + _ => None, } +} + +fn validate_parts( + dtype: &DType, + len: usize, + exponents: Exponents, + encoded: &ArrayRef, + patches: Option, +) -> VortexResult<()> { + let logical_dtype = ALPData::logical_dtype(encoded)?; + ALPData::validate_components(encoded, exponents, patches.as_ref())?; + vortex_ensure!( + encoded.len() == len, + "ALP encoded len {} != outer len {len}", + encoded.len(), + ); + vortex_ensure!( + &logical_dtype == dtype, + "ALP dtype {} does not match encoded logical dtype {}", + dtype, + logical_dtype, + ); + Ok(()) +} + +impl> ALPArrayExt for T {} + +pub trait ALPArrayOwnedExt { + fn into_parts(self) -> (ArrayRef, Exponents, Option); +} - /// Consumes the array and returns its parts. +impl ALPArrayOwnedExt for Array { #[inline] - pub fn into_parts(mut self) -> (ArrayRef, Exponents, Option) { + fn into_parts(self) -> (ArrayRef, Exponents, Option) { let patches = self.patches(); - let encoded = self.slots[ENCODED_SLOT] - .take() - .vortex_expect("ALPArray encoded slot"); - (encoded, self.exponents, patches) + let exponents = self.exponents(); + let encoded = self.encoded().clone(); + (encoded, exponents, patches) } } impl ValidityChild for ALP { - fn validity_child(array: &ALPData) -> &ArrayRef { - array.encoded() + fn validity_child(array: ArrayView<'_, ALP>) -> ArrayRef { + array.encoded().clone() } } diff --git a/encodings/alp/src/alp/compress.rs b/encodings/alp/src/alp/compress.rs index 7fda471bbda..a8215eef6fb 100644 --- a/encodings/alp/src/alp/compress.rs +++ b/encodings/alp/src/alp/compress.rs @@ -134,6 +134,7 @@ mod tests { use vortex_buffer::buffer; use super::*; + use crate::alp::array::ALPArrayExt; use crate::decompress_into_array; #[test] diff --git a/encodings/alp/src/alp/compute/between.rs b/encodings/alp/src/alp/compute/between.rs index efaaf747768..163a236b483 100644 --- a/encodings/alp/src/alp/compute/between.rs +++ b/encodings/alp/src/alp/compute/between.rs @@ -19,6 +19,7 @@ use vortex_error::VortexResult; use crate::ALP; use crate::ALPFloat; +use crate::alp::array::ALPArrayExt; use crate::match_each_alp_float_ptype; impl BetweenReduce for ALP { @@ -101,6 +102,7 @@ mod tests { use vortex_array::scalar_fn::fns::between::StrictComparison; use crate::ALPArray; + use crate::alp::array::ALPArrayExt; use crate::alp::compute::between::between_impl; use crate::alp_encode; diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index bd8a8b38381..950688dfcac 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -10,6 +10,7 @@ use vortex_array::patches::Patches; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; +use crate::ALPArrayExt; use crate::alp::ALP; impl CastReduce for ALP { @@ -70,6 +71,7 @@ mod tests { use vortex_error::VortexExpect; use vortex_error::VortexResult; + use crate::alp::array::ALPArrayExt; use crate::alp_encode; #[test] diff --git a/encodings/alp/src/alp/compute/compare.rs b/encodings/alp/src/alp/compute/compare.rs index 867a14db79a..f15821c725b 100644 --- a/encodings/alp/src/alp/compute/compare.rs +++ b/encodings/alp/src/alp/compute/compare.rs @@ -19,6 +19,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_err; use crate::ALP; +use crate::ALPArrayExt; use crate::ALPFloat; use crate::match_each_alp_float_ptype; diff --git a/encodings/alp/src/alp/compute/filter.rs b/encodings/alp/src/alp/compute/filter.rs index 3daa6d53d7b..51cc206e2bb 100644 --- a/encodings/alp/src/alp/compute/filter.rs +++ b/encodings/alp/src/alp/compute/filter.rs @@ -10,6 +10,7 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::ALP; +use crate::ALPArrayExt; impl FilterKernel for ALP { fn filter( diff --git a/encodings/alp/src/alp/compute/mask.rs b/encodings/alp/src/alp/compute/mask.rs index 62e0110ad87..3d8172fa7e9 100644 --- a/encodings/alp/src/alp/compute/mask.rs +++ b/encodings/alp/src/alp/compute/mask.rs @@ -12,6 +12,7 @@ use vortex_array::validity::Validity; use vortex_error::VortexResult; use crate::ALP; +use crate::ALPArrayExt; impl MaskReduce for ALP { fn mask(array: ArrayView<'_, Self>, mask: &ArrayRef) -> VortexResult> { @@ -64,6 +65,7 @@ mod test { use vortex_array::scalar_fn::fns::mask::MaskKernel; use vortex_buffer::buffer; + use crate::alp::array::ALPArrayExt; use crate::alp_encode; #[rstest] diff --git a/encodings/alp/src/alp/compute/nan_count.rs b/encodings/alp/src/alp/compute/nan_count.rs index ace09429c30..b76dd6e7a97 100644 --- a/encodings/alp/src/alp/compute/nan_count.rs +++ b/encodings/alp/src/alp/compute/nan_count.rs @@ -11,6 +11,7 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::ALP; +use crate::ALPArrayExt; /// ALP-specific NaN count kernel. /// diff --git a/encodings/alp/src/alp/compute/slice.rs b/encodings/alp/src/alp/compute/slice.rs index 9ebf0926d1e..3959b624083 100644 --- a/encodings/alp/src/alp/compute/slice.rs +++ b/encodings/alp/src/alp/compute/slice.rs @@ -11,6 +11,7 @@ use vortex_array::arrays::slice::SliceKernel; use vortex_error::VortexResult; use crate::ALP; +use crate::ALPArrayExt; impl SliceKernel for ALP { fn slice( diff --git a/encodings/alp/src/alp/compute/take.rs b/encodings/alp/src/alp/compute/take.rs index 77d161737c3..22dcdeaf6a7 100644 --- a/encodings/alp/src/alp/compute/take.rs +++ b/encodings/alp/src/alp/compute/take.rs @@ -9,6 +9,7 @@ use vortex_array::arrays::dict::TakeExecute; use vortex_error::VortexResult; use crate::ALP; +use crate::ALPArrayExt; impl TakeExecute for ALP { fn take( diff --git a/encodings/alp/src/alp/decompress.rs b/encodings/alp/src/alp/decompress.rs index ea52e362efe..6bc5b469d42 100644 --- a/encodings/alp/src/alp/decompress.rs +++ b/encodings/alp/src/alp/decompress.rs @@ -15,6 +15,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::ALPArray; +use crate::ALPArrayOwnedExt; use crate::ALPFloat; use crate::Exponents; use crate::match_each_alp_float_ptype; @@ -29,7 +30,7 @@ pub fn decompress_into_array( ctx: &mut ExecutionCtx, ) -> VortexResult { let dtype = array.dtype().clone(); - let (encoded, exponents, patches) = array.into_data().into_parts(); + let (encoded, exponents, patches) = ALPArrayOwnedExt::into_parts(array); if let Some(ref patches) = patches && let Some(chunk_offsets) = patches.chunk_offsets() { @@ -62,7 +63,7 @@ pub fn decompress_into_array( /// A `PrimitiveArray` containing the decompressed floating-point values with all patches applied. pub fn execute_decompress(array: ALPArray, ctx: &mut ExecutionCtx) -> VortexResult { let dtype = array.dtype().clone(); - let (encoded, exponents, patches) = array.into_data().into_parts(); + let (encoded, exponents, patches) = ALPArrayOwnedExt::into_parts(array); if let Some(ref patches) = patches && let Some(chunk_offsets) = patches.chunk_offsets() { diff --git a/encodings/alp/src/alp/ops.rs b/encodings/alp/src/alp/ops.rs index 25cf19a3e55..47cce5b45f6 100644 --- a/encodings/alp/src/alp/ops.rs +++ b/encodings/alp/src/alp/ops.rs @@ -9,6 +9,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::ALP; +use crate::ALPArrayExt; use crate::ALPFloat; use crate::match_each_alp_float_ptype; diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index b16470372bc..fe7113c288f 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -3,6 +3,7 @@ use std::fmt::Debug; use std::hash::Hash; +use std::hash::Hasher; use itertools::Itertools; use prost::Message; @@ -17,6 +18,7 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::TypedArrayRef; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; @@ -62,6 +64,25 @@ pub struct ALPRDMetadata { patches: Option, } +impl ArrayHash for ALPRDData { + fn array_hash(&self, state: &mut H, precision: Precision) { + self.left_parts_dictionary.array_hash(state, precision); + self.right_bit_width.hash(state); + self.patch_offset.hash(state); + self.patch_offset_within_chunk.hash(state); + } +} + +impl ArrayEq for ALPRDData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + self.left_parts_dictionary + .array_eq(&other.left_parts_dictionary, precision) + && self.right_bit_width == other.right_bit_width + && self.patch_offset == other.patch_offset + && self.patch_offset_within_chunk == other.patch_offset_within_chunk + } +} + impl VTable for ALPRD { type ArrayData = ALPRDData; @@ -72,28 +93,26 @@ impl VTable for ALPRD { Self::ID } - fn validate(&self, data: &ALPRDData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate_against_outer(dtype, len) - } - - fn array_hash(array: &ALPRDData, state: &mut H, precision: Precision) { - array.left_parts().array_hash(state, precision); - array.left_parts_dictionary.array_hash(state, precision); - array.right_parts().array_hash(state, precision); - array.right_bit_width.hash(state); - array.left_parts_patches.array_hash(state, precision); - } - - fn array_eq(array: &ALPRDData, other: &ALPRDData, precision: Precision) -> bool { - array.left_parts().array_eq(other.left_parts(), precision) - && array - .left_parts_dictionary - .array_eq(&other.left_parts_dictionary, precision) - && array.right_parts().array_eq(other.right_parts(), precision) - && array.right_bit_width == other.right_bit_width - && array - .left_parts_patches - .array_eq(&other.left_parts_patches, precision) + fn validate( + &self, + data: &ALPRDData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + validate_parts( + dtype, + len, + left_parts_from_slots(slots), + right_parts_from_slots(slots), + patches_from_slots( + slots, + data.patch_offset, + data.patch_offset_within_chunk, + len, + ) + .as_ref(), + ) } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -123,7 +142,7 @@ impl VTable for ALPRD { left_parts_ptype: array.left_parts().dtype().as_ptype() as i32, patches: array .left_parts_patches() - .map(|p| p.to_metadata(array.len(), array.left_parts().dtype())) + .map(|p| p.to_metadata(array.len(), p.dtype())) .transpose()?, } .encode_to_vec(), @@ -138,7 +157,7 @@ impl VTable for ALPRD { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = ALPRDMetadata::decode(metadata)?; if children.len() < 2 { vortex_bail!( @@ -173,7 +192,7 @@ impl VTable for ALPRD { .patches .map(|p| { let indices = children.get(2, &p.indices_dtype()?, p.len()?)?; - let values = children.get(3, &left_parts_dtype, p.len()?)?; + let values = children.get(3, &left_parts_dtype.as_nonnullable(), p.len()?)?; Patches::new( len, @@ -185,12 +204,10 @@ impl VTable for ALPRD { ) }) .transpose()?; - - ALPRDData::try_new( - dtype.clone(), - left_parts, + let left_parts_patches = ALPRDData::canonicalize_patches(&left_parts, left_parts_patches)?; + let slots = ALPRDData::make_slots(&left_parts, &right_parts, &left_parts_patches); + let data = ALPRDData::new( left_parts_dictionary, - right_parts, u8::try_from(metadata.right_bit_width).map_err(|_| { vortex_err!( "right_bit_width {} out of u8 range", @@ -198,47 +215,14 @@ impl VTable for ALPRD { ) })?, left_parts_patches, - ) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + ); + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "ALPRDArray expects {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - - // Reconstruct patches from slots + existing metadata - array.left_parts_patches = - match (&slots[LP_PATCH_INDICES_SLOT], &slots[LP_PATCH_VALUES_SLOT]) { - (Some(indices), Some(values)) => { - let old = array - .left_parts_patches - .as_ref() - .vortex_expect("ALPRDArray had patch slots but no patches metadata"); - Some(Patches::new( - old.array_len(), - old.offset(), - indices.clone(), - values.clone(), - slots[LP_PATCH_CHUNK_OFFSETS_SLOT].clone(), - )?) - } - _ => None, - }; - array.slots = slots; - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { let array = require_child!(array, array.left_parts(), 0 => Primitive); let array = require_child!(array, array.right_parts(), 1 => Primitive); @@ -257,15 +241,15 @@ impl VTable for ALPRD { right_parts, left_parts_dictionary, left_parts_patches, - } = array.into_data().into_parts(); + } = ALPRDArrayOwnedExt::into_data_parts(array); let ptype = dtype.as_ptype(); let left_parts = left_parts - .try_into::() + .try_downcast::() .ok() .vortex_expect("ALPRD execute: left_parts is primitive"); let right_parts = right_parts - .try_into::() + .try_downcast::() .ok() .vortex_expect("ALPRD execute: right_parts is primitive"); @@ -341,8 +325,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = [ #[derive(Clone, Debug)] pub struct ALPRDData { - slots: Vec>, - left_parts_patches: Option, + patch_offset: Option, + patch_offset_within_chunk: Option, left_parts_dictionary: Buffer, right_bit_width: u8, } @@ -370,24 +354,14 @@ impl ALPRD { left_parts_patches: Option, ) -> VortexResult { let len = left_parts.len(); - let logical_dtype = dtype.clone(); - let data = ALPRDData::try_new( - dtype, - left_parts, - left_parts_dictionary, - right_parts, - right_bit_width, - left_parts_patches, - )?; - Ok( - unsafe { - Array::from_parts_unchecked(ArrayParts::new(ALPRD, logical_dtype, len, data)) - }, - ) + let left_parts_patches = ALPRDData::canonicalize_patches(&left_parts, left_parts_patches)?; + let slots = ALPRDData::make_slots(&left_parts, &right_parts, &left_parts_patches); + let data = ALPRDData::new(left_parts_dictionary, right_bit_width, left_parts_patches); + Array::try_from_parts(ArrayParts::new(ALPRD, dtype, len, data).with_slots(slots)) } /// # Safety - /// See [`ALPRDData::try_new`] for preconditions. + /// See [`ALPRD::try_new`] for preconditions. pub unsafe fn new_unchecked( dtype: DType, left_parts: ArrayRef, @@ -397,165 +371,64 @@ impl ALPRD { left_parts_patches: Option, ) -> ALPRDArray { let len = left_parts.len(); - let logical_dtype = dtype.clone(); + let slots = ALPRDData::make_slots(&left_parts, &right_parts, &left_parts_patches); let data = unsafe { - ALPRDData::new_unchecked( - dtype, - left_parts, - left_parts_dictionary, - right_parts, - right_bit_width, - left_parts_patches, - ) + ALPRDData::new_unchecked(left_parts_dictionary, right_bit_width, left_parts_patches) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(ALPRD, logical_dtype, len, data)) } + unsafe { + Array::from_parts_unchecked(ArrayParts::new(ALPRD, dtype, len, data).with_slots(slots)) + } } } impl ALPRDData { - fn validate_parts( - dtype: &DType, - len: usize, + fn canonicalize_patches( left_parts: &ArrayRef, - right_parts: &ArrayRef, - left_parts_patches: Option<&Patches>, - ) -> VortexResult<()> { - if !dtype.is_float() { - vortex_bail!("ALPRDArray given invalid DType ({dtype})"); - } - - vortex_ensure!( - left_parts.len() == len, - "left_parts len {} != outer len {len}", - left_parts.len(), - ); - vortex_ensure!( - right_parts.len() == len, - "right_parts len {} != outer len {len}", - right_parts.len(), - ); - - if !left_parts.dtype().is_unsigned_int() { - vortex_bail!("left_parts dtype must be uint"); - } - // we delegate array validity to the left_parts child - if dtype.is_nullable() != left_parts.dtype().is_nullable() { - vortex_bail!( - "ALPRDArray dtype nullability ({}) must match left_parts dtype nullability ({})", - dtype, - left_parts.dtype() - ); - } - - let expected_right_parts_dtype = match dtype { - DType::Primitive(PType::F32, _) => { - DType::Primitive(PType::U32, Nullability::NonNullable) - } - DType::Primitive(PType::F64, _) => { - DType::Primitive(PType::U64, Nullability::NonNullable) - } - _ => vortex_bail!("Expected f32 or f64 dtype, got {:?}", dtype), - }; - vortex_ensure!( - right_parts.dtype() == &expected_right_parts_dtype, - "right_parts dtype {} does not match expected {}", - right_parts.dtype(), - expected_right_parts_dtype, - ); - - if let Some(patches) = left_parts_patches { - vortex_ensure!( - patches.array_len() == len, - "patches array_len {} != outer len {len}", - patches.array_len(), - ); - vortex_ensure!( - patches.dtype() == left_parts.dtype(), - "patches dtype {} does not match left_parts dtype {}", - patches.dtype(), - left_parts.dtype(), - ); - vortex_ensure!( - patches.values().all_valid()?, - "patches must be all valid: {}", - patches.values() - ); - } - - Ok(()) - } - - fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { - Self::validate_parts( - dtype, - len, - self.left_parts(), - self.right_parts(), - self.left_parts_patches.as_ref(), - ) - } - - /// Build a new `ALPRDArray` from components. - pub fn try_new( - dtype: DType, - left_parts: ArrayRef, - left_parts_dictionary: Buffer, - right_parts: ArrayRef, - right_bit_width: u8, left_parts_patches: Option, - ) -> VortexResult { - let len = left_parts.len(); - let left_parts_patches = left_parts_patches + ) -> VortexResult> { + left_parts_patches .map(|patches| { if !patches.values().all_valid()? { vortex_bail!("patches must be all valid: {}", patches.values()); } // TODO(ngates): assert the DType, don't cast it. // TODO(joe): assert the DType, don't cast it in the next PR. - let mut patches = patches.cast_values(left_parts.dtype())?; + let mut patches = patches.cast_values(&left_parts.dtype().as_nonnullable())?; // Force execution of the lazy cast so patch values are materialized // before serialization. *patches.values_mut() = patches.values().to_canonical()?.into_array(); Ok(patches) }) - .transpose()?; - - Self::validate_parts( - &dtype, - len, - &left_parts, - &right_parts, - left_parts_patches.as_ref(), - )?; + .transpose() + } - let slots = Self::make_slots(&left_parts, &right_parts, &left_parts_patches); + /// Build a new `ALPRDArray` from components. + pub fn new( + left_parts_dictionary: Buffer, + right_bit_width: u8, + left_parts_patches: Option, + ) -> Self { + let (patch_offset, patch_offset_within_chunk) = match &left_parts_patches { + Some(patches) => (Some(patches.offset()), patches.offset_within_chunk()), + None => (None, None), + }; - Ok(Self { - slots, - left_parts_patches, + Self { + patch_offset, + patch_offset_within_chunk, left_parts_dictionary, right_bit_width, - }) + } } /// Build a new `ALPRDArray` from components. This does not perform any validation, and instead /// it constructs it from parts. pub(crate) unsafe fn new_unchecked( - _dtype: DType, - left_parts: ArrayRef, left_parts_dictionary: Buffer, - right_parts: ArrayRef, right_bit_width: u8, left_parts_patches: Option, ) -> Self { - let slots = Self::make_slots(&left_parts, &right_parts, &left_parts_patches); - - Self { - slots, - left_parts_patches, - left_parts_dictionary, - right_bit_width, - } + Self::new(left_parts_dictionary, right_bit_width, left_parts_patches) } fn make_slots( @@ -581,74 +454,181 @@ impl ALPRDData { } /// Return all the owned parts of the array - pub fn into_parts(mut self) -> ALPRDDataParts { - let left_parts = self.slots[LEFT_PARTS_SLOT] - .take() - .vortex_expect("ALPRDArray left_parts slot"); - let right_parts = self.slots[RIGHT_PARTS_SLOT] - .take() - .vortex_expect("ALPRDArray right_parts slot"); + pub fn into_parts(self, left_parts: ArrayRef, right_parts: ArrayRef) -> ALPRDDataParts { ALPRDDataParts { left_parts, - left_parts_patches: self.left_parts_patches, + left_parts_patches: None, left_parts_dictionary: self.left_parts_dictionary, right_parts, } } - /// The leftmost (most significant) bits of the floating point values stored in the array. - /// - /// These are bit-packed and dictionary encoded, and cannot directly be interpreted without - /// the metadata of this array. - pub fn left_parts(&self) -> &ArrayRef { - self.slots[LEFT_PARTS_SLOT] - .as_ref() - .vortex_expect("ALPRDArray left_parts slot") - } - - /// The rightmost (least significant) bits of the floating point values stored in the array. - pub fn right_parts(&self) -> &ArrayRef { - self.slots[RIGHT_PARTS_SLOT] - .as_ref() - .vortex_expect("ALPRDArray right_parts slot") - } - #[inline] pub fn right_bit_width(&self) -> u8 { self.right_bit_width } - /// Patches of left-most bits. - pub fn left_parts_patches(&self) -> Option { - self.left_parts_patches.clone() - } - /// The dictionary that maps the codes in `left_parts` into bit patterns. #[inline] pub fn left_parts_dictionary(&self) -> &Buffer { &self.left_parts_dictionary } +} - pub fn replace_left_parts_patches(&mut self, patches: Option) { - // Update both the patches and the corresponding slots to keep them in sync. - let (pi, pv, pco) = match &patches { - Some(p) => ( - Some(p.indices().clone()), - Some(p.values().clone()), - p.chunk_offsets().clone(), - ), - None => (None, None, None), +fn left_parts_from_slots(slots: &[Option]) -> &ArrayRef { + slots[LEFT_PARTS_SLOT] + .as_ref() + .vortex_expect("ALPRDArray left_parts slot") +} + +fn right_parts_from_slots(slots: &[Option]) -> &ArrayRef { + slots[RIGHT_PARTS_SLOT] + .as_ref() + .vortex_expect("ALPRDArray right_parts slot") +} + +fn patches_from_slots( + slots: &[Option], + patch_offset: Option, + patch_offset_within_chunk: Option, + len: usize, +) -> Option { + match (&slots[LP_PATCH_INDICES_SLOT], &slots[LP_PATCH_VALUES_SLOT]) { + (Some(indices), Some(values)) => { + let patch_offset = patch_offset.vortex_expect("ALPRDArray patch slots without offset"); + Some(unsafe { + Patches::new_unchecked( + len, + patch_offset, + indices.clone(), + values.clone(), + slots[LP_PATCH_CHUNK_OFFSETS_SLOT].clone(), + patch_offset_within_chunk, + ) + }) + } + _ => None, + } +} + +fn validate_parts( + dtype: &DType, + len: usize, + left_parts: &ArrayRef, + right_parts: &ArrayRef, + left_parts_patches: Option<&Patches>, +) -> VortexResult<()> { + if !dtype.is_float() { + vortex_bail!("ALPRDArray given invalid DType ({dtype})"); + } + + vortex_ensure!( + left_parts.len() == len, + "left_parts len {} != outer len {len}", + left_parts.len(), + ); + vortex_ensure!( + right_parts.len() == len, + "right_parts len {} != outer len {len}", + right_parts.len(), + ); + + if !left_parts.dtype().is_unsigned_int() { + vortex_bail!("left_parts dtype must be uint"); + } + if dtype.is_nullable() != left_parts.dtype().is_nullable() { + vortex_bail!( + "ALPRDArray dtype nullability ({}) must match left_parts dtype nullability ({})", + dtype, + left_parts.dtype() + ); + } + + let expected_right_parts_dtype = match dtype { + DType::Primitive(PType::F32, _) => DType::Primitive(PType::U32, Nullability::NonNullable), + DType::Primitive(PType::F64, _) => DType::Primitive(PType::U64, Nullability::NonNullable), + _ => vortex_bail!("Expected f32 or f64 dtype, got {:?}", dtype), + }; + vortex_ensure!( + right_parts.dtype() == &expected_right_parts_dtype, + "right_parts dtype {} does not match expected {}", + right_parts.dtype(), + expected_right_parts_dtype, + ); + + if let Some(patches) = left_parts_patches { + vortex_ensure!( + patches.array_len() == len, + "patches array_len {} != outer len {len}", + patches.array_len(), + ); + vortex_ensure!( + patches.dtype().eq_ignore_nullability(left_parts.dtype()), + "patches dtype {} does not match left_parts dtype {}", + patches.dtype(), + left_parts.dtype(), + ); + vortex_ensure!( + patches.values().all_valid()?, + "patches must be all valid: {}", + patches.values() + ); + } + + Ok(()) +} + +pub trait ALPRDArrayExt: TypedArrayRef { + fn left_parts(&self) -> &ArrayRef { + left_parts_from_slots(self.as_ref().slots()) + } + + fn right_parts(&self) -> &ArrayRef { + right_parts_from_slots(self.as_ref().slots()) + } + + fn right_bit_width(&self) -> u8 { + ALPRDData::right_bit_width(self) + } + + fn left_parts_patches(&self) -> Option { + patches_from_slots( + self.as_ref().slots(), + self.patch_offset, + self.patch_offset_within_chunk, + self.as_ref().len(), + ) + } + + fn left_parts_dictionary(&self) -> &Buffer { + ALPRDData::left_parts_dictionary(self) + } +} +impl> ALPRDArrayExt for T {} + +pub trait ALPRDArrayOwnedExt { + fn into_data_parts(self) -> ALPRDDataParts; +} + +impl ALPRDArrayOwnedExt for Array { + fn into_data_parts(self) -> ALPRDDataParts { + let left_parts_patches = self.left_parts_patches(); + let left_parts = self.left_parts().clone(); + let right_parts = self.right_parts().clone(); + let mut parts = ALPRDDataParts { + left_parts, + left_parts_patches: None, + left_parts_dictionary: self.left_parts_dictionary().clone(), + right_parts, }; - self.slots[LP_PATCH_INDICES_SLOT] = pi; - self.slots[LP_PATCH_VALUES_SLOT] = pv; - self.slots[LP_PATCH_CHUNK_OFFSETS_SLOT] = pco; - self.left_parts_patches = patches; + parts.left_parts_patches = left_parts_patches; + parts } } impl ValidityChild for ALPRD { - fn validity_child(array: &ALPRDData) -> &ArrayRef { - array.left_parts() + fn validity_child(array: ArrayView<'_, ALPRD>) -> ArrayRef { + array.left_parts().clone() } } diff --git a/encodings/alp/src/alp_rd/compute/cast.rs b/encodings/alp/src/alp_rd/compute/cast.rs index 2919a22294a..01eab5d126f 100644 --- a/encodings/alp/src/alp_rd/compute/cast.rs +++ b/encodings/alp/src/alp_rd/compute/cast.rs @@ -9,6 +9,7 @@ use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; +use crate::ALPRDArrayExt; use crate::alp_rd::ALPRD; impl CastReduce for ALPRD { diff --git a/encodings/alp/src/alp_rd/compute/filter.rs b/encodings/alp/src/alp_rd/compute/filter.rs index b6bf409d90c..8815a049dc6 100644 --- a/encodings/alp/src/alp_rd/compute/filter.rs +++ b/encodings/alp/src/alp_rd/compute/filter.rs @@ -10,6 +10,7 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::ALPRD; +use crate::ALPRDArrayExt; impl FilterKernel for ALPRD { fn filter( @@ -48,6 +49,7 @@ mod test { use vortex_buffer::buffer; use vortex_mask::Mask; + use crate::ALPRDArrayExt; use crate::ALPRDFloat; use crate::RDEncoder; diff --git a/encodings/alp/src/alp_rd/compute/mask.rs b/encodings/alp/src/alp_rd/compute/mask.rs index 066760c1165..001c44d4825 100644 --- a/encodings/alp/src/alp_rd/compute/mask.rs +++ b/encodings/alp/src/alp_rd/compute/mask.rs @@ -4,13 +4,14 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; -use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnFactoryExt; use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::fns::mask::Mask as MaskExpr; use vortex_array::scalar_fn::fns::mask::MaskReduce; use vortex_error::VortexResult; use crate::ALPRD; +use crate::ALPRDArrayExt; impl MaskReduce for ALPRD { fn mask(array: ArrayView<'_, Self>, mask: &ArrayRef) -> VortexResult> { diff --git a/encodings/alp/src/alp_rd/compute/take.rs b/encodings/alp/src/alp_rd/compute/take.rs index 16e73487d0e..fe12255f063 100644 --- a/encodings/alp/src/alp_rd/compute/take.rs +++ b/encodings/alp/src/alp_rd/compute/take.rs @@ -11,6 +11,7 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::ALPRD; +use crate::ALPRDArrayExt; impl TakeExecute for ALPRD { fn take( @@ -62,6 +63,7 @@ mod test { use vortex_array::assert_arrays_eq; use vortex_array::compute::conformance::take::test_take_conformance; + use crate::ALPRDArrayExt; use crate::ALPRDFloat; use crate::RDEncoder; diff --git a/encodings/alp/src/alp_rd/ops.rs b/encodings/alp/src/alp_rd/ops.rs index 6f77fdbd843..ed4dcba2dfd 100644 --- a/encodings/alp/src/alp_rd/ops.rs +++ b/encodings/alp/src/alp_rd/ops.rs @@ -10,6 +10,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::ALPRD; +use crate::ALPRDArrayExt; impl OperationsVTable for ALPRD { fn scalar_at( @@ -69,6 +70,7 @@ mod test { use vortex_array::assert_arrays_eq; use vortex_array::scalar::Scalar; + use crate::ALPRDArrayExt; use crate::ALPRDFloat; use crate::RDEncoder; diff --git a/encodings/alp/src/alp_rd/slice.rs b/encodings/alp/src/alp_rd/slice.rs index 8f716a0070b..c36e138ff84 100644 --- a/encodings/alp/src/alp_rd/slice.rs +++ b/encodings/alp/src/alp_rd/slice.rs @@ -10,6 +10,7 @@ use vortex_array::IntoArray; use vortex_array::arrays::slice::SliceKernel; use vortex_error::VortexResult; +use crate::ALPRDArrayExt; use crate::alp_rd::ALPRD; impl SliceKernel for ALPRD { diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index f626040d55d..612746bff2a 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -28,15 +28,11 @@ pub type vortex_bytebool::ByteBool::OperationsVTable = vortex_bytebool::ByteBool pub type vortex_bytebool::ByteBool::ValidityVTable = vortex_bytebool::ByteBool -pub fn vortex_bytebool::ByteBool::array_eq(array: &vortex_bytebool::ByteBoolData, other: &vortex_bytebool::ByteBoolData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_bytebool::ByteBool::array_hash(array: &vortex_bytebool::ByteBoolData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_bytebool::ByteBool::buffer(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_bytebool::ByteBool::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_bytebool::ByteBool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_bytebool::ByteBool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_bytebool::ByteBool::execute(array: vortex_array::array::typed::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -52,11 +48,7 @@ pub fn vortex_bytebool::ByteBool::serialize(_array: vortex_array::array::view::A pub fn vortex_bytebool::ByteBool::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_bytebool::ByteBool::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_bytebool::ByteBool::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_bytebool::ByteBool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_bytebool::ByteBool::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_bytebool::ByteBool @@ -100,10 +92,6 @@ pub fn vortex_bytebool::ByteBoolData::new(buffer: vortex_array::buffer::BufferHa pub fn vortex_bytebool::ByteBoolData::validate(buffer: &vortex_array::buffer::BufferHandle, validity: &vortex_array::validity::Validity, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> -pub fn vortex_bytebool::ByteBoolData::validity(&self) -> vortex_array::validity::Validity - -pub fn vortex_bytebool::ByteBoolData::validity_mask(&self) -> vortex_mask::Mask - impl core::clone::Clone for vortex_bytebool::ByteBoolData pub fn vortex_bytebool::ByteBoolData::clone(&self) -> vortex_bytebool::ByteBoolData @@ -120,4 +108,24 @@ impl core::fmt::Debug for vortex_bytebool::ByteBoolData pub fn vortex_bytebool::ByteBoolData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_bytebool::ByteBoolData + +pub fn vortex_bytebool::ByteBoolData::array_eq(&self, other: &Self, precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_bytebool::ByteBoolData + +pub fn vortex_bytebool::ByteBoolData::array_hash(&self, state: &mut H, precision: vortex_array::hash::Precision) + +pub trait vortex_bytebool::ByteBoolArrayExt: vortex_array::array::typed::TypedArrayRef + +pub fn vortex_bytebool::ByteBoolArrayExt::validity(&self) -> vortex_array::validity::Validity + +pub fn vortex_bytebool::ByteBoolArrayExt::validity_mask(&self) -> vortex_mask::Mask + +impl> vortex_bytebool::ByteBoolArrayExt for T + +pub fn T::validity(&self) -> vortex_array::validity::Validity + +pub fn T::validity_mask(&self) -> vortex_mask::Mask + pub type vortex_bytebool::ByteBoolArray = vortex_array::array::typed::Array diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index 44a10177999..ce2661ef082 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::fmt::Debug; +use std::hash::Hasher; use vortex_array::Array; use vortex_array::ArrayEq; @@ -14,10 +15,10 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::TypedArrayRef; use vortex_array::arrays::BoolArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; -use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; use vortex_array::validity::Validity; @@ -40,6 +41,18 @@ use crate::kernel::PARENT_KERNELS; vtable!(ByteBool, ByteBool, ByteBoolData); +impl ArrayHash for ByteBoolData { + fn array_hash(&self, state: &mut H, precision: Precision) { + self.buffer.array_hash(state, precision); + } +} + +impl ArrayEq for ByteBoolData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + self.buffer.array_eq(&other.buffer, precision) + } +} + impl VTable for ByteBool { type ArrayData = ByteBoolData; @@ -50,21 +63,17 @@ impl VTable for ByteBool { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - let validity = data.validity(); + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let validity = child_to_validity(&slots[VALIDITY_SLOT], dtype.nullability()); ByteBoolData::validate(data.buffer(), &validity, dtype, len) } - fn array_hash(array: &ByteBoolData, state: &mut H, precision: Precision) { - array.buffer.array_hash(state, precision); - array.validity().array_hash(state, precision); - } - - fn array_eq(array: &ByteBoolData, other: &ByteBoolData, precision: Precision) -> bool { - array.buffer.array_eq(&other.buffer, precision) - && array.validity().array_eq(&other.validity(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 1 } @@ -95,7 +104,7 @@ impl VTable for ByteBool { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { if !metadata.is_empty() { vortex_bail!( "ByteBoolArray expects empty metadata, got {} bytes", @@ -116,28 +125,15 @@ impl VTable for ByteBool { } let buffer = buffers[0].clone(); - Ok(ByteBoolData::new(buffer, validity)) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let data = ByteBoolData::new(buffer, validity.clone()); + let slots = ByteBoolData::make_slots(&validity, len); + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "ByteBoolArray expects {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn reduce_parent( array: ArrayView<'_, Self>, parent: &ArrayRef, @@ -172,10 +168,23 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; #[derive(Clone, Debug)] pub struct ByteBoolData { buffer: BufferHandle, - nullability: Nullability, - pub(super) slots: Vec>, } +pub trait ByteBoolArrayExt: TypedArrayRef { + fn validity(&self) -> Validity { + child_to_validity( + &self.as_ref().slots()[VALIDITY_SLOT], + self.as_ref().dtype().nullability(), + ) + } + + fn validity_mask(&self) -> Mask { + self.validity().to_mask(self.as_ref().len()) + } +} + +impl> ByteBoolArrayExt for T {} + #[derive(Clone, Debug)] pub struct ByteBool; @@ -184,25 +193,42 @@ impl ByteBool { pub fn new(buffer: BufferHandle, validity: Validity) -> ByteBoolArray { let dtype = DType::Bool(validity.nullability()); + let slots = ByteBoolData::make_slots(&validity, buffer.len()); let data = ByteBoolData::new(buffer, validity); let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(ByteBool, dtype, len, data)) } + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(ByteBool, dtype, len, data).with_slots(slots), + ) + } } /// Construct a [`ByteBoolArray`] from a `Vec` and validity. pub fn from_vec>(data: Vec, validity: V) -> ByteBoolArray { - let data = ByteBoolData::from_vec(data, validity); - let dtype = DType::Bool(data.nullability); + let validity = validity.into(); + let data = ByteBoolData::from_vec(data, validity.clone()); + let dtype = DType::Bool(validity.nullability()); let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(ByteBool, dtype, len, data)) } + let slots = ByteBoolData::make_slots(&validity, len); + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(ByteBool, dtype, len, data).with_slots(slots), + ) + } } /// Construct a [`ByteBoolArray`] from optional bools. pub fn from_option_vec(data: Vec>) -> ByteBoolArray { + let validity = Validity::from_iter(data.iter().map(|v| v.is_some())); let data = ByteBoolData::from(data); - let dtype = DType::Bool(data.nullability); + let dtype = DType::Bool(validity.nullability()); let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(ByteBool, dtype, len, data)) } + let slots = ByteBoolData::make_slots(&validity, len); + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(ByteBool, dtype, len, data).with_slots(slots), + ) + } } } @@ -233,10 +259,6 @@ impl ByteBoolData { vec![validity_to_child(validity, len)] } - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) - } - pub fn new(buffer: BufferHandle, validity: Validity) -> Self { let length = buffer.len(); if let Some(vlen) = validity.maybe_len() @@ -248,12 +270,7 @@ impl ByteBoolData { vlen ); } - let slots = Self::make_slots(&validity, length); - Self { - buffer, - nullability: validity.nullability(), - slots, - } + Self { buffer } } /// Returns the number of elements in the array. @@ -266,11 +283,6 @@ impl ByteBoolData { self.buffer.len() == 0 } - /// Returns the validity mask for this array. - pub fn validity_mask(&self) -> Mask { - self.validity().to_mask(self.len()) - } - // TODO(ngates): deprecate construction from vec pub fn from_vec>(data: Vec, validity: V) -> Self { let validity = validity.into(); @@ -291,7 +303,7 @@ impl ByteBoolData { impl ValidityVTable for ByteBool { fn validity(array: ArrayView<'_, ByteBool>) -> VortexResult { - Ok(array.data().validity()) + Ok(ByteBoolArrayExt::validity(&array)) } } diff --git a/encodings/datetime-parts/public-api.lock b/encodings/datetime-parts/public-api.lock index a06b945b65d..f0418a8b4d1 100644 --- a/encodings/datetime-parts/public-api.lock +++ b/encodings/datetime-parts/public-api.lock @@ -26,15 +26,11 @@ pub type vortex_datetime_parts::DateTimeParts::OperationsVTable = vortex_datetim pub type vortex_datetime_parts::DateTimeParts::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild -pub fn vortex_datetime_parts::DateTimeParts::array_eq(array: &vortex_datetime_parts::DateTimePartsData, other: &vortex_datetime_parts::DateTimePartsData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_datetime_parts::DateTimeParts::array_hash(array: &vortex_datetime_parts::DateTimePartsData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_datetime_parts::DateTimeParts::buffer(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_datetime_parts::DateTimeParts::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_datetime_parts::DateTimeParts::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_datetime_parts::DateTimeParts::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_datetime_parts::DateTimeParts::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -50,11 +46,7 @@ pub fn vortex_datetime_parts::DateTimeParts::serialize(array: vortex_array::arra pub fn vortex_datetime_parts::DateTimeParts::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_datetime_parts::DateTimeParts::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_datetime_parts::DateTimeParts::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_datetime_parts::DateTimeParts::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_datetime_parts::DateTimeParts::validate(&self, _data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_datetime_parts::DateTimeParts @@ -62,7 +54,7 @@ pub fn vortex_datetime_parts::DateTimeParts::scalar_at(array: vortex_array::arra impl vortex_array::array::vtable::validity::ValidityChild for vortex_datetime_parts::DateTimeParts -pub fn vortex_datetime_parts::DateTimeParts::validity_child(array: &vortex_datetime_parts::DateTimePartsData) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_datetime_parts::DateTimeParts::validity_child(array: vortex_array::array::view::ArrayView<'_, vortex_datetime_parts::DateTimeParts>) -> vortex_array::array::erased::ArrayRef impl vortex_array::arrays::dict::take::TakeExecute for vortex_datetime_parts::DateTimeParts @@ -92,16 +84,6 @@ pub struct vortex_datetime_parts::DateTimePartsData impl vortex_datetime_parts::DateTimePartsData -pub fn vortex_datetime_parts::DateTimePartsData::days(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn vortex_datetime_parts::DateTimePartsData::is_empty(&self) -> bool - -pub fn vortex_datetime_parts::DateTimePartsData::len(&self) -> usize - -pub fn vortex_datetime_parts::DateTimePartsData::seconds(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn vortex_datetime_parts::DateTimePartsData::subseconds(&self) -> &vortex_array::array::erased::ArrayRef - pub fn vortex_datetime_parts::DateTimePartsData::validate(dtype: &vortex_array::dtype::DType, days: &vortex_array::array::erased::ArrayRef, seconds: &vortex_array::array::erased::ArrayRef, subseconds: &vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_datetime_parts::DateTimePartsData @@ -118,6 +100,14 @@ impl core::fmt::Debug for vortex_datetime_parts::DateTimePartsData pub fn vortex_datetime_parts::DateTimePartsData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_datetime_parts::DateTimePartsData + +pub fn vortex_datetime_parts::DateTimePartsData::array_eq(&self, _other: &Self, _precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_datetime_parts::DateTimePartsData + +pub fn vortex_datetime_parts::DateTimePartsData::array_hash(&self, _state: &mut H, _precision: vortex_array::hash::Precision) + #[repr(C)] pub struct vortex_datetime_parts::DateTimePartsMetadata pub vortex_datetime_parts::DateTimePartsMetadata::days_ptype: i32 @@ -174,6 +164,22 @@ pub vortex_datetime_parts::TemporalParts::seconds: vortex_array::array::erased:: pub vortex_datetime_parts::TemporalParts::subseconds: vortex_array::array::erased::ArrayRef +pub trait vortex_datetime_parts::DateTimePartsArrayExt: vortex_array::array::typed::TypedArrayRef + +pub fn vortex_datetime_parts::DateTimePartsArrayExt::days(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn vortex_datetime_parts::DateTimePartsArrayExt::seconds(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn vortex_datetime_parts::DateTimePartsArrayExt::subseconds(&self) -> &vortex_array::array::erased::ArrayRef + +impl> vortex_datetime_parts::DateTimePartsArrayExt for T + +pub fn T::days(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn T::seconds(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn T::subseconds(&self) -> &vortex_array::array::erased::ArrayRef + pub fn vortex_datetime_parts::initialize(session: &vortex_session::VortexSession) pub fn vortex_datetime_parts::split_temporal(array: vortex_array::arrays::datetime::TemporalArray) -> vortex_error::VortexResult diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index f59cec3dd11..c3053dcac23 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::fmt::Debug; +use std::hash::Hasher; use prost::Message; use vortex_array::Array; @@ -15,6 +16,7 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::TypedArrayRef; use vortex_array::arrays::TemporalArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; @@ -33,12 +35,24 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::TemporalParts; use crate::canonical::decode_to_temporal; use crate::compute::kernel::PARENT_KERNELS; use crate::compute::rules::PARENT_RULES; +use crate::split_temporal; vtable!(DateTimeParts, DateTimeParts, DateTimePartsData); +impl ArrayHash for DateTimePartsData { + fn array_hash(&self, _state: &mut H, _precision: Precision) {} +} + +impl ArrayEq for DateTimePartsData { + fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { + true + } +} + #[derive(Clone, prost::Message)] #[repr(C)] pub struct DateTimePartsMetadata { @@ -79,28 +93,23 @@ impl VTable for DateTimeParts { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - DateTimePartsData::validate(dtype, data.days(), data.seconds(), data.subseconds(), len) - } - - fn array_hash( - array: &DateTimePartsData, - state: &mut H, - precision: Precision, - ) { - array.days().array_hash(state, precision); - array.seconds().array_hash(state, precision); - array.subseconds().array_hash(state, precision); - } - - fn array_eq( - array: &DateTimePartsData, - other: &DateTimePartsData, - precision: Precision, - ) -> bool { - array.days().array_eq(other.days(), precision) - && array.seconds().array_eq(other.seconds(), precision) - && array.subseconds().array_eq(other.subseconds(), precision) + fn validate( + &self, + _data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let days = slots[DAYS_SLOT] + .as_ref() + .vortex_expect("DateTimePartsArray days slot"); + let seconds = slots[SECONDS_SLOT] + .as_ref() + .vortex_expect("DateTimePartsArray seconds slot"); + let subseconds = slots[SUBSECONDS_SLOT] + .as_ref() + .vortex_expect("DateTimePartsArray subseconds slot"); + DateTimePartsData::validate(dtype, days, seconds, subseconds, len) } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -134,7 +143,7 @@ impl VTable for DateTimeParts { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = DateTimePartsMetadata::decode(metadata)?; if children.len() != 3 { vortex_bail!( @@ -159,28 +168,15 @@ impl VTable for DateTimeParts { len, )?; - DateTimePartsData::try_new(dtype.clone(), days, seconds, subseconds) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let slots = vec![Some(days), Some(seconds), Some(subseconds)]; + let data = DateTimePartsData {}; + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "DateTimePartsArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done( decode_to_temporal(&array, ctx)?.into_array(), @@ -215,10 +211,30 @@ pub(super) const NUM_SLOTS: usize = 3; pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["days", "seconds", "subseconds"]; #[derive(Clone, Debug)] -pub struct DateTimePartsData { - pub(super) slots: Vec>, +pub struct DateTimePartsData {} + +pub trait DateTimePartsArrayExt: TypedArrayRef { + fn days(&self) -> &ArrayRef { + self.as_ref().slots()[DAYS_SLOT] + .as_ref() + .vortex_expect("DateTimePartsArray days slot") + } + + fn seconds(&self) -> &ArrayRef { + self.as_ref().slots()[SECONDS_SLOT] + .as_ref() + .vortex_expect("DateTimePartsArray seconds slot") + } + + fn subseconds(&self) -> &ArrayRef { + self.as_ref().slots()[SUBSECONDS_SLOT] + .as_ref() + .vortex_expect("DateTimePartsArray subseconds slot") + } } +impl> DateTimePartsArrayExt for T {} + #[derive(Clone, Debug)] pub struct DateTimeParts; @@ -232,25 +248,26 @@ impl DateTimeParts { seconds: ArrayRef, subseconds: ArrayRef, ) -> VortexResult { - let data = DateTimePartsData::try_new(dtype.clone(), days, seconds, subseconds)?; - let len = data.len(); - Ok( - unsafe { - Array::from_parts_unchecked(ArrayParts::new(DateTimeParts, dtype, len, data)) - }, - ) + let len = days.len(); + DateTimePartsData::validate(&dtype, &days, &seconds, &subseconds, len)?; + let slots = vec![Some(days), Some(seconds), Some(subseconds)]; + let data = DateTimePartsData {}; + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new(DateTimeParts, dtype, len, data).with_slots(slots), + ) + }) } /// Construct a [`DateTimePartsArray`] from a [`TemporalArray`]. pub fn try_from_temporal(temporal: TemporalArray) -> VortexResult { let dtype = temporal.dtype().clone(); - let data = DateTimePartsData::try_from(temporal)?; - let len = data.len(); - Ok( - unsafe { - Array::from_parts_unchecked(ArrayParts::new(DateTimeParts, dtype, len, data)) - }, - ) + let TemporalParts { + days, + seconds, + subseconds, + } = split_temporal(temporal)?; + Self::try_new(dtype, days, seconds, subseconds) } } @@ -289,50 +306,10 @@ impl DateTimePartsData { Ok(()) } - - pub(crate) fn try_new( - dtype: DType, - days: ArrayRef, - seconds: ArrayRef, - subseconds: ArrayRef, - ) -> VortexResult { - Self::validate(&dtype, &days, &seconds, &subseconds, days.len())?; - Ok(Self { - slots: vec![Some(days), Some(seconds), Some(subseconds)], - }) - } - - /// Returns the number of elements in the array. - pub fn len(&self) -> usize { - self.days().len() - } - - /// Returns `true` if the array contains no elements. - pub fn is_empty(&self) -> bool { - self.days().len() == 0 - } - - pub fn days(&self) -> &ArrayRef { - self.slots[DAYS_SLOT] - .as_ref() - .vortex_expect("DateTimePartsArray days slot") - } - - pub fn seconds(&self) -> &ArrayRef { - self.slots[SECONDS_SLOT] - .as_ref() - .vortex_expect("DateTimePartsArray seconds slot") - } - - pub fn subseconds(&self) -> &ArrayRef { - self.slots[SUBSECONDS_SLOT] - .as_ref() - .vortex_expect("DateTimePartsArray subseconds slot") - } } impl ValidityChild for DateTimeParts { - fn validity_child(array: &DateTimePartsData) -> &ArrayRef { - array.days() + fn validity_child(array: ArrayView<'_, DateTimeParts>) -> ArrayRef { + array.days().clone() } } diff --git a/encodings/datetime-parts/src/canonical.rs b/encodings/datetime-parts/src/canonical.rs index c5b969278c2..f1d1aaf6914 100644 --- a/encodings/datetime-parts/src/canonical.rs +++ b/encodings/datetime-parts/src/canonical.rs @@ -19,6 +19,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_panic; use crate::DateTimePartsArray; +use crate::array::DateTimePartsArrayExt; /// Decode an [Array] into a [TemporalArray]. /// diff --git a/encodings/datetime-parts/src/compress.rs b/encodings/datetime-parts/src/compress.rs index 6be0d24f3f1..2edb6097c7b 100644 --- a/encodings/datetime-parts/src/compress.rs +++ b/encodings/datetime-parts/src/compress.rs @@ -67,7 +67,14 @@ impl TryFrom for DateTimePartsData { seconds, subseconds, } = split_temporal(array)?; - DateTimePartsData::try_new(DType::Extension(ext_dtype), days, seconds, subseconds) + DateTimePartsData::validate( + &DType::Extension(ext_dtype), + &days, + &seconds, + &subseconds, + days.len(), + )?; + Ok(DateTimePartsData {}) } } diff --git a/encodings/datetime-parts/src/compute/cast.rs b/encodings/datetime-parts/src/compute/cast.rs index ca846bbd424..0e56d594dcb 100644 --- a/encodings/datetime-parts/src/compute/cast.rs +++ b/encodings/datetime-parts/src/compute/cast.rs @@ -10,6 +10,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::DateTimeParts; +use crate::array::DateTimePartsArrayExt; impl CastReduce for DateTimeParts { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { if !array.dtype().eq_ignore_nullability(dtype) { diff --git a/encodings/datetime-parts/src/compute/compare.rs b/encodings/datetime-parts/src/compute/compare.rs index 76250bd320e..bd50a7a5d0f 100644 --- a/encodings/datetime-parts/src/compute/compare.rs +++ b/encodings/datetime-parts/src/compute/compare.rs @@ -17,6 +17,7 @@ use vortex_array::scalar_fn::fns::operators::Operator; use vortex_error::VortexResult; use crate::array::DateTimeParts; +use crate::array::DateTimePartsArrayExt; use crate::timestamp; impl CompareKernel for DateTimeParts { diff --git a/encodings/datetime-parts/src/compute/filter.rs b/encodings/datetime-parts/src/compute/filter.rs index da825d8a95e..d717d3d3500 100644 --- a/encodings/datetime-parts/src/compute/filter.rs +++ b/encodings/datetime-parts/src/compute/filter.rs @@ -9,6 +9,7 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::DateTimeParts; +use crate::array::DateTimePartsArrayExt; impl FilterReduce for DateTimeParts { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { Ok(Some( diff --git a/encodings/datetime-parts/src/compute/is_constant.rs b/encodings/datetime-parts/src/compute/is_constant.rs index 5c38c3469a1..94e6806e785 100644 --- a/encodings/datetime-parts/src/compute/is_constant.rs +++ b/encodings/datetime-parts/src/compute/is_constant.rs @@ -11,6 +11,7 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::DateTimeParts; +use crate::array::DateTimePartsArrayExt; /// DateTimeParts-specific is_constant kernel. /// diff --git a/encodings/datetime-parts/src/compute/mask.rs b/encodings/datetime-parts/src/compute/mask.rs index 290707ef8ef..de737105ffe 100644 --- a/encodings/datetime-parts/src/compute/mask.rs +++ b/encodings/datetime-parts/src/compute/mask.rs @@ -9,6 +9,7 @@ use vortex_array::scalar_fn::fns::mask::MaskReduce; use vortex_error::VortexResult; use crate::DateTimeParts; +use crate::array::DateTimePartsArrayExt; impl MaskReduce for DateTimeParts { fn mask(array: ArrayView<'_, Self>, mask: &ArrayRef) -> VortexResult> { diff --git a/encodings/datetime-parts/src/compute/rules.rs b/encodings/datetime-parts/src/compute/rules.rs index 92c1f877c08..c18d959cd7e 100644 --- a/encodings/datetime-parts/src/compute/rules.rs +++ b/encodings/datetime-parts/src/compute/rules.rs @@ -10,6 +10,7 @@ use vortex_array::arrays::Filter; use vortex_array::arrays::ScalarFnArray; use vortex_array::arrays::filter::FilterReduceAdaptor; use vortex_array::arrays::scalar_fn::AnyScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_array::arrays::scalar_fn::ScalarFnVTable; use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::builtins::ArrayBuiltins; @@ -26,6 +27,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DateTimeParts; +use crate::array::DateTimePartsArrayExt; use crate::timestamp; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&DTPFilterPushDownRule), @@ -179,7 +181,7 @@ fn is_constant_zero(array: &ArrayRef) -> bool { mod tests { use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; - use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; + use vortex_array::arrays::scalar_fn::ScalarFnFactoryExt; use vortex_array::extension::datetime::TimeUnit; use vortex_array::extension::datetime::TimestampOptions; use vortex_array::optimizer::ArrayOptimizer; diff --git a/encodings/datetime-parts/src/compute/slice.rs b/encodings/datetime-parts/src/compute/slice.rs index 5bad980d8dc..46819a41562 100644 --- a/encodings/datetime-parts/src/compute/slice.rs +++ b/encodings/datetime-parts/src/compute/slice.rs @@ -10,6 +10,7 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::DateTimeParts; +use crate::array::DateTimePartsArrayExt; impl SliceReduce for DateTimeParts { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { diff --git a/encodings/datetime-parts/src/compute/take.rs b/encodings/datetime-parts/src/compute/take.rs index 4cd214663c1..cd15c466b2a 100644 --- a/encodings/datetime-parts/src/compute/take.rs +++ b/encodings/datetime-parts/src/compute/take.rs @@ -16,6 +16,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_panic; use crate::DateTimeParts; +use crate::array::DateTimePartsArrayExt; fn take_datetime_parts( array: ArrayView, indices: &ArrayRef, diff --git a/encodings/datetime-parts/src/ops.rs b/encodings/datetime-parts/src/ops.rs index c0acf4d6a32..2c224b078d5 100644 --- a/encodings/datetime-parts/src/ops.rs +++ b/encodings/datetime-parts/src/ops.rs @@ -12,6 +12,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_panic; use crate::DateTimeParts; +use crate::array::DateTimePartsArrayExt; use crate::timestamp; use crate::timestamp::TimestampParts; diff --git a/encodings/decimal-byte-parts/public-api.lock b/encodings/decimal-byte-parts/public-api.lock index 268967616c6..0df4b476b17 100644 --- a/encodings/decimal-byte-parts/public-api.lock +++ b/encodings/decimal-byte-parts/public-api.lock @@ -24,15 +24,11 @@ pub type vortex_decimal_byte_parts::DecimalByteParts::OperationsVTable = vortex_ pub type vortex_decimal_byte_parts::DecimalByteParts::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild -pub fn vortex_decimal_byte_parts::DecimalByteParts::array_eq(array: &vortex_decimal_byte_parts::DecimalBytePartsData, other: &vortex_decimal_byte_parts::DecimalBytePartsData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_decimal_byte_parts::DecimalByteParts::array_hash(array: &vortex_decimal_byte_parts::DecimalBytePartsData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_decimal_byte_parts::DecimalByteParts::buffer(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_decimal_byte_parts::DecimalByteParts::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_decimal_byte_parts::DecimalByteParts::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_decimal_byte_parts::DecimalByteParts::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_decimal_byte_parts::DecimalByteParts::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -48,11 +44,7 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::serialize(array: vortex_arra pub fn vortex_decimal_byte_parts::DecimalByteParts::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_decimal_byte_parts::DecimalByteParts::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_decimal_byte_parts::DecimalByteParts::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_decimal_byte_parts::DecimalByteParts::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_decimal_byte_parts::DecimalByteParts::validate(&self, _data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_decimal_byte_parts::DecimalByteParts @@ -60,7 +52,7 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::scalar_at(array: vortex_arra impl vortex_array::array::vtable::validity::ValidityChild for vortex_decimal_byte_parts::DecimalByteParts -pub fn vortex_decimal_byte_parts::DecimalByteParts::validity_child(array: &vortex_decimal_byte_parts::DecimalBytePartsData) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_decimal_byte_parts::DecimalByteParts::validity_child(array: vortex_array::array::view::ArrayView<'_, vortex_decimal_byte_parts::DecimalByteParts>) -> vortex_array::array::erased::ArrayRef impl vortex_array::arrays::dict::take::TakeExecute for vortex_decimal_byte_parts::DecimalByteParts @@ -90,12 +82,6 @@ pub struct vortex_decimal_byte_parts::DecimalBytePartsData impl vortex_decimal_byte_parts::DecimalBytePartsData -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::into_parts(self) -> vortex_decimal_byte_parts::DecimalBytePartsDataParts - -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::is_empty(&self) -> bool - -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::len(&self) -> usize - pub fn vortex_decimal_byte_parts::DecimalBytePartsData::validate(msp: &vortex_array::array::erased::ArrayRef, decimal_dtype: vortex_array::dtype::decimal::DecimalDType, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_decimal_byte_parts::DecimalBytePartsData @@ -106,6 +92,14 @@ impl core::fmt::Debug for vortex_decimal_byte_parts::DecimalBytePartsData pub fn vortex_decimal_byte_parts::DecimalBytePartsData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_decimal_byte_parts::DecimalBytePartsData + +pub fn vortex_decimal_byte_parts::DecimalBytePartsData::array_eq(&self, _other: &Self, _precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_decimal_byte_parts::DecimalBytePartsData + +pub fn vortex_decimal_byte_parts::DecimalBytePartsData::array_hash(&self, _state: &mut H, _precision: vortex_array::hash::Precision) + pub struct vortex_decimal_byte_parts::DecimalBytePartsDataParts pub vortex_decimal_byte_parts::DecimalBytePartsDataParts::msp: vortex_array::array::erased::ArrayRef @@ -136,6 +130,14 @@ pub fn vortex_decimal_byte_parts::DecimalBytesPartsMetadata::clear(&mut self) pub fn vortex_decimal_byte_parts::DecimalBytesPartsMetadata::encoded_len(&self) -> usize +pub trait vortex_decimal_byte_parts::DecimalBytePartsArrayExt: vortex_array::array::typed::TypedArrayRef + +pub fn vortex_decimal_byte_parts::DecimalBytePartsArrayExt::msp(&self) -> &vortex_array::array::erased::ArrayRef + +impl> vortex_decimal_byte_parts::DecimalBytePartsArrayExt for T + +pub fn T::msp(&self) -> &vortex_array::array::erased::ArrayRef + pub fn vortex_decimal_byte_parts::initialize(session: &vortex_session::VortexSession) pub type vortex_decimal_byte_parts::DecimalBytePartsArray = vortex_array::array::typed::Array diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs index d78c6e2a96d..e554bbfae71 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs @@ -11,6 +11,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DecimalByteParts; +use crate::decimal_byte_parts::DecimalBytePartsArrayExt; impl CastReduce for DecimalByteParts { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // DecimalBytePartsArray can only have Decimal dtype, so we only handle decimal-to-decimal casts diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs index b0afad10cfa..43f06b6e695 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs @@ -25,6 +25,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DecimalByteParts; +use crate::decimal_byte_parts::DecimalBytePartsArrayExt; use crate::decimal_byte_parts::compute::compare::Sign::Positive; impl CompareKernel for DecimalByteParts { diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs index 49435845476..c82171b0e13 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs @@ -10,6 +10,7 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::DecimalByteParts; +use crate::decimal_byte_parts::DecimalBytePartsArrayExt; impl FilterReduce for DecimalByteParts { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { DecimalByteParts::try_new( diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/is_constant.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/is_constant.rs index bca11eda747..4d271eb80b3 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/is_constant.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/is_constant.rs @@ -11,6 +11,7 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::DecimalByteParts; +use crate::decimal_byte_parts::DecimalBytePartsArrayExt; /// DecimalByteParts-specific is_constant kernel. /// diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs index 4c1b1a36c64..0f007519894 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs @@ -4,7 +4,7 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; -use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnFactoryExt; use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::fns::mask::Mask as MaskExpr; use vortex_array::scalar_fn::fns::mask::MaskReduce; @@ -12,6 +12,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DecimalByteParts; +use crate::decimal_byte_parts::DecimalBytePartsArrayExt; impl MaskReduce for DecimalByteParts { fn mask(array: ArrayView<'_, Self>, mask: &ArrayRef) -> VortexResult> { diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs index 3c7f7ecfabf..807b660ec73 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs @@ -10,6 +10,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DecimalByteParts; +use crate::decimal_byte_parts::DecimalBytePartsArrayExt; impl TakeExecute for DecimalByteParts { fn take( diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index 6623e554f36..e3fbdb4f479 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::hash::Hasher; + use vortex_array::Array; use vortex_array::ArrayParts; use vortex_array::ArrayView; @@ -17,6 +19,7 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::TypedArrayRef; use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; @@ -45,6 +48,16 @@ use crate::decimal_byte_parts::rules::PARENT_RULES; vtable!(DecimalByteParts, DecimalByteParts, DecimalBytePartsData); +impl ArrayHash for DecimalBytePartsData { + fn array_hash(&self, _state: &mut H, _precision: Precision) {} +} + +impl ArrayEq for DecimalBytePartsData { + fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { + true + } +} + #[derive(Clone, prost::Message)] pub struct DecimalBytesPartsMetadata { #[prost(enumeration = "PType", tag = "1")] @@ -63,27 +76,20 @@ impl VTable for DecimalByteParts { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + _data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { let Some(decimal_dtype) = dtype.as_decimal_opt() else { vortex_bail!("expected decimal dtype, got {}", dtype) }; - DecimalBytePartsData::validate(data.msp(), *decimal_dtype, dtype, len) - } - - fn array_hash( - array: &DecimalBytePartsData, - state: &mut H, - precision: Precision, - ) { - array.msp().array_hash(state, precision); - } - - fn array_eq( - array: &DecimalBytePartsData, - other: &DecimalBytePartsData, - precision: Precision, - ) -> bool { - array.msp().array_eq(other.msp(), precision) + let msp = slots[MSP_SLOT] + .as_ref() + .vortex_expect("DecimalBytePartsArray msp slot"); + DecimalBytePartsData::validate(msp, *decimal_dtype, dtype, len) } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -116,7 +122,7 @@ impl VTable for DecimalByteParts { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = DecimalBytesPartsMetadata::decode(metadata)?; let Some(decimal_dtype) = dtype.as_decimal_opt() else { vortex_bail!("decoding decimal but given non decimal dtype {}", dtype) @@ -131,28 +137,15 @@ impl VTable for DecimalByteParts { "lower_part_count > 0 not currently supported" ); - DecimalBytePartsData::try_new(msp, *decimal_dtype) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let slots = vec![Some(msp.clone())]; + let data = DecimalBytePartsData::try_new(msp.dtype(), msp.len(), *decimal_dtype)?; + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "DecimalBytePartsArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn reduce_parent( array: ArrayView<'_, Self>, parent: &ArrayRef, @@ -187,7 +180,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["msp"]; /// e.g. for a decimal i128 \[ 127..64 | 64..0 \] msp = 127..64 and lower_part\[0\] = 64..0 #[derive(Clone, Debug)] pub struct DecimalBytePartsData { - pub(super) slots: Vec>, // NOTE: the lower_parts is currently unused, we reserve this field so that it is properly // read/written during serde, but provide no constructor to initialize this to anything // other than the empty Vec. @@ -198,6 +190,16 @@ pub struct DecimalBytePartsDataParts { pub msp: ArrayRef, } +pub trait DecimalBytePartsArrayExt: TypedArrayRef { + fn msp(&self) -> &ArrayRef { + self.as_ref().slots()[MSP_SLOT] + .as_ref() + .vortex_expect("DecimalBytePartsArray msp slot") + } +} + +impl> DecimalBytePartsArrayExt for T {} + impl DecimalBytePartsData { pub fn validate( msp: &ArrayRef, @@ -218,38 +220,22 @@ impl DecimalBytePartsData { Ok(()) } - pub(crate) fn try_new(msp: ArrayRef, decimal_dtype: DecimalDType) -> VortexResult { - let dtype = DType::Decimal(decimal_dtype, msp.dtype().nullability()); - Self::validate(&msp, decimal_dtype, &dtype, msp.len())?; + pub(crate) fn try_new( + msp_dtype: &DType, + msp_len: usize, + decimal_dtype: DecimalDType, + ) -> VortexResult { + let expected_dtype = DType::Decimal(decimal_dtype, msp_dtype.nullability()); + vortex_ensure!( + msp_dtype.is_signed_int(), + "decimal bytes parts, first part must be a signed array" + ); + let _ = msp_len; + drop(expected_dtype); Ok(Self { - slots: vec![Some(msp)], _lower_parts: Vec::new(), }) } - - /// Returns the number of elements in the array. - pub fn len(&self) -> usize { - self.msp().len() - } - - /// Returns `true` if the array contains no elements. - pub fn is_empty(&self) -> bool { - self.msp().len() == 0 - } - - pub(crate) fn msp(&self) -> &ArrayRef { - self.slots[MSP_SLOT] - .as_ref() - .vortex_expect("DecimalBytePartsArray msp slot") - } - - pub fn into_parts(mut self) -> DecimalBytePartsDataParts { - DecimalBytePartsDataParts { - msp: self.slots[MSP_SLOT] - .take() - .vortex_expect("DecimalBytePartsArray msp slot"), - } - } } #[derive(Clone, Debug)] @@ -263,11 +249,14 @@ impl DecimalByteParts { msp: ArrayRef, decimal_dtype: DecimalDType, ) -> VortexResult { - let data = DecimalBytePartsData::try_new(msp, decimal_dtype)?; - let dtype = DType::Decimal(decimal_dtype, data.msp().dtype().nullability()); - let len = data.len(); + let len = msp.len(); + let dtype = DType::Decimal(decimal_dtype, msp.dtype().nullability()); + let slots = vec![Some(msp.clone())]; + let data = DecimalBytePartsData::try_new(msp.dtype(), msp.len(), decimal_dtype)?; Ok(unsafe { - Array::from_parts_unchecked(ArrayParts::new(DecimalByteParts, dtype, len, data)) + Array::from_parts_unchecked( + ArrayParts::new(DecimalByteParts, dtype, len, data).with_slots(slots), + ) }) } } @@ -320,9 +309,9 @@ impl OperationsVTable for DecimalByteParts { } impl ValidityChild for DecimalByteParts { - fn validity_child(array: &DecimalBytePartsData) -> &ArrayRef { + fn validity_child(array: ArrayView<'_, DecimalByteParts>) -> ArrayRef { // validity stored in 0th child - array.msp() + array.msp().clone() } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs index 0eeaeed5716..7fafa0d5289 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs @@ -15,6 +15,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DecimalByteParts; +use crate::decimal_byte_parts::DecimalBytePartsArrayExt; pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&DecimalBytePartsFilterPushDownRule), diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs index 89275570a53..a9c022d2051 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs @@ -11,6 +11,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DecimalByteParts; +use crate::decimal_byte_parts::DecimalBytePartsArrayExt; impl SliceReduce for DecimalByteParts { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { diff --git a/encodings/fastlanes/benches/bitpacking_take.rs b/encodings/fastlanes/benches/bitpacking_take.rs index 6129f479d47..61bd234c0f1 100644 --- a/encodings/fastlanes/benches/bitpacking_take.rs +++ b/encodings/fastlanes/benches/bitpacking_take.rs @@ -17,6 +17,7 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::buffer; +use vortex_fastlanes::BitPackedArrayExt; use vortex_fastlanes::bitpack_compress::bitpack_to_best_bit_width; fn main() { @@ -160,9 +161,9 @@ fn patched_take_10_stratified(bencher: Bencher) { let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); - assert!(packed.patches(packed.len()).is_some()); + assert!(packed.patches().is_some()); assert_eq!( - packed.patches(packed.len()).unwrap().num_patches(), + packed.patches().unwrap().num_patches(), NUM_EXCEPTIONS as usize ); @@ -185,9 +186,9 @@ fn patched_take_10_contiguous(bencher: Bencher) { let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); - assert!(packed.patches(packed.len()).is_some()); + assert!(packed.patches().is_some()); assert_eq!( - packed.patches(packed.len()).unwrap().num_patches(), + packed.patches().unwrap().num_patches(), NUM_EXCEPTIONS as usize ); @@ -249,9 +250,9 @@ fn patched_take_10k_contiguous_patches(bencher: Bencher) { let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); - assert!(packed.patches(packed.len()).is_some()); + assert!(packed.patches().is_some()); assert_eq!( - packed.patches(packed.len()).unwrap().num_patches(), + packed.patches().unwrap().num_patches(), NUM_EXCEPTIONS as usize ); diff --git a/encodings/fastlanes/benches/compute_between.rs b/encodings/fastlanes/benches/compute_between.rs index 799e6d081fc..440bcafddf8 100644 --- a/encodings/fastlanes/benches/compute_between.rs +++ b/encodings/fastlanes/benches/compute_between.rs @@ -6,6 +6,7 @@ use num_traits::NumCast; use rand::RngExt; use rand::rngs::StdRng; use vortex_alp::ALP; +use vortex_alp::ALPArrayExt; use vortex_alp::alp_encode; use vortex_array::ArrayRef; use vortex_array::IntoArray; diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index 98354d33f0b..e8146076954 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -146,15 +146,11 @@ pub type vortex_fastlanes::BitPacked::ValidityVTable = vortex_fastlanes::BitPack pub fn vortex_fastlanes::BitPacked::append_to_builder(array: vortex_array::array::view::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_fastlanes::BitPacked::array_eq(array: &vortex_fastlanes::BitPackedData, other: &vortex_fastlanes::BitPackedData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_fastlanes::BitPacked::array_hash(array: &vortex_fastlanes::BitPackedData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_fastlanes::BitPacked::buffer(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_fastlanes::BitPacked::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_fastlanes::BitPacked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_fastlanes::BitPacked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_fastlanes::BitPacked::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -170,11 +166,7 @@ pub fn vortex_fastlanes::BitPacked::serialize(array: vortex_array::array::view:: pub fn vortex_fastlanes::BitPacked::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_fastlanes::BitPacked::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_fastlanes::BitPacked::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_fastlanes::BitPacked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_fastlanes::BitPacked::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_fastlanes::BitPacked @@ -208,8 +200,6 @@ pub fn vortex_fastlanes::BitPackedData::bit_width(&self) -> u8 pub fn vortex_fastlanes::BitPackedData::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8) -> vortex_error::VortexResult -pub fn vortex_fastlanes::BitPackedData::into_parts(self, len: usize, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_fastlanes::BitPackedDataParts - pub fn vortex_fastlanes::BitPackedData::max_packed_value(&self) -> usize pub fn vortex_fastlanes::BitPackedData::offset(&self) -> u16 @@ -218,28 +208,12 @@ pub fn vortex_fastlanes::BitPackedData::packed(&self) -> &vortex_array::buffer:: pub fn vortex_fastlanes::BitPackedData::packed_slice(&self) -> &[T] -pub fn vortex_fastlanes::BitPackedData::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_fastlanes::BitPackedData::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_fastlanes::BitPackedData::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_fastlanes::BitPackedData::patches(&self, len: usize) -> core::option::Option - pub fn vortex_fastlanes::BitPackedData::ptype(&self, dtype: &vortex_array::dtype::DType) -> vortex_array::dtype::ptype::PType -pub fn vortex_fastlanes::BitPackedData::replace_patches(&mut self, patches: core::option::Option) - -pub fn vortex_fastlanes::BitPackedData::try_new(packed: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::ptype::PType, validity: vortex_array::validity::Validity, patches: core::option::Option, bit_width: u8, length: usize, offset: u16) -> vortex_error::VortexResult +pub fn vortex_fastlanes::BitPackedData::try_new(packed: vortex_array::buffer::BufferHandle, patches: core::option::Option, bit_width: u8, offset: u16) -> vortex_error::VortexResult pub fn vortex_fastlanes::BitPackedData::unpacked_chunks(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::BitPackedData::validity(&self, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_array::validity::Validity - -pub fn vortex_fastlanes::BitPackedData::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_fastlanes::BitPackedData::validity_mask(&self, len: usize, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_mask::Mask - impl core::clone::Clone for vortex_fastlanes::BitPackedData pub fn vortex_fastlanes::BitPackedData::clone(&self) -> vortex_fastlanes::BitPackedData @@ -248,6 +222,14 @@ impl core::fmt::Debug for vortex_fastlanes::BitPackedData pub fn vortex_fastlanes::BitPackedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_fastlanes::BitPackedData + +pub fn vortex_fastlanes::BitPackedData::array_eq(&self, other: &Self, precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_fastlanes::BitPackedData + +pub fn vortex_fastlanes::BitPackedData::array_hash(&self, state: &mut H, precision: vortex_array::hash::Precision) + pub struct vortex_fastlanes::BitPackedDataParts pub vortex_fastlanes::BitPackedDataParts::bit_width: u8 @@ -288,15 +270,11 @@ pub type vortex_fastlanes::Delta::OperationsVTable = vortex_fastlanes::Delta pub type vortex_fastlanes::Delta::ValidityVTable = vortex_fastlanes::Delta -pub fn vortex_fastlanes::Delta::array_eq(array: &vortex_fastlanes::DeltaData, other: &vortex_fastlanes::DeltaData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_fastlanes::Delta::array_hash(array: &vortex_fastlanes::DeltaData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_fastlanes::Delta::buffer(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_fastlanes::Delta::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_fastlanes::Delta::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_fastlanes::Delta::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_fastlanes::Delta::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -310,11 +288,7 @@ pub fn vortex_fastlanes::Delta::serialize(array: vortex_array::array::view::Arra pub fn vortex_fastlanes::Delta::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_fastlanes::Delta::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_fastlanes::Delta::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_fastlanes::Delta::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_fastlanes::Delta::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_fastlanes::Delta @@ -336,13 +310,7 @@ pub struct vortex_fastlanes::DeltaData impl vortex_fastlanes::DeltaData -pub fn vortex_fastlanes::DeltaData::bases(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn vortex_fastlanes::DeltaData::deltas(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn vortex_fastlanes::DeltaData::offset(&self) -> usize - -pub fn vortex_fastlanes::DeltaData::try_new(bases: vortex_array::array::erased::ArrayRef, deltas: vortex_array::array::erased::ArrayRef, offset: usize, len: usize) -> vortex_error::VortexResult +pub fn vortex_fastlanes::DeltaData::try_new(offset: usize) -> vortex_error::VortexResult impl core::clone::Clone for vortex_fastlanes::DeltaData @@ -352,6 +320,14 @@ impl core::fmt::Debug for vortex_fastlanes::DeltaData pub fn vortex_fastlanes::DeltaData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_fastlanes::DeltaData + +pub fn vortex_fastlanes::DeltaData::array_eq(&self, other: &Self, _precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_fastlanes::DeltaData + +pub fn vortex_fastlanes::DeltaData::array_hash(&self, state: &mut H, _precision: vortex_array::hash::Precision) + pub struct vortex_fastlanes::FoR impl vortex_fastlanes::FoR @@ -378,15 +354,11 @@ pub type vortex_fastlanes::FoR::OperationsVTable = vortex_fastlanes::FoR pub type vortex_fastlanes::FoR::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild -pub fn vortex_fastlanes::FoR::array_eq(array: &vortex_fastlanes::FoRData, other: &vortex_fastlanes::FoRData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_fastlanes::FoR::array_hash(array: &vortex_fastlanes::FoRData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_fastlanes::FoR::buffer(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_fastlanes::FoR::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_fastlanes::FoR::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_fastlanes::FoR::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_fastlanes::FoR::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -402,11 +374,7 @@ pub fn vortex_fastlanes::FoR::serialize(array: vortex_array::array::view::ArrayV pub fn vortex_fastlanes::FoR::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_fastlanes::FoR::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_fastlanes::FoR::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_fastlanes::FoR::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_fastlanes::FoR::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_fastlanes::FoR @@ -414,7 +382,7 @@ pub fn vortex_fastlanes::FoR::scalar_at(array: vortex_array::array::view::ArrayV impl vortex_array::array::vtable::validity::ValidityChild for vortex_fastlanes::FoR -pub fn vortex_fastlanes::FoR::validity_child(array: &vortex_fastlanes::FoRData) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_fastlanes::FoR::validity_child(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::FoR>) -> vortex_array::array::erased::ArrayRef impl vortex_array::arrays::dict::take::TakeExecute for vortex_fastlanes::FoR @@ -440,21 +408,11 @@ pub struct vortex_fastlanes::FoRData impl vortex_fastlanes::FoRData -pub fn vortex_fastlanes::FoRData::dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_fastlanes::FoRData::encoded(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn vortex_fastlanes::FoRData::is_empty(&self) -> bool - -pub fn vortex_fastlanes::FoRData::len(&self) -> usize - -pub fn vortex_fastlanes::FoRData::ptype(&self) -> vortex_array::dtype::ptype::PType - -pub fn vortex_fastlanes::FoRData::reference_scalar(&self) -> &vortex_array::scalar::Scalar +pub fn vortex_fastlanes::FoRData::encode(array: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult impl vortex_fastlanes::FoRData -pub fn vortex_fastlanes::FoRData::encode(array: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult +pub fn vortex_fastlanes::FoRData::ptype(&self) -> vortex_array::dtype::ptype::PType impl core::clone::Clone for vortex_fastlanes::FoRData @@ -464,6 +422,14 @@ impl core::fmt::Debug for vortex_fastlanes::FoRData pub fn vortex_fastlanes::FoRData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_fastlanes::FoRData + +pub fn vortex_fastlanes::FoRData::array_eq(&self, other: &Self, _precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_fastlanes::FoRData + +pub fn vortex_fastlanes::FoRData::array_hash(&self, state: &mut H, _precision: vortex_array::hash::Precision) + pub struct vortex_fastlanes::RLE impl vortex_fastlanes::RLE @@ -492,15 +458,11 @@ pub type vortex_fastlanes::RLE::OperationsVTable = vortex_fastlanes::RLE pub type vortex_fastlanes::RLE::ValidityVTable = vortex_fastlanes::RLE -pub fn vortex_fastlanes::RLE::array_eq(array: &vortex_fastlanes::RLEData, other: &vortex_fastlanes::RLEData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_fastlanes::RLE::array_hash(array: &vortex_fastlanes::RLEData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_fastlanes::RLE::buffer(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_fastlanes::RLE::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_fastlanes::RLE::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_fastlanes::RLE::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_fastlanes::RLE::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -516,11 +478,7 @@ pub fn vortex_fastlanes::RLE::serialize(array: vortex_array::array::view::ArrayV pub fn vortex_fastlanes::RLE::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_fastlanes::RLE::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_fastlanes::RLE::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_fastlanes::RLE::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_fastlanes::RLE::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_fastlanes::RLE @@ -546,17 +504,11 @@ pub fn vortex_fastlanes::RLEData::encode(array: &vortex_array::arrays::primitive impl vortex_fastlanes::RLEData -pub fn vortex_fastlanes::RLEData::indices(&self) -> &vortex_array::array::erased::ArrayRef - -pub unsafe fn vortex_fastlanes::RLEData::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize) -> Self +pub unsafe fn vortex_fastlanes::RLEData::new_unchecked(offset: usize) -> Self pub fn vortex_fastlanes::RLEData::offset(&self) -> usize -pub fn vortex_fastlanes::RLEData::try_new(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::RLEData::values(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn vortex_fastlanes::RLEData::values_idx_offsets(&self) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_fastlanes::RLEData::try_new(offset: usize) -> vortex_error::VortexResult impl core::clone::Clone for vortex_fastlanes::RLEData @@ -566,15 +518,17 @@ impl core::fmt::Debug for vortex_fastlanes::RLEData pub fn vortex_fastlanes::RLEData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub trait vortex_fastlanes::BitPackedArrayExt +impl vortex_array::hash::ArrayEq for vortex_fastlanes::RLEData -pub fn vortex_fastlanes::BitPackedArrayExt::bit_width(&self) -> u8 +pub fn vortex_fastlanes::RLEData::array_eq(&self, other: &Self, _precision: vortex_array::hash::Precision) -> bool -pub fn vortex_fastlanes::BitPackedArrayExt::bitpacked_data(&self) -> &vortex_fastlanes::BitPackedData +impl vortex_array::hash::ArrayHash for vortex_fastlanes::RLEData -pub fn vortex_fastlanes::BitPackedArrayExt::bitpacked_dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_fastlanes::RLEData::array_hash(&self, state: &mut H, _precision: vortex_array::hash::Precision) -pub fn vortex_fastlanes::BitPackedArrayExt::bitpacked_len(&self) -> usize +pub trait vortex_fastlanes::BitPackedArrayExt: vortex_array::array::typed::TypedArrayRef + +pub fn vortex_fastlanes::BitPackedArrayExt::bit_width(&self) -> u8 pub fn vortex_fastlanes::BitPackedArrayExt::offset(&self) -> u16 @@ -598,69 +552,67 @@ pub fn vortex_fastlanes::BitPackedArrayExt::validity_child(&self) -> core::optio pub fn vortex_fastlanes::BitPackedArrayExt::validity_mask(&self) -> vortex_mask::Mask -impl vortex_fastlanes::BitPackedArrayExt for vortex_array::array::typed::Array - -pub fn vortex_array::array::typed::Array::bit_width(&self) -> u8 +impl> vortex_fastlanes::BitPackedArrayExt for T -pub fn vortex_array::array::typed::Array::bitpacked_data(&self) -> &vortex_fastlanes::BitPackedData +pub fn T::bit_width(&self) -> u8 -pub fn vortex_array::array::typed::Array::bitpacked_dtype(&self) -> &vortex_array::dtype::DType +pub fn T::offset(&self) -> u16 -pub fn vortex_array::array::typed::Array::bitpacked_len(&self) -> usize +pub fn T::packed(&self) -> &vortex_array::buffer::BufferHandle -pub fn vortex_array::array::typed::Array::offset(&self) -> u16 +pub fn T::packed_slice(&self) -> &[T] -pub fn vortex_array::array::typed::Array::packed(&self) -> &vortex_array::buffer::BufferHandle +pub fn T::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> -pub fn vortex_array::array::typed::Array::packed_slice(&self) -> &[T] +pub fn T::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> -pub fn vortex_array::array::typed::Array::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> +pub fn T::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> -pub fn vortex_array::array::typed::Array::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> +pub fn T::patches(&self) -> core::option::Option -pub fn vortex_array::array::typed::Array::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> +pub fn T::unpacked_chunks(&self) -> vortex_error::VortexResult> -pub fn vortex_array::array::typed::Array::patches(&self) -> core::option::Option +pub fn T::validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::array::typed::Array::unpacked_chunks(&self) -> vortex_error::VortexResult> +pub fn T::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> -pub fn vortex_array::array::typed::Array::validity(&self) -> vortex_array::validity::Validity +pub fn T::validity_mask(&self) -> vortex_mask::Mask -pub fn vortex_array::array::typed::Array::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> +pub trait vortex_fastlanes::FoRArrayExt: vortex_array::array::typed::TypedArrayRef -pub fn vortex_array::array::typed::Array::validity_mask(&self) -> vortex_mask::Mask +pub fn vortex_fastlanes::FoRArrayExt::encoded(&self) -> &vortex_array::array::erased::ArrayRef -impl vortex_fastlanes::BitPackedArrayExt for vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked> +pub fn vortex_fastlanes::FoRArrayExt::reference_scalar(&self) -> &vortex_array::scalar::Scalar -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bit_width(&self) -> u8 +impl> vortex_fastlanes::FoRArrayExt for T -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bitpacked_data(&self) -> &vortex_fastlanes::BitPackedData +pub fn T::encoded(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bitpacked_dtype(&self) -> &vortex_array::dtype::DType +pub fn T::reference_scalar(&self) -> &vortex_array::scalar::Scalar -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bitpacked_len(&self) -> usize +pub trait vortex_fastlanes::RLEArrayExt: vortex_array::array::typed::TypedArrayRef -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::offset(&self) -> u16 +pub fn vortex_fastlanes::RLEArrayExt::indices(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::packed(&self) -> &vortex_array::buffer::BufferHandle +pub fn vortex_fastlanes::RLEArrayExt::offset(&self) -> usize -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::packed_slice(&self) -> &[T] +pub fn vortex_fastlanes::RLEArrayExt::values(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> +pub fn vortex_fastlanes::RLEArrayExt::values_idx_offset(&self, chunk_idx: usize) -> usize -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> +pub fn vortex_fastlanes::RLEArrayExt::values_idx_offsets(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> +impl> vortex_fastlanes::RLEArrayExt for T -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patches(&self) -> core::option::Option +pub fn T::indices(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::unpacked_chunks(&self) -> vortex_error::VortexResult> +pub fn T::offset(&self) -> usize -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::validity(&self) -> vortex_array::validity::Validity +pub fn T::values(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> +pub fn T::values_idx_offset(&self, chunk_idx: usize) -> usize -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::validity_mask(&self) -> vortex_mask::Mask +pub fn T::values_idx_offsets(&self) -> &vortex_array::array::erased::ArrayRef pub fn vortex_fastlanes::delta_compress(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<(vortex_array::arrays::primitive::vtable::PrimitiveArray, vortex_array::arrays::primitive::vtable::PrimitiveArray)> diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index 9b6fcbc9fd1..dfbb0ed7b17 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -6,6 +6,7 @@ use itertools::Itertools; use num_traits::PrimInt; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::IntegerPType; use vortex_array::dtype::NativePType; @@ -425,6 +426,7 @@ mod test { use super::*; use crate::BitPackedData; use crate::bitpack_compress::test_harness::make_array; + use crate::bitpacking::array::BitPackedArrayExt; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); @@ -449,7 +451,7 @@ mod test { ); assert!(values.ptype().is_unsigned_int()); let compressed = BitPackedData::encode(&values.into_array(), 4).unwrap(); - assert!(compressed.patches(compressed.len()).is_none()); + assert!(compressed.patches().is_none()); assert_eq!( (0..(1 << 4)).collect::>(), compressed @@ -511,7 +513,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches(bitpacked.len()).unwrap(); + let patches = bitpacked.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // chunk 0 (0-1023): patches at 100, 200 -> starts at patch index 0 @@ -534,7 +536,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches(bitpacked.len()).unwrap(); + let patches = bitpacked.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); assert_arrays_eq!(chunk_offsets, PrimitiveArray::from_iter([0u64, 2, 2])); @@ -553,7 +555,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches(bitpacked.len()).unwrap(); + let patches = bitpacked.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // chunk 0 (0-1023): patches at 100, 200 -> starts at patch index 0 @@ -577,7 +579,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches(bitpacked.len()).unwrap(); + let patches = bitpacked.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // Single chunk starting at patch index 0. diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index 9e8c98b1c04..935cac63016 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -2,20 +2,17 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use fastlanes::BitPacking; -use vortex_array::Array; use vortex_array::ArrayRef; -use vortex_array::ArrayView; +use vortex_array::TypedArrayRef; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; -use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::patches::Patches; use vortex_array::validity::Validity; use vortex_array::vtable::child_to_validity; -use vortex_array::vtable::validity_to_child; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -57,7 +54,6 @@ pub struct BitPackedDataParts { #[derive(Clone, Debug)] pub struct BitPackedData { - pub(super) slots: Vec>, /// The offset within the first block (created with a slice). /// 0 <= offset < 1024 pub(super) offset: u16, @@ -90,47 +86,6 @@ impl BitPackedData { /// /// See also the [`encode`][Self::encode] method on this type for a safe path to create a new /// bit-packed array. - pub(crate) unsafe fn new_unchecked( - packed: BufferHandle, - validity: Validity, - patches: Option, - bit_width: u8, - len: usize, - offset: u16, - ) -> Self { - let slots = Self::make_slots(&patches, &validity, len); - let (patch_offset, patch_offset_within_chunk) = match &patches { - Some(p) => (Some(p.offset()), p.offset_within_chunk()), - None => (None, None), - }; - - Self { - slots, - offset, - bit_width, - packed, - patch_offset, - patch_offset_within_chunk, - } - } - - fn make_slots( - patches: &Option, - validity: &Validity, - len: usize, - ) -> Vec> { - let (pi, pv, pco) = match patches { - Some(p) => ( - Some(p.indices().clone()), - Some(p.values().clone()), - p.chunk_offsets().clone(), - ), - None => (None, None, None), - }; - let validity_slot = validity_to_child(validity, len); - vec![pi, pv, pco, validity_slot] - } - /// A safe constructor for a `BitPackedArray` from its components: /// /// * `packed` is ByteBuffer holding the compressed data that was packed with FastLanes @@ -154,46 +109,31 @@ impl BitPackedData { /// Any violation of these preconditions will result in an error. pub fn try_new( packed: BufferHandle, - ptype: PType, - validity: Validity, patches: Option, bit_width: u8, - length: usize, offset: u16, ) -> VortexResult { - Self::validate( - &packed, - ptype, - &validity, - patches.as_ref(), - bit_width, - length, - offset, - )?; + vortex_ensure!(bit_width <= 64, "Unsupported bit width {bit_width}"); + vortex_ensure!( + offset < 1024, + "Offset must be less than the full block i.e., 1024, got {offset}" + ); - // SAFETY: all components validated above - unsafe { - Ok(Self::new_unchecked( - packed, validity, patches, bit_width, length, offset, - )) - } - } + let (patch_offset, patch_offset_within_chunk) = match &patches { + Some(p) => (Some(p.offset()), p.offset_within_chunk()), + None => (None, None), + }; - pub(crate) fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { - let validity = self.validity(dtype.nullability()); - let patches = self.patches(len); - Self::validate( - &self.packed, - dtype.as_ptype(), - &validity, - patches.as_ref(), - self.bit_width, - len, - self.offset, - ) + Ok(Self { + offset, + bit_width, + packed, + patch_offset, + patch_offset_within_chunk, + }) } - fn validate( + pub(crate) fn validate( packed: &BufferHandle, ptype: PType, validity: &Validity, @@ -212,12 +152,6 @@ impl BitPackedData { ); } - // Validate offset for sliced arrays - vortex_ensure!( - offset < 1024, - "Offset must be less than the full block i.e., 1024, got {offset}" - ); - // Validate patches if let Some(patches) = patches { Self::validate_patches(patches, ptype, length)?; @@ -254,11 +188,6 @@ impl BitPackedData { Ok(()) } - /// Returns the validity as a [`Mask`](vortex_mask::Mask). - pub fn validity_mask(&self, len: usize, nullability: Nullability) -> vortex_mask::Mask { - self.validity(nullability).to_mask(len) - } - pub fn ptype(&self, dtype: &DType) -> PType { dtype.as_ptype() } @@ -283,26 +212,6 @@ impl BitPackedData { unsafe { std::slice::from_raw_parts(packed_ptr, packed_len) } } - #[inline] - pub fn patch_indices(&self) -> Option<&ArrayRef> { - self.slots[PATCH_INDICES_SLOT].as_ref() - } - - #[inline] - pub fn patch_values(&self) -> Option<&ArrayRef> { - self.slots[PATCH_VALUES_SLOT].as_ref() - } - - #[inline] - pub fn patch_chunk_offsets(&self) -> Option<&ArrayRef> { - self.slots[PATCH_CHUNK_OFFSETS_SLOT].as_ref() - } - - #[inline] - pub fn validity_child(&self) -> Option<&ArrayRef> { - self.slots[VALIDITY_SLOT].as_ref() - } - /// Accessor for bit unpacked chunks pub fn unpacked_chunks( &self, @@ -323,53 +232,6 @@ impl BitPackedData { self.bit_width } - /// Access the patches array. - /// - /// Reconstructs a `Patches` from the stored slots and patch metadata. - /// If present, patches MUST be a `SparseArray` with equal-length to this array, and whose - /// indices indicate the locations of patches. The indices must have non-zero length. - pub fn patches(&self, len: usize) -> Option { - match (self.patch_indices(), self.patch_values()) { - (Some(indices), Some(values)) => { - let patch_offset = self - .patch_offset - .vortex_expect("has patch slots but no patch_offset"); - Some(unsafe { - Patches::new_unchecked( - len, - patch_offset, - indices.clone(), - values.clone(), - self.patch_chunk_offsets().cloned(), - self.patch_offset_within_chunk, - ) - }) - } - _ => None, - } - } - - /// Returns the validity, reconstructed from the stored slot. - pub fn validity(&self, nullability: Nullability) -> Validity { - child_to_validity(&self.validity_child().cloned(), nullability) - } - - pub fn replace_patches(&mut self, patches: Option) { - let (pi, pv, pco) = match &patches { - Some(p) => ( - Some(p.indices().clone()), - Some(p.values().clone()), - p.chunk_offsets().clone(), - ), - None => (None, None, None), - }; - self.slots[PATCH_INDICES_SLOT] = pi; - self.slots[PATCH_VALUES_SLOT] = pv; - self.slots[PATCH_CHUNK_OFFSETS_SLOT] = pco; - self.patch_offset = patches.as_ref().map(|p| p.offset()); - self.patch_offset_within_chunk = patches.as_ref().and_then(|p| p.offset_within_chunk()); - } - #[inline] pub fn offset(&self) -> u16 { self.offset @@ -389,7 +251,7 @@ impl BitPackedData { pub fn encode(array: &ArrayRef, bit_width: u8) -> VortexResult { let parray: PrimitiveArray = array .clone() - .try_into::() + .try_downcast::() .map_err(|a| vortex_err!(InvalidArgument: "Bitpacking can only encode primitive arrays, got {}", a.encoding_id()))?; bitpack_encode(&parray, bit_width, None) } @@ -401,116 +263,91 @@ impl BitPackedData { pub fn max_packed_value(&self) -> usize { (1 << self.bit_width()) - 1 } - - pub fn into_parts(self, len: usize, nullability: Nullability) -> BitPackedDataParts { - let patches = self.patches(len); - let validity = self.validity(nullability); - BitPackedDataParts { - offset: self.offset, - bit_width: self.bit_width, - len, - packed: self.packed, - patches, - validity, - } - } } -pub trait BitPackedArrayExt { - fn bitpacked_data(&self) -> &BitPackedData; - fn bitpacked_dtype(&self) -> &DType; - fn bitpacked_len(&self) -> usize; - +pub trait BitPackedArrayExt: TypedArrayRef { #[inline] fn packed(&self) -> &BufferHandle { - self.bitpacked_data().packed() + BitPackedData::packed(self) } #[inline] fn bit_width(&self) -> u8 { - self.bitpacked_data().bit_width() + BitPackedData::bit_width(self) } #[inline] fn offset(&self) -> u16 { - self.bitpacked_data().offset() + BitPackedData::offset(self) } #[inline] fn patch_indices(&self) -> Option<&ArrayRef> { - self.bitpacked_data().patch_indices() + self.as_ref().slots()[PATCH_INDICES_SLOT].as_ref() } #[inline] fn patch_values(&self) -> Option<&ArrayRef> { - self.bitpacked_data().patch_values() + self.as_ref().slots()[PATCH_VALUES_SLOT].as_ref() } #[inline] fn patch_chunk_offsets(&self) -> Option<&ArrayRef> { - self.bitpacked_data().patch_chunk_offsets() + self.as_ref().slots()[PATCH_CHUNK_OFFSETS_SLOT].as_ref() } #[inline] fn validity_child(&self) -> Option<&ArrayRef> { - self.bitpacked_data().validity_child() + self.as_ref().slots()[VALIDITY_SLOT].as_ref() } #[inline] fn patches(&self) -> Option { - self.bitpacked_data().patches(self.bitpacked_len()) + match (self.patch_indices(), self.patch_values()) { + (Some(indices), Some(values)) => { + let patch_offset = self + .patch_offset + .vortex_expect("has patch slots but no patch_offset"); + Some(unsafe { + Patches::new_unchecked( + self.as_ref().len(), + patch_offset, + indices.clone(), + values.clone(), + self.patch_chunk_offsets().cloned(), + self.patch_offset_within_chunk, + ) + }) + } + _ => None, + } } #[inline] fn validity(&self) -> Validity { - self.bitpacked_data() - .validity(self.bitpacked_dtype().nullability()) + child_to_validity( + &self.validity_child().cloned(), + self.as_ref().dtype().nullability(), + ) } #[inline] fn validity_mask(&self) -> vortex_mask::Mask { - self.validity().to_mask(self.bitpacked_len()) + self.validity().to_mask(self.as_ref().len()) } #[inline] fn packed_slice(&self) -> &[T] { - self.bitpacked_data().packed_slice::() + BitPackedData::packed_slice::(self) } #[inline] fn unpacked_chunks(&self) -> VortexResult> { - self.bitpacked_data() - .unpacked_chunks::(self.bitpacked_dtype(), self.bitpacked_len()) + BitPackedData::unpacked_chunks::(self, self.as_ref().dtype(), self.as_ref().len()) } } -impl BitPackedArrayExt for Array { - fn bitpacked_data(&self) -> &BitPackedData { - self.data() - } - - fn bitpacked_dtype(&self) -> &DType { - self.dtype() - } - - fn bitpacked_len(&self) -> usize { - self.len() - } -} - -impl BitPackedArrayExt for ArrayView<'_, crate::BitPacked> { - fn bitpacked_data(&self) -> &BitPackedData { - self.data() - } - - fn bitpacked_dtype(&self) -> &DType { - self.dtype() - } - - fn bitpacked_len(&self) -> usize { - self.len() - } -} +impl> BitPackedArrayExt for T {} #[cfg(test)] mod test { @@ -521,6 +358,7 @@ mod test { use vortex_buffer::Buffer; use crate::BitPackedData; + use crate::bitpacking::array::BitPackedArrayExt; #[test] fn test_encode() { @@ -555,11 +393,7 @@ mod test { let parray = values.clone().into_array(); let packed_with_patches = BitPackedData::encode(&parray, 9).unwrap(); - assert!( - packed_with_patches - .patches(packed_with_patches.len()) - .is_some() - ); + assert!(packed_with_patches.patches().is_some()); assert_arrays_eq!( packed_with_patches.as_array().to_primitive(), PrimitiveArray::new(values, vortex_array::validity::Validity::NonNullable) diff --git a/encodings/fastlanes/src/bitpacking/compute/cast.rs b/encodings/fastlanes/src/bitpacking/compute/cast.rs index 3ebb97c511b..fbcd48ce75b 100644 --- a/encodings/fastlanes/src/bitpacking/compute/cast.rs +++ b/encodings/fastlanes/src/bitpacking/compute/cast.rs @@ -11,6 +11,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::bitpacking::BitPacked; +use crate::bitpacking::array::BitPackedArrayExt; impl CastReduce for BitPacked { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { if array.dtype().eq_ignore_nullability(dtype) { @@ -23,7 +24,7 @@ impl CastReduce for BitPacked { dtype.as_ptype(), new_validity, array - .patches(array.len()) + .patches() .map(|patches| { let new_values = patches.values().cast(dtype.clone())?; Patches::new( diff --git a/encodings/fastlanes/src/bitpacking/compute/filter.rs b/encodings/fastlanes/src/bitpacking/compute/filter.rs index b0d7629d089..ec2e7f6ea68 100644 --- a/encodings/fastlanes/src/bitpacking/compute/filter.rs +++ b/encodings/fastlanes/src/bitpacking/compute/filter.rs @@ -186,6 +186,7 @@ mod test { use vortex_mask::Mask; use crate::BitPackedData; + use crate::bitpacking::array::BitPackedArrayExt; #[test] fn take_indices() { @@ -275,7 +276,7 @@ mod test { let unpacked = PrimitiveArray::from_iter(values.clone()); let bitpacked = BitPackedData::encode(&unpacked.into_array(), 7).unwrap(); assert!( - bitpacked.patches(bitpacked.len()).is_some(), + bitpacked.patches().is_some(), "Expected patches for values exceeding bit width" ); @@ -307,7 +308,7 @@ mod test { let unpacked = PrimitiveArray::from_iter(values.clone()); let bitpacked = BitPackedData::encode(&unpacked.into_array(), 7).unwrap(); assert!( - bitpacked.patches(bitpacked.len()).is_some(), + bitpacked.patches().is_some(), "Expected patches for values exceeding bit width" ); diff --git a/encodings/fastlanes/src/bitpacking/compute/slice.rs b/encodings/fastlanes/src/bitpacking/compute/slice.rs index 0e7dc4d7e43..0c016b17b49 100644 --- a/encodings/fastlanes/src/bitpacking/compute/slice.rs +++ b/encodings/fastlanes/src/bitpacking/compute/slice.rs @@ -11,6 +11,7 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::BitPacked; +use crate::bitpacking::array::BitPackedArrayExt; impl SliceReduce for BitPacked { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { @@ -29,7 +30,7 @@ impl SliceReduce for BitPacked { array.dtype().as_ptype(), array.validity()?.slice(range.clone())?, array - .patches(array.len()) + .patches() .map(|p| p.slice(range.clone())) .transpose()? .flatten(), diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index 6a8415e293d..9422b44be1b 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -176,6 +176,7 @@ mod test { use crate::BitPackedArray; use crate::BitPackedData; + use crate::bitpacking::array::BitPackedArrayExt; use crate::bitpacking::compute::take::take_primitive; #[test] @@ -224,7 +225,7 @@ mod test { let values = (0..u16::MAX as u32 + num_patches as u32).collect::>(); let uncompressed = PrimitiveArray::new(values.clone(), Validity::NonNullable); let packed = BitPackedData::encode(&uncompressed.into_array(), 16).unwrap(); - assert!(packed.patches(packed.len()).is_some()); + assert!(packed.patches().is_some()); let rng = rng(); let range = Uniform::new(0, values.len()).unwrap(); diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index 29741c89623..9e9ed2a400d 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::hash::Hash; +use std::hash::Hasher; use prost::Message; use vortex_array::AnyCanonical; @@ -29,10 +30,11 @@ use vortex_array::serde::ArrayChildren; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; +use vortex_array::vtable::child_to_validity; +use vortex_array::vtable::validity_to_child; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; @@ -42,7 +44,6 @@ use crate::BitPackedData; use crate::BitPackedDataParts; use crate::bitpack_decompress::unpack_array; use crate::bitpack_decompress::unpack_into_primitive_builder; -use crate::bitpacking::array::NUM_SLOTS; use crate::bitpacking::array::PATCH_CHUNK_OFFSETS_SLOT; use crate::bitpacking::array::PATCH_INDICES_SLOT; use crate::bitpacking::array::PATCH_VALUES_SLOT; @@ -67,6 +68,26 @@ pub struct BitPackedMetadata { pub(crate) patches: Option, } +impl ArrayHash for BitPackedData { + fn array_hash(&self, state: &mut H, precision: Precision) { + self.offset.hash(state); + self.bit_width.hash(state); + self.packed.array_hash(state, precision); + self.patch_offset.hash(state); + self.patch_offset_within_chunk.hash(state); + } +} + +impl ArrayEq for BitPackedData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + self.offset == other.offset + && self.bit_width == other.bit_width + && self.packed.array_eq(&other.packed, precision) + && self.patch_offset == other.patch_offset + && self.patch_offset_within_chunk == other.patch_offset_within_chunk + } +} + impl VTable for BitPacked { type ArrayData = BitPackedData; @@ -77,76 +98,41 @@ impl VTable for BitPacked { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate_against_outer(dtype, len) - } - - fn array_hash( - array: &BitPackedData, - state: &mut H, - precision: Precision, - ) { - array.offset.hash(state); - array.bit_width.hash(state); - array.packed.array_hash(state, precision); - match array.patch_indices() { - Some(indices) => { - true.hash(state); - indices.array_hash(state, precision); - } - None => false.hash(state), - } - match array.patch_values() { - Some(values) => { - true.hash(state); - values.array_hash(state, precision); - } - None => false.hash(state), - } - match array.patch_chunk_offsets() { - Some(offsets) => { - true.hash(state); - offsets.array_hash(state, precision); - } - None => false.hash(state), - } - match array.validity_child() { - Some(validity) => { - true.hash(state); - validity.array_hash(state, precision); - } - None => false.hash(state), - } - array.patch_offset.hash(state); - array.patch_offset_within_chunk.hash(state); - } - - fn array_eq(array: &BitPackedData, other: &BitPackedData, precision: Precision) -> bool { - array.offset == other.offset - && array.bit_width == other.bit_width - && array.packed.array_eq(&other.packed, precision) - && match (array.patch_indices(), other.patch_indices()) { - (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), - (None, None) => true, - _ => false, - } - && match (array.patch_values(), other.patch_values()) { - (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), - (None, None) => true, - _ => false, - } - && match (array.patch_chunk_offsets(), other.patch_chunk_offsets()) { - (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), - (None, None) => true, - _ => false, - } - && match (array.validity_child(), other.validity_child()) { - (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), - (None, None) => true, - _ => false, + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let validity = child_to_validity(&slots[VALIDITY_SLOT].clone(), dtype.nullability()); + let patches = match (&slots[PATCH_INDICES_SLOT], &slots[PATCH_VALUES_SLOT]) { + (Some(indices), Some(values)) => { + let patch_offset = data + .patch_offset + .vortex_expect("has patch slots but no patch_offset"); + Some(unsafe { + Patches::new_unchecked( + len, + patch_offset, + indices.clone(), + values.clone(), + slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), + data.patch_offset_within_chunk, + ) + }) } - && array.patch_offset == other.patch_offset - && array.patch_offset_within_chunk == other.patch_offset_within_chunk + _ => None, + }; + BitPackedData::validate( + &data.packed, + dtype.as_ptype(), + &validity, + patches.as_ref(), + data.bit_width, + len, + data.offset, + ) } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -175,32 +161,10 @@ impl VTable for BitPacked { RULES.evaluate(array, parent, child_idx) } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "BitPackedArray expects {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - - // If patch slots are being cleared, clear the metadata too - if slots[PATCH_INDICES_SLOT].is_none() || slots[PATCH_VALUES_SLOT].is_none() { - array.patch_offset = None; - array.patch_offset_within_chunk = None; - } - - array.slots = slots; - Ok(()) - } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( BitPackedMetadata { @@ -223,7 +187,7 @@ impl VTable for BitPacked { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = BitPackedMetadata::decode(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); @@ -268,10 +232,20 @@ impl VTable for BitPacked { }) .transpose()?; - BitPackedData::try_new( + let slots = { + let (pi, pv, pco) = match &patches { + Some(p) => ( + Some(p.indices().clone()), + Some(p.values().clone()), + p.chunk_offsets().clone(), + ), + None => (None, None, None), + }; + let validity_slot = validity_to_child(&validity, len); + vec![pi, pv, pco, validity_slot] + }; + let data = BitPackedData::try_new( packed, - PType::try_from(dtype)?, - validity, patches, u8::try_from(metadata.bit_width).map_err(|_| { vortex_err!( @@ -279,14 +253,14 @@ impl VTable for BitPacked { metadata.bit_width ) })?, - len, u16::try_from(metadata.offset).map_err(|_| { vortex_err!( "BitPackedMetadata offset {} does not fit in u16", metadata.offset ) })?, - ) + )?; + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn append_to_builder( @@ -352,15 +326,35 @@ impl BitPacked { offset: u16, ) -> VortexResult { let dtype = DType::Primitive(ptype, validity.nullability()); - let data = - BitPackedData::try_new(packed, ptype, validity, patches, bit_width, len, offset)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(BitPacked, dtype, len, data)) }) + let slots = { + let (pi, pv, pco) = match &patches { + Some(p) => ( + Some(p.indices().clone()), + Some(p.values().clone()), + p.chunk_offsets().clone(), + ), + None => (None, None, None), + }; + let validity_slot = validity_to_child(&validity, len); + vec![pi, pv, pco, validity_slot] + }; + let data = BitPackedData::try_new(packed, patches, bit_width, offset)?; + Array::try_from_parts(ArrayParts::new(BitPacked, dtype, len, data).with_slots(slots)) } pub fn into_parts(array: BitPackedArray) -> BitPackedDataParts { let len = array.len(); - let nullability = array.dtype().nullability(); - array.into_data().into_parts(len, nullability) + let patches = array.patches(); + let validity = array.validity().vortex_expect("BitPacked validity"); + let data = array.into_data(); + BitPackedDataParts { + offset: data.offset, + bit_width: data.bit_width, + len, + packed: data.packed, + patches, + validity, + } } /// Encode an array into a bitpacked representation with the given bit width. diff --git a/encodings/fastlanes/src/bitpacking/vtable/operations.rs b/encodings/fastlanes/src/bitpacking/vtable/operations.rs index 49984c80ade..4e2213cf87a 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/operations.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/operations.rs @@ -9,6 +9,7 @@ use vortex_error::VortexResult; use crate::BitPacked; use crate::bitpack_decompress; +use crate::bitpacking::array::BitPackedArrayExt; impl OperationsVTable for BitPacked { fn scalar_at( array: ArrayView<'_, BitPacked>, @@ -16,7 +17,7 @@ impl OperationsVTable for BitPacked { _ctx: &mut ExecutionCtx, ) -> VortexResult { Ok( - if let Some(patches) = array.patches(array.len()) + if let Some(patches) = array.patches() && let Some(patch) = patches.get_patched(index)? { patch @@ -52,6 +53,7 @@ mod test { use crate::BitPacked; use crate::BitPackedArray; use crate::BitPackedData; + use crate::bitpacking::array::BitPackedArrayExt; fn bp(array: &ArrayRef, bit_width: u8) -> BitPackedArray { BitPackedData::encode(array, bit_width).unwrap() @@ -141,14 +143,14 @@ mod test { // We create an array that has 1 element that does not fit in the 6-bit range. let array = BitPackedData::encode(&buffer![0u32..=64].into_array(), 6).unwrap(); - assert!(array.patches(array.len()).is_some()); + assert!(array.patches().is_some()); - let patch_indices = array.patches(array.len()).unwrap().indices().clone(); + let patch_indices = array.patches().unwrap().indices().clone(); assert_eq!(patch_indices.len(), 1); // Slicing drops the empty patches array. let sliced_bp = slice_via_reduce(&array, 0..64); - assert!(sliced_bp.patches(sliced_bp.len()).is_none()); + assert!(sliced_bp.patches().is_none()); } #[test] @@ -211,9 +213,9 @@ mod test { let values = (0u32..257).collect::>(); let uncompressed = values.clone().into_array(); let packed = BitPackedData::encode(&uncompressed, 8).unwrap(); - assert!(packed.patches(packed.len()).is_some()); + assert!(packed.patches().is_some()); - let patches = packed.patches(packed.len()).unwrap().indices().clone(); + let patches = packed.patches().unwrap().indices().clone(); assert_eq!( usize::try_from(&patches.scalar_at(0).unwrap()).unwrap(), 256 diff --git a/encodings/fastlanes/src/bitpacking/vtable/validity.rs b/encodings/fastlanes/src/bitpacking/vtable/validity.rs index 2313e0112eb..b3d61324af5 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/validity.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/validity.rs @@ -7,9 +7,10 @@ use vortex_array::vtable::ValidityVTable; use vortex_error::VortexResult; use crate::BitPacked; +use crate::BitPackedArrayExt; impl ValidityVTable for BitPacked { fn validity(array: ArrayView<'_, BitPacked>) -> VortexResult { - Ok(array.data().validity(array.dtype().nullability())) + Ok(BitPackedArrayExt::validity(&array)) } } diff --git a/encodings/fastlanes/src/delta/array/delta_decompress.rs b/encodings/fastlanes/src/delta/array/delta_decompress.rs index 7539bcdb0e2..d6404dfb905 100644 --- a/encodings/fastlanes/src/delta/array/delta_decompress.rs +++ b/encodings/fastlanes/src/delta/array/delta_decompress.rs @@ -18,6 +18,7 @@ use vortex_error::VortexResult; use crate::DeltaArray; use crate::bit_transpose::untranspose_validity; +use crate::delta::array::DeltaArrayExt; pub fn delta_decompress( array: &DeltaArray, diff --git a/encodings/fastlanes/src/delta/array/mod.rs b/encodings/fastlanes/src/delta/array/mod.rs index 05d9dfbbadb..4012bab8f13 100644 --- a/encodings/fastlanes/src/delta/array/mod.rs +++ b/encodings/fastlanes/src/delta/array/mod.rs @@ -3,7 +3,7 @@ use fastlanes::FastLanes; use vortex_array::ArrayRef; -use vortex_array::dtype::DType; +use vortex_array::TypedArrayRef; use vortex_array::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; use vortex_error::VortexExpect; @@ -62,110 +62,32 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["bases", "deltas"]; #[derive(Clone, Debug)] pub struct DeltaData { pub(super) offset: usize, - pub(super) slots: Vec>, } -impl DeltaData { - /// Create a DeltaArray from the given `bases` and `deltas` arrays - /// with given `offset` into first chunk and `logical_len` length. - pub fn try_new( - bases: ArrayRef, - deltas: ArrayRef, - offset: usize, - len: usize, - ) -> VortexResult { - Self::validate_parts(&bases, &deltas, offset, len)?; - - // SAFETY: validation done above - Ok(unsafe { Self::new_unchecked(bases, deltas, offset) }) - } - - pub(crate) fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { - Self::validate_parts(self.bases(), self.deltas(), self.offset, len)?; - let expected_dtype = self - .bases() - .dtype() - .with_nullability(self.deltas().dtype().nullability()); - vortex_ensure!( - dtype == &expected_dtype, - "DeltaArray dtype mismatch: expected {expected_dtype}, got {dtype}" - ); - Ok(()) - } - - fn validate_parts( - bases: &ArrayRef, - deltas: &ArrayRef, - offset: usize, - len: usize, - ) -> VortexResult<()> { - vortex_ensure!(offset < 1024, "offset must be less than 1024: {offset}"); - vortex_ensure!( - offset + len <= deltas.len(), - "offset + len, {offset} + {len}, must be less than or equal to the size of deltas: {}", - deltas.len() - ); - vortex_ensure!( - bases.dtype().eq_ignore_nullability(deltas.dtype()), - "DeltaArray: bases and deltas must have the same dtype, got {} and {}", - bases.dtype(), - deltas.dtype() - ); - - vortex_ensure!( - bases.dtype().is_unsigned_int(), - "DeltaArray: dtype must be an unsigned integer, got {}", - bases.dtype() - ); - - let lanes = lane_count(bases.dtype().as_ptype()); - - vortex_ensure!( - deltas.len().is_multiple_of(1024), - "deltas length ({}) must be a multiple of 1024", - deltas.len(), - ); - vortex_ensure!( - bases.len().is_multiple_of(lanes), - "bases length ({}) must be a multiple of LANES ({lanes})", - bases.len(), - ); - Ok(()) - } - - pub(crate) unsafe fn new_unchecked(bases: ArrayRef, deltas: ArrayRef, offset: usize) -> Self { - Self { - offset, - slots: vec![Some(bases), Some(deltas)], - } - } - - #[inline] - pub fn bases(&self) -> &ArrayRef { - self.slots[BASES_SLOT] +pub trait DeltaArrayExt: TypedArrayRef { + fn bases(&self) -> &ArrayRef { + self.as_ref().slots()[BASES_SLOT] .as_ref() .vortex_expect("DeltaArray bases slot") } - #[inline] - pub fn deltas(&self) -> &ArrayRef { - self.slots[DELTAS_SLOT] + fn deltas(&self) -> &ArrayRef { + self.as_ref().slots()[DELTAS_SLOT] .as_ref() .vortex_expect("DeltaArray deltas slot") } - #[inline] - /// The logical offset into the first chunk of [`Self::deltas`]. - pub fn offset(&self) -> usize { + fn offset(&self) -> usize { self.offset } +} - pub(crate) fn bases_len(&self) -> usize { - self.bases().len() - } +impl> DeltaArrayExt for T {} - pub(crate) fn deltas_len(&self) -> usize { - self.deltas().len() +impl DeltaData { + pub fn try_new(offset: usize) -> VortexResult { + vortex_ensure!(offset < 1024, "offset must be less than 1024: {offset}"); + Ok(Self { offset }) } } diff --git a/encodings/fastlanes/src/delta/compute/cast.rs b/encodings/fastlanes/src/delta/compute/cast.rs index a9af6d09d60..4beb558dca0 100644 --- a/encodings/fastlanes/src/delta/compute/cast.rs +++ b/encodings/fastlanes/src/delta/compute/cast.rs @@ -12,6 +12,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_panic; use crate::delta::Delta; +use crate::delta::array::DeltaArrayExt; impl CastReduce for Delta { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // Delta encoding stores differences between consecutive values, which requires diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index 746221a5b1e..2dc2f040941 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::hash::Hash; +use std::hash::Hasher; use fastlanes::FastLanes; use prost::Message; @@ -24,6 +25,7 @@ use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::serde::ArrayChildren; use vortex_array::vtable; use vortex_array::vtable::VTable; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; @@ -31,9 +33,12 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::DeltaData; -use crate::delta::array::NUM_SLOTS; +use crate::delta::array::BASES_SLOT; +use crate::delta::array::DELTAS_SLOT; +use crate::delta::array::DeltaArrayExt; use crate::delta::array::SLOT_NAMES; use crate::delta::array::delta_decompress::delta_decompress; +use crate::delta::array::lane_count; mod operations; mod rules; @@ -51,6 +56,18 @@ pub struct DeltaMetadata { offset: u32, // must be <1024 } +impl ArrayHash for DeltaData { + fn array_hash(&self, state: &mut H, _precision: Precision) { + self.offset.hash(state); + } +} + +impl ArrayEq for DeltaData { + fn array_eq(&self, other: &Self, _precision: Precision) -> bool { + self.offset == other.offset + } +} + impl VTable for Delta { type ArrayData = DeltaData; @@ -61,20 +78,20 @@ impl VTable for Delta { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate(dtype, len) - } - - fn array_hash(array: &DeltaData, state: &mut H, precision: Precision) { - array.offset().hash(state); - array.bases().array_hash(state, precision); - array.deltas().array_hash(state, precision); - } - - fn array_eq(array: &DeltaData, other: &DeltaData, precision: Precision) -> bool { - array.offset() == other.offset() - && array.bases().array_eq(other.bases(), precision) - && array.deltas().array_eq(other.deltas(), precision) + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let bases = slots[BASES_SLOT] + .as_ref() + .vortex_expect("DeltaArray bases slot"); + let deltas = slots[DELTAS_SLOT] + .as_ref() + .vortex_expect("DeltaArray deltas slot"); + validate_parts(bases, deltas, data.offset, dtype, len) } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -97,25 +114,10 @@ impl VTable for Delta { rules::RULES.evaluate(array, parent, child_idx) } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "DeltaArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( DeltaMetadata { @@ -134,7 +136,7 @@ impl VTable for Delta { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { vortex_ensure!( buffers.is_empty(), "DeltaArray expects 0 buffers, got {}", @@ -159,7 +161,9 @@ impl VTable for Delta { let bases = children.get(0, dtype, bases_len)?; let deltas = children.get(1, dtype, deltas_len)?; - DeltaData::try_new(bases, deltas, metadata.offset as usize, len) + let data = DeltaData::try_new(metadata.offset as usize)?; + let slots = vec![Some(bases), Some(deltas)]; + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { @@ -182,8 +186,9 @@ impl Delta { len: usize, ) -> VortexResult { let dtype = bases.dtype().with_nullability(deltas.dtype().nullability()); - let data = DeltaData::try_new(bases, deltas, offset, len)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Delta, dtype, len, data)) }) + let data = DeltaData::try_new(offset)?; + let slots = vec![Some(bases), Some(deltas)]; + Array::try_from_parts(ArrayParts::new(Delta, dtype, len, data).with_slots(slots)) } /// Compress a primitive array using Delta encoding. @@ -197,6 +202,52 @@ impl Delta { } } +fn validate_parts( + bases: &ArrayRef, + deltas: &ArrayRef, + offset: usize, + dtype: &DType, + len: usize, +) -> VortexResult<()> { + vortex_ensure!( + offset + len <= deltas.len(), + "offset + len, {offset} + {len}, must be less than or equal to the size of deltas: {}", + deltas.len() + ); + vortex_ensure!( + bases.dtype().eq_ignore_nullability(deltas.dtype()), + "DeltaArray: bases and deltas must have the same dtype, got {} and {}", + bases.dtype(), + deltas.dtype() + ); + + vortex_ensure!( + bases.dtype().is_unsigned_int(), + "DeltaArray: dtype must be an unsigned integer, got {}", + bases.dtype() + ); + + let expected_dtype = bases.dtype().with_nullability(deltas.dtype().nullability()); + vortex_ensure!( + dtype == &expected_dtype, + "DeltaArray dtype mismatch: expected {expected_dtype}, got {dtype}" + ); + + let lanes = lane_count(bases.dtype().as_ptype()); + + vortex_ensure!( + deltas.len().is_multiple_of(1024), + "deltas length ({}) must be a multiple of 1024", + deltas.len(), + ); + vortex_ensure!( + bases.len().is_multiple_of(lanes), + "bases length ({}) must be a multiple of LANES ({lanes})", + bases.len(), + ); + Ok(()) +} + #[cfg(test)] mod tests { use prost::Message; diff --git a/encodings/fastlanes/src/delta/vtable/slice.rs b/encodings/fastlanes/src/delta/vtable/slice.rs index c836bed33fd..17b088dac92 100644 --- a/encodings/fastlanes/src/delta/vtable/slice.rs +++ b/encodings/fastlanes/src/delta/vtable/slice.rs @@ -10,6 +10,7 @@ use vortex_array::IntoArray; use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; +use crate::delta::array::DeltaArrayExt; use crate::delta::array::lane_count; use crate::delta::vtable::Delta; @@ -25,13 +26,11 @@ impl SliceReduce for Delta { let deltas = array.deltas(); let lanes = lane_count(array.dtype().as_ptype()); - let new_bases = bases.slice( - min(start_chunk * lanes, array.bases_len())..min(stop_chunk * lanes, array.bases_len()), - )?; + let new_bases = bases + .slice(min(start_chunk * lanes, bases.len())..min(stop_chunk * lanes, bases.len()))?; - let new_deltas = deltas.slice( - min(start_chunk * 1024, array.deltas_len())..min(stop_chunk * 1024, array.deltas_len()), - )?; + let new_deltas = deltas + .slice(min(start_chunk * 1024, deltas.len())..min(stop_chunk * 1024, deltas.len()))?; Ok(Some( Delta::try_new(new_bases, new_deltas, physical_start % 1024, range.len())?.into_array(), diff --git a/encodings/fastlanes/src/delta/vtable/validity.rs b/encodings/fastlanes/src/delta/vtable/validity.rs index d1e6fc28870..d6a87ff5956 100644 --- a/encodings/fastlanes/src/delta/vtable/validity.rs +++ b/encodings/fastlanes/src/delta/vtable/validity.rs @@ -10,6 +10,7 @@ use vortex_error::VortexResult; use crate::Delta; use crate::bit_transpose::untranspose_validity; +use crate::delta::array::DeltaArrayExt; impl ValidityVTable for Delta { fn validity(array: ArrayView<'_, Delta>) -> VortexResult { diff --git a/encodings/fastlanes/src/for/array/for_compress.rs b/encodings/fastlanes/src/for/array/for_compress.rs index 86bb6de2947..266e6306a9c 100644 --- a/encodings/fastlanes/src/for/array/for_compress.rs +++ b/encodings/fastlanes/src/for/array/for_compress.rs @@ -35,20 +35,14 @@ fn compress_primitive( ) -> VortexResult { // Set null values to the min value, ensuring that decompress into a value in the primitive // range (and stop them wrapping around). - let encoded = parray - .into_data() - .map_each_with_validity::(|(v, bool)| { - if bool { - v.wrapping_sub(&min) - } else { - T::zero() - } - })?; - Ok(PrimitiveArray::from_buffer_handle( - encoded.buffer_handle().clone(), - encoded.ptype(), - encoded.validity(), - )) + let encoded = parray.map_each_with_validity::(|(v, bool)| { + if bool { + v.wrapping_sub(&min) + } else { + T::zero() + } + })?; + Ok(encoded) } #[cfg(test)] @@ -58,6 +52,7 @@ mod test { use itertools::Itertools; use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; + use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::assert_arrays_eq; use vortex_array::dtype::PType; use vortex_array::expr::stats::StatsProvider; @@ -69,6 +64,7 @@ mod test { use super::*; use crate::BitPackedData; + use crate::r#for::array::FoRArrayExt; use crate::r#for::array::for_decompress::decompress; use crate::r#for::array::for_decompress::fused_decompress; diff --git a/encodings/fastlanes/src/for/array/for_decompress.rs b/encodings/fastlanes/src/for/array/for_decompress.rs index 11471162285..e19ef8d0ff3 100644 --- a/encodings/fastlanes/src/for/array/for_decompress.rs +++ b/encodings/fastlanes/src/for/array/for_decompress.rs @@ -21,6 +21,7 @@ use crate::BitPacked; use crate::BitPackedArrayExt; use crate::FoRArray; use crate::bitpack_decompress; +use crate::r#for::array::FoRArrayExt; use crate::unpack_iter::UnpackStrategy; use crate::unpack_iter::UnpackedChunks; diff --git a/encodings/fastlanes/src/for/array/mod.rs b/encodings/fastlanes/src/for/array/mod.rs index f2cd3108250..48a6b84345b 100644 --- a/encodings/fastlanes/src/for/array/mod.rs +++ b/encodings/fastlanes/src/for/array/mod.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::ArrayRef; -use vortex_array::dtype::DType; +use vortex_array::TypedArrayRef; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; use vortex_error::VortexExpect as _; @@ -23,82 +23,36 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["encoded"]; /// storage requirements when values are clustered around a specific point. #[derive(Clone, Debug)] pub struct FoRData { - pub(super) slots: Vec>, pub(super) reference: Scalar, } -impl FoRData { - pub(crate) fn try_new(encoded: ArrayRef, reference: Scalar) -> VortexResult { - Self::validate_parts(&encoded, &reference, reference.dtype(), encoded.len())?; - - Ok(Self { - slots: vec![Some(encoded)], - reference, - }) +pub trait FoRArrayExt: TypedArrayRef { + fn encoded(&self) -> &ArrayRef { + self.as_ref().slots()[ENCODED_SLOT] + .as_ref() + .vortex_expect("FoRArray encoded slot") } - pub(crate) fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { - Self::validate_parts(self.encoded(), &self.reference, dtype, len) + fn reference_scalar(&self) -> &Scalar { + &self.reference } +} - fn validate_parts( - encoded: &ArrayRef, - reference: &Scalar, - dtype: &DType, - len: usize, - ) -> VortexResult<()> { +impl> FoRArrayExt for T {} + +impl FoRData { + pub(crate) fn try_new(reference: Scalar) -> VortexResult { vortex_ensure!(!reference.is_null(), "Reference value cannot be null"); - vortex_ensure!(dtype.is_int(), "FoR requires an integer dtype, got {dtype}"); vortex_ensure!( - reference.dtype() == dtype, - "FoR reference dtype mismatch: expected {dtype}, got {}", + reference.dtype().is_int(), + "FoR requires an integer reference dtype, got {}", reference.dtype() ); - vortex_ensure!( - encoded.dtype() == dtype, - "FoR encoded dtype mismatch: expected {dtype}, got {}", - encoded.dtype() - ); - vortex_ensure!( - encoded.len() == len, - "FoR encoded length mismatch: expected {len}, got {}", - encoded.len() - ); - Ok(()) - } - - /// Returns the length of the array. - #[inline] - pub fn len(&self) -> usize { - self.encoded().len() - } - - /// Returns `true` if the array is empty. - #[inline] - pub fn is_empty(&self) -> bool { - self.encoded().is_empty() - } - - /// Returns the dtype of the array. - #[inline] - pub fn dtype(&self) -> &DType { - self.reference.dtype() + Ok(Self { reference }) } #[inline] pub fn ptype(&self) -> PType { - self.dtype().as_ptype() - } - - #[inline] - pub fn encoded(&self) -> &ArrayRef { - self.slots[ENCODED_SLOT] - .as_ref() - .vortex_expect("FoRArray encoded slot") - } - - #[inline] - pub fn reference_scalar(&self) -> &Scalar { - &self.reference + self.reference.dtype().as_ptype() } } diff --git a/encodings/fastlanes/src/for/compute/cast.rs b/encodings/fastlanes/src/for/compute/cast.rs index e37b25b64a1..e534f3c76fa 100644 --- a/encodings/fastlanes/src/for/compute/cast.rs +++ b/encodings/fastlanes/src/for/compute/cast.rs @@ -10,6 +10,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::r#for::FoR; +use crate::r#for::array::FoRArrayExt; impl CastReduce for FoR { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // FoR only supports integer types diff --git a/encodings/fastlanes/src/for/compute/compare.rs b/encodings/fastlanes/src/for/compute/compare.rs index 853bf4c2955..e5eae1b90b0 100644 --- a/encodings/fastlanes/src/for/compute/compare.rs +++ b/encodings/fastlanes/src/for/compute/compare.rs @@ -23,6 +23,7 @@ use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use crate::FoR; +use crate::r#for::array::FoRArrayExt; impl CompareKernel for FoR { fn compare( diff --git a/encodings/fastlanes/src/for/compute/is_constant.rs b/encodings/fastlanes/src/for/compute/is_constant.rs index 779e9bc900f..246a9c93226 100644 --- a/encodings/fastlanes/src/for/compute/is_constant.rs +++ b/encodings/fastlanes/src/for/compute/is_constant.rs @@ -11,6 +11,7 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::FoR; +use crate::r#for::array::FoRArrayExt; /// FoR-specific is_constant kernel. /// diff --git a/encodings/fastlanes/src/for/compute/is_sorted.rs b/encodings/fastlanes/src/for/compute/is_sorted.rs index 8e86dba1a2c..3b81c45c03d 100644 --- a/encodings/fastlanes/src/for/compute/is_sorted.rs +++ b/encodings/fastlanes/src/for/compute/is_sorted.rs @@ -15,6 +15,7 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::FoR; +use crate::r#for::array::FoRArrayExt; #[derive(Debug)] pub(crate) struct FoRIsSortedKernel; @@ -63,6 +64,7 @@ mod test { use vortex_buffer::buffer; use crate::FoRData; + use crate::r#for::array::FoRArrayExt; #[test] fn test_sorted() { diff --git a/encodings/fastlanes/src/for/compute/mod.rs b/encodings/fastlanes/src/for/compute/mod.rs index 8bf7e93d029..7307b50d776 100644 --- a/encodings/fastlanes/src/for/compute/mod.rs +++ b/encodings/fastlanes/src/for/compute/mod.rs @@ -16,6 +16,7 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::FoR; +use crate::r#for::array::FoRArrayExt; impl TakeExecute for FoR { fn take( diff --git a/encodings/fastlanes/src/for/mod.rs b/encodings/fastlanes/src/for/mod.rs index 99f19486e91..8a70fe30499 100644 --- a/encodings/fastlanes/src/for/mod.rs +++ b/encodings/fastlanes/src/for/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::FoRArrayExt; pub use array::FoRData; pub(crate) mod compute; diff --git a/encodings/fastlanes/src/for/vtable/mod.rs b/encodings/fastlanes/src/for/vtable/mod.rs index 157645c410f..e016d31f661 100644 --- a/encodings/fastlanes/src/for/vtable/mod.rs +++ b/encodings/fastlanes/src/for/vtable/mod.rs @@ -3,6 +3,7 @@ use std::fmt::Debug; use std::hash::Hash; +use std::hash::Hasher; use vortex_array::Array; use vortex_array::ArrayEq; @@ -24,6 +25,7 @@ use vortex_array::serde::ArrayChildren; use vortex_array::vtable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTableFromChild; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -31,7 +33,7 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::FoRData; -use crate::r#for::array::NUM_SLOTS; +use crate::r#for::array::FoRArrayExt; use crate::r#for::array::SLOT_NAMES; use crate::r#for::array::for_decompress::decompress; use crate::r#for::vtable::kernels::PARENT_KERNELS; @@ -45,6 +47,18 @@ mod validity; vtable!(FoR, FoR, FoRData); +impl ArrayHash for FoRData { + fn array_hash(&self, state: &mut H, _precision: Precision) { + self.reference.hash(state); + } +} + +impl ArrayEq for FoRData { + fn array_eq(&self, other: &Self, _precision: Precision) -> bool { + self.reference == other.reference + } +} + impl VTable for FoR { type ArrayData = FoRData; @@ -55,18 +69,15 @@ impl VTable for FoR { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate(dtype, len) - } - - fn array_hash(array: &FoRData, state: &mut H, precision: Precision) { - array.encoded().array_hash(state, precision); - array.reference_scalar().hash(state); - } - - fn array_eq(array: &FoRData, other: &FoRData, precision: Precision) -> bool { - array.encoded().array_eq(other.encoded(), precision) - && array.reference_scalar() == other.reference_scalar() + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let encoded = slots[0].as_ref().vortex_expect("FoRArray encoded slot"); + validate_parts(encoded.dtype(), encoded.len(), &data.reference, dtype, len) } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -81,25 +92,10 @@ impl VTable for FoR { None } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "FoRArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { // Note that we **only** serialize the optional scalar value (not including the dtype). Ok(Some(ScalarValue::to_proto_bytes( @@ -115,7 +111,7 @@ impl VTable for FoR { buffers: &[BufferHandle], children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { vortex_ensure!( buffers.is_empty(), "FoRArray expects 0 buffers, got {}", @@ -131,8 +127,10 @@ impl VTable for FoR { let scalar_value = ScalarValue::from_proto_bytes(metadata, dtype, session)?; let reference = Scalar::try_new(dtype.clone(), scalar_value)?; let encoded = children.get(0, dtype, len)?; + let slots = vec![Some(encoded)]; - FoRData::try_new(encoded, reference) + let data = FoRData::try_new(reference)?; + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn reduce_parent( @@ -171,8 +169,9 @@ impl FoR { .with_nullability(encoded.dtype().nullability()); let reference = reference.cast(&dtype)?; let len = encoded.len(); - let data = FoRData::try_new(encoded, reference)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(FoR, dtype, len, data)) }) + let data = FoRData::try_new(reference)?; + let slots = vec![Some(encoded)]; + Array::try_from_parts(ArrayParts::new(FoR, dtype, len, data).with_slots(slots)) } /// Encode a primitive array using Frame of Reference encoding. @@ -181,6 +180,32 @@ impl FoR { } } +fn validate_parts( + encoded_dtype: &DType, + encoded_len: usize, + reference: &Scalar, + dtype: &DType, + len: usize, +) -> VortexResult<()> { + vortex_ensure!(dtype.is_int(), "FoR requires an integer dtype, got {dtype}"); + vortex_ensure!( + reference.dtype() == dtype, + "FoR reference dtype mismatch: expected {dtype}, got {}", + reference.dtype() + ); + vortex_ensure!( + encoded_dtype == dtype, + "FoR encoded dtype mismatch: expected {dtype}, got {}", + encoded_dtype + ); + vortex_ensure!( + encoded_len == len, + "FoR encoded length mismatch: expected {len}, got {}", + encoded_len + ); + Ok(()) +} + #[cfg(test)] mod tests { use vortex_array::scalar::ScalarValue; diff --git a/encodings/fastlanes/src/for/vtable/operations.rs b/encodings/fastlanes/src/for/vtable/operations.rs index a91eb83dfa3..e90ff7f4ad2 100644 --- a/encodings/fastlanes/src/for/vtable/operations.rs +++ b/encodings/fastlanes/src/for/vtable/operations.rs @@ -10,6 +10,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use super::FoR; +use crate::r#for::array::FoRArrayExt; impl OperationsVTable for FoR { fn scalar_at( array: ArrayView<'_, FoR>, diff --git a/encodings/fastlanes/src/for/vtable/rules.rs b/encodings/fastlanes/src/for/vtable/rules.rs index 3f839fe74dd..8d220a86d7f 100644 --- a/encodings/fastlanes/src/for/vtable/rules.rs +++ b/encodings/fastlanes/src/for/vtable/rules.rs @@ -13,6 +13,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; use vortex_error::VortexResult; use crate::FoR; +use crate::r#for::array::FoRArrayExt; pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ // TODO: add BetweenReduceAdaptor(FoR) diff --git a/encodings/fastlanes/src/for/vtable/slice.rs b/encodings/fastlanes/src/for/vtable/slice.rs index acd8a4d0254..206c47ef509 100644 --- a/encodings/fastlanes/src/for/vtable/slice.rs +++ b/encodings/fastlanes/src/for/vtable/slice.rs @@ -10,6 +10,7 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::FoR; +use crate::r#for::array::FoRArrayExt; impl SliceReduce for FoR { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { diff --git a/encodings/fastlanes/src/for/vtable/validity.rs b/encodings/fastlanes/src/for/vtable/validity.rs index 2131f0ba2f4..dd2d28802ae 100644 --- a/encodings/fastlanes/src/for/vtable/validity.rs +++ b/encodings/fastlanes/src/for/vtable/validity.rs @@ -2,13 +2,14 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::ArrayRef; +use vortex_array::ArrayView; use vortex_array::vtable::ValidityChild; use super::FoR; -use crate::FoRData; +use crate::r#for::array::FoRArrayExt; impl ValidityChild for FoR { - fn validity_child(array: &FoRData) -> &ArrayRef { - array.encoded() + fn validity_child(array: ArrayView<'_, FoR>) -> ArrayRef { + array.encoded().clone() } } diff --git a/encodings/fastlanes/src/lib.rs b/encodings/fastlanes/src/lib.rs index b46624589e5..d34250a597a 100644 --- a/encodings/fastlanes/src/lib.rs +++ b/encodings/fastlanes/src/lib.rs @@ -8,6 +8,7 @@ pub use delta::*; pub use r#for::*; pub use rle::*; use vortex_array::ToCanonical; +use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index 2ee4a082dfa..ef0780fd6b3 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -2,15 +2,11 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::ArrayRef; -use vortex_array::dtype::DType; -use vortex_array::dtype::Nullability; -use vortex_array::dtype::PType; +use vortex_array::TypedArrayRef; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use vortex_error::vortex_ensure; -use crate::FL_CHUNK_SIZE; - pub mod rle_compress; pub mod rle_decompress; @@ -33,102 +29,11 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["values", "indices", "values_i #[derive(Clone, Debug)] pub struct RLEData { - pub(super) slots: Vec>, // Offset relative to the start of the chunk. pub(super) offset: usize, } impl RLEData { - fn validate( - values: &ArrayRef, - indices: &ArrayRef, - value_idx_offsets: &ArrayRef, - offset: usize, - length: usize, - ) -> VortexResult<()> { - Self::validate_parts(values, indices, value_idx_offsets, offset, length)?; - Ok(()) - } - - pub(crate) fn validate_against_outer(&self, dtype: &DType, length: usize) -> VortexResult<()> { - Self::validate_parts( - self.values(), - self.indices(), - self.values_idx_offsets(), - self.offset, - length, - )?; - let expected_dtype = DType::Primitive( - self.values().dtype().as_ptype(), - self.indices().dtype().nullability(), - ); - vortex_ensure!( - dtype == &expected_dtype, - "RLE dtype mismatch: expected {expected_dtype}, got {dtype}" - ); - Ok(()) - } - - fn validate_parts( - values: &ArrayRef, - indices: &ArrayRef, - value_idx_offsets: &ArrayRef, - offset: usize, - length: usize, - ) -> VortexResult<()> { - vortex_ensure!( - offset < 1024, - "Offset must be smaller than 1024, got {}", - offset - ); - - vortex_ensure!( - matches!( - values.dtype(), - DType::Primitive(_, Nullability::NonNullable) - ), - "RLE values must be a non-nullable primitive type, got {}", - values.dtype() - ); - - vortex_ensure!( - matches!(indices.dtype().as_ptype(), PType::U8 | PType::U16), - "RLE indices must be u8 or u16, got {}", - indices.dtype() - ); - - vortex_ensure!( - value_idx_offsets.dtype().is_unsigned_int() && !value_idx_offsets.dtype().is_nullable(), - "RLE value idx offsets must be non-nullable unsigned integer, got {}", - value_idx_offsets.dtype() - ); - vortex_ensure!( - indices.len().is_multiple_of(FL_CHUNK_SIZE), - "RLE indices length must be a multiple of {FL_CHUNK_SIZE}, got {}", - indices.len() - ); - vortex_ensure!( - offset + length <= indices.len(), - "RLE offset + length, {offset} + {length}, must not exceed the indices length {}", - indices.len() - ); - - vortex_ensure!( - indices.len().div_ceil(FL_CHUNK_SIZE) == value_idx_offsets.len(), - "RLE must have one value idx offset per chunk, got {}", - value_idx_offsets.len() - ); - - vortex_ensure!( - indices.len() >= values.len(), - "RLE must have at least as many indices as values, got {} indices and {} values", - indices.len(), - values.len() - ); - - Ok(()) - } - /// Create a new chunk-based RLE array from its components. /// /// # Arguments @@ -138,19 +43,13 @@ impl RLEData { /// * `values_idx_offsets` - Start indices for each value chunk. /// * `offset` - Offset into the first chunk /// * `length` - Array length - pub fn try_new( - values: ArrayRef, - indices: ArrayRef, - values_idx_offsets: ArrayRef, - offset: usize, - length: usize, - ) -> VortexResult { - Self::validate(&values, &indices, &values_idx_offsets, offset, length)?; - - Ok(Self { - slots: vec![Some(values), Some(indices), Some(values_idx_offsets)], - offset, - }) + pub fn try_new(offset: usize) -> VortexResult { + vortex_ensure!( + offset < 1024, + "Offset must be smaller than 1024, got {}", + offset + ); + Ok(Self { offset }) } /// Create a new RLEArray without validation. @@ -160,35 +59,34 @@ impl RLEData { /// - `offset + length` does not exceed the length of the indices array /// - The `indices` array contains valid indices into chunks of the `values` array /// - The `values_idx_offsets` array contains valid chunk start offsets - pub unsafe fn new_unchecked( - values: ArrayRef, - indices: ArrayRef, - values_idx_offsets: ArrayRef, - offset: usize, - ) -> Self { - Self { - slots: vec![Some(values), Some(indices), Some(values_idx_offsets)], - offset, - } + pub unsafe fn new_unchecked(offset: usize) -> Self { + Self { offset } } #[inline] - pub fn values(&self) -> &ArrayRef { - self.slots[VALUES_SLOT] + pub fn offset(&self) -> usize { + self.offset + } +} + +pub trait RLEArrayExt: TypedArrayRef { + #[inline] + fn values(&self) -> &ArrayRef { + self.as_ref().slots()[VALUES_SLOT] .as_ref() .vortex_expect("RLEArray values slot must be populated") } #[inline] - pub fn indices(&self) -> &ArrayRef { - self.slots[INDICES_SLOT] + fn indices(&self) -> &ArrayRef { + self.as_ref().slots()[INDICES_SLOT] .as_ref() .vortex_expect("RLEArray indices slot must be populated") } #[inline] - pub fn values_idx_offsets(&self) -> &ArrayRef { - self.slots[VALUES_IDX_OFFSETS_SLOT] + fn values_idx_offsets(&self) -> &ArrayRef { + self.as_ref().slots()[VALUES_IDX_OFFSETS_SLOT] .as_ref() .vortex_expect("RLEArray values_idx_offsets slot must be populated") } @@ -202,7 +100,7 @@ impl RLEData { clippy::expect_used, reason = "expect is safe here as scalar_at returns a valid primitive" )] - pub(crate) fn values_idx_offset(&self, chunk_idx: usize) -> usize { + fn values_idx_offset(&self, chunk_idx: usize) -> usize { self.values_idx_offsets() .scalar_at(chunk_idx) .expect("index must be in bounds") @@ -220,11 +118,13 @@ impl RLEData { /// Index offset into the array #[inline] - pub fn offset(&self) -> usize { + fn offset(&self) -> usize { self.offset } } +impl> RLEArrayExt for T {} + #[cfg(test)] mod tests { use vortex_array::ArrayContext; @@ -233,6 +133,7 @@ mod tests { use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; @@ -249,6 +150,7 @@ mod tests { use crate::FL_CHUNK_SIZE; use crate::RLE; use crate::RLEData; + use crate::rle::array::RLEArrayExt; use crate::test::SESSION; #[test] diff --git a/encodings/fastlanes/src/rle/array/rle_compress.rs b/encodings/fastlanes/src/rle/array/rle_compress.rs index 4663f13d182..b4ec00f6b7d 100644 --- a/encodings/fastlanes/src/rle/array/rle_compress.rs +++ b/encodings/fastlanes/src/rle/array/rle_compress.rs @@ -6,6 +6,7 @@ use fastlanes::RLE as FastLanesRLE; use vortex_array::IntoArray; use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::arrays::primitive::NativeValue; use vortex_array::dtype::NativePType; use vortex_array::match_each_native_ptype; @@ -149,6 +150,7 @@ mod tests { use vortex_error::VortexExpect; use super::*; + use crate::rle::array::RLEArrayExt; #[test] fn test_encode_decode() { diff --git a/encodings/fastlanes/src/rle/array/rle_decompress.rs b/encodings/fastlanes/src/rle/array/rle_decompress.rs index d448ca1de0b..356dcb06dbe 100644 --- a/encodings/fastlanes/src/rle/array/rle_decompress.rs +++ b/encodings/fastlanes/src/rle/array/rle_decompress.rs @@ -18,6 +18,7 @@ use vortex_error::vortex_panic; use crate::FL_CHUNK_SIZE; use crate::RLEArray; +use crate::rle::RLEArrayExt; /// Decompresses an RLE array back into a primitive array. #[expect( diff --git a/encodings/fastlanes/src/rle/compute/cast.rs b/encodings/fastlanes/src/rle/compute/cast.rs index c3ae7d42e81..95286d2f3e7 100644 --- a/encodings/fastlanes/src/rle/compute/cast.rs +++ b/encodings/fastlanes/src/rle/compute/cast.rs @@ -11,6 +11,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::rle::RLE; +use crate::rle::RLEArrayExt; impl CastReduce for RLE { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // Cast RLE values. diff --git a/encodings/fastlanes/src/rle/kernel.rs b/encodings/fastlanes/src/rle/kernel.rs index 65f8bfdb907..14eaf6dda3d 100644 --- a/encodings/fastlanes/src/rle/kernel.rs +++ b/encodings/fastlanes/src/rle/kernel.rs @@ -14,6 +14,7 @@ use vortex_error::VortexResult; use crate::FL_CHUNK_SIZE; use crate::RLE; +use crate::rle::RLEArrayExt; pub(crate) static PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ParentKernelSet::lift(&SliceExecuteAdaptor(RLE))]); diff --git a/encodings/fastlanes/src/rle/mod.rs b/encodings/fastlanes/src/rle/mod.rs index 1dd0b644156..1c580683908 100644 --- a/encodings/fastlanes/src/rle/mod.rs +++ b/encodings/fastlanes/src/rle/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::RLEArrayExt; pub use array::RLEData; mod compute; diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index 776fa1b84ad..7b7c64fa8ba 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::hash::Hash; +use std::hash::Hasher; use prost::Message; use vortex_array::Array; @@ -23,12 +24,17 @@ use vortex_array::dtype::PType; use vortex_array::serde::ArrayChildren; use vortex_array::vtable; use vortex_array::vtable::VTable; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::RLEData; +use crate::rle::array::INDICES_SLOT; +use crate::rle::array::RLEArrayExt; +use crate::rle::array::VALUES_IDX_OFFSETS_SLOT; +use crate::rle::array::VALUES_SLOT; use crate::rle::array::rle_decompress::rle_decompress; use crate::rle::kernel::PARENT_KERNELS; use crate::rle::vtable::rules::RULES; @@ -55,6 +61,18 @@ pub struct RLEMetadata { pub offset: u64, } +impl ArrayHash for RLEData { + fn array_hash(&self, state: &mut H, _precision: Precision) { + self.offset.hash(state); + } +} + +impl ArrayEq for RLEData { + fn array_eq(&self, other: &Self, _precision: Precision) -> bool { + self.offset == other.offset + } +} + impl VTable for RLE { type ArrayData = RLEData; @@ -65,24 +83,27 @@ impl VTable for RLE { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate_against_outer(dtype, len) - } - - fn array_hash(array: &RLEData, state: &mut H, precision: Precision) { - array.values().array_hash(state, precision); - array.indices().array_hash(state, precision); - array.values_idx_offsets().array_hash(state, precision); - array.offset().hash(state); - } - - fn array_eq(array: &RLEData, other: &RLEData, precision: Precision) -> bool { - array.values().array_eq(other.values(), precision) - && array.indices().array_eq(other.indices(), precision) - && array - .values_idx_offsets() - .array_eq(other.values_idx_offsets(), precision) - && array.offset() == other.offset() + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + validate_parts( + slots[VALUES_SLOT] + .as_ref() + .vortex_expect("RLEArray values slot must be populated"), + slots[INDICES_SLOT] + .as_ref() + .vortex_expect("RLEArray indices slot must be populated"), + slots[VALUES_IDX_OFFSETS_SLOT] + .as_ref() + .vortex_expect("RLEArray values_idx_offsets slot must be populated"), + data.offset, + dtype, + len, + ) } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -105,25 +126,10 @@ impl VTable for RLE { RULES.evaluate(array, parent, child_idx) } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { crate::rle::array::SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == crate::rle::array::NUM_SLOTS, - "RLEArray expects {} slots, got {}", - crate::rle::array::NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( RLEMetadata { @@ -147,7 +153,7 @@ impl VTable for RLE { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { vortex_ensure!( buffers.is_empty(), "RLEArray expects 0 buffers, got {}", @@ -175,13 +181,9 @@ impl VTable for RLE { usize::try_from(metadata.values_idx_offsets_len)?, )?; - RLEData::try_new( - values, - indices, - values_idx_offsets, - metadata.offset as usize, - len, - ) + let slots = vec![Some(values), Some(indices), Some(values_idx_offsets)]; + let data = RLEData::try_new(metadata.offset as usize)?; + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn execute_parent( @@ -214,14 +216,15 @@ impl RLE { length: usize, ) -> VortexResult { let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); - let data = RLEData::try_new(values, indices, values_idx_offsets, offset, length)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RLE, dtype, length, data)) }) + let slots = vec![Some(values), Some(indices), Some(values_idx_offsets)]; + let data = RLEData::try_new(offset)?; + Array::try_from_parts(ArrayParts::new(RLE, dtype, length, data).with_slots(slots)) } /// Create a new RLE array without validation. /// /// # Safety - /// See [`RLEData::new_unchecked`] for preconditions. + /// See [`RLE::validate`] for preconditions. pub unsafe fn new_unchecked( values: ArrayRef, indices: ArrayRef, @@ -230,8 +233,11 @@ impl RLE { length: usize, ) -> RLEArray { let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); - let data = unsafe { RLEData::new_unchecked(values, indices, values_idx_offsets, offset) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(RLE, dtype, length, data)) } + let slots = vec![Some(values), Some(indices), Some(values_idx_offsets)]; + let data = unsafe { RLEData::new_unchecked(offset) }; + unsafe { + Array::from_parts_unchecked(ArrayParts::new(RLE, dtype, length, data).with_slots(slots)) + } } /// Encode a primitive array using FastLanes RLE. @@ -240,6 +246,70 @@ impl RLE { } } +fn validate_parts( + values: &ArrayRef, + indices: &ArrayRef, + values_idx_offsets: &ArrayRef, + offset: usize, + dtype: &DType, + length: usize, +) -> VortexResult<()> { + vortex_ensure!( + matches!( + values.dtype(), + DType::Primitive(_, Nullability::NonNullable) + ), + "RLE values must be a non-nullable primitive type, got {}", + values.dtype() + ); + + vortex_ensure!( + matches!(indices.dtype().as_ptype(), PType::U8 | PType::U16), + "RLE indices must be u8 or u16, got {}", + indices.dtype() + ); + + vortex_ensure!( + values_idx_offsets.dtype().is_unsigned_int() && !values_idx_offsets.dtype().is_nullable(), + "RLE value idx offsets must be non-nullable unsigned integer, got {}", + values_idx_offsets.dtype() + ); + + vortex_ensure!( + indices.len().is_multiple_of(crate::FL_CHUNK_SIZE), + "RLE indices length must be a multiple of {}, got {}", + crate::FL_CHUNK_SIZE, + indices.len() + ); + + vortex_ensure!( + offset + length <= indices.len(), + "RLE offset + length, {offset} + {length}, must not exceed the indices length {}", + indices.len() + ); + + vortex_ensure!( + indices.len().div_ceil(crate::FL_CHUNK_SIZE) == values_idx_offsets.len(), + "RLE must have one value idx offset per chunk, got {}", + values_idx_offsets.len() + ); + + vortex_ensure!( + indices.len() >= values.len(), + "RLE must have at least as many indices as values, got {} indices and {} values", + indices.len(), + values.len() + ); + + let expected_dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); + vortex_ensure!( + dtype == &expected_dtype, + "RLE dtype mismatch: expected {expected_dtype}, got {dtype}" + ); + + Ok(()) +} + #[cfg(test)] mod tests { use prost::Message; diff --git a/encodings/fastlanes/src/rle/vtable/operations.rs b/encodings/fastlanes/src/rle/vtable/operations.rs index e8dfc159f70..25069ecccc1 100644 --- a/encodings/fastlanes/src/rle/vtable/operations.rs +++ b/encodings/fastlanes/src/rle/vtable/operations.rs @@ -10,6 +10,7 @@ use vortex_error::VortexResult; use super::RLE; use crate::FL_CHUNK_SIZE; +use crate::rle::RLEArrayExt; impl OperationsVTable for RLE { fn scalar_at( diff --git a/encodings/fastlanes/src/rle/vtable/validity.rs b/encodings/fastlanes/src/rle/vtable/validity.rs index ecef8047cb1..917b43da996 100644 --- a/encodings/fastlanes/src/rle/vtable/validity.rs +++ b/encodings/fastlanes/src/rle/vtable/validity.rs @@ -6,6 +6,7 @@ use vortex_array::vtable::ValidityVTable; use vortex_error::VortexResult; use crate::rle::RLE; +use crate::rle::RLEArrayExt; impl ValidityVTable for RLE { fn validity(array: ArrayView<'_, RLE>) -> VortexResult { diff --git a/encodings/fsst/benches/fsst_like.rs b/encodings/fsst/benches/fsst_like.rs index 12e78e2d7fb..b4b0e959351 100644 --- a/encodings/fsst/benches/fsst_like.rs +++ b/encodings/fsst/benches/fsst_like.rs @@ -11,7 +11,7 @@ use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; -use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnFactoryExt; use vortex_array::scalar_fn::fns::like::Like; use vortex_array::scalar_fn::fns::like::LikeOptions; use vortex_array::session::ArraySession; diff --git a/encodings/fsst/public-api.lock b/encodings/fsst/public-api.lock index 4989f13dca7..260e14d7f09 100644 --- a/encodings/fsst/public-api.lock +++ b/encodings/fsst/public-api.lock @@ -26,15 +26,11 @@ pub type vortex_fsst::FSST::ValidityVTable = vortex_array::array::vtable::validi pub fn vortex_fsst::FSST::append_to_builder(array: vortex_array::array::view::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_fsst::FSST::array_eq(array: &vortex_fsst::FSSTData, other: &vortex_fsst::FSSTData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_fsst::FSST::array_hash(array: &vortex_fsst::FSSTData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_fsst::FSST::buffer(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_fsst::FSST::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_fsst::FSST::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_fsst::FSST::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_fsst::FSST::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -50,11 +46,7 @@ pub fn vortex_fsst::FSST::serialize(array: vortex_array::array::view::ArrayView< pub fn vortex_fsst::FSST::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_fsst::FSST::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_fsst::FSST::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_fsst::FSST::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_fsst::FSST::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_fsst::FSST @@ -62,7 +54,7 @@ pub fn vortex_fsst::FSST::scalar_at(array: vortex_array::array::view::ArrayView< impl vortex_array::array::vtable::validity::ValidityChild for vortex_fsst::FSST -pub fn vortex_fsst::FSST::validity_child(array: &vortex_fsst::FSSTData) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_fsst::FSST::validity_child(array: vortex_array::array::view::ArrayView<'_, vortex_fsst::FSST>) -> vortex_array::array::erased::ArrayRef impl vortex_array::arrays::dict::take::TakeExecute for vortex_fsst::FSST @@ -108,13 +100,9 @@ pub fn vortex_fsst::FSSTData::symbol_lengths(&self) -> &vortex_buffer::buffer::B pub fn vortex_fsst::FSSTData::symbols(&self) -> &vortex_buffer::buffer::Buffer -pub fn vortex_fsst::FSSTData::try_new(symbols: vortex_buffer::buffer::Buffer, symbol_lengths: vortex_buffer::buffer::Buffer, codes: vortex_array::arrays::varbin::vtable::VarBinArray, uncompressed_lengths: vortex_array::array::erased::ArrayRef, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult +pub fn vortex_fsst::FSSTData::try_new(symbols: vortex_buffer::buffer::Buffer, symbol_lengths: vortex_buffer::buffer::Buffer, codes: vortex_array::arrays::varbin::vtable::VarBinArray, _dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult -pub fn vortex_fsst::FSSTData::uncompressed_lengths(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn vortex_fsst::FSSTData::uncompressed_lengths_dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_fsst::FSSTData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_fsst::FSSTData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_fsst::FSSTData @@ -124,6 +112,14 @@ impl core::fmt::Debug for vortex_fsst::FSSTData pub fn vortex_fsst::FSSTData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_fsst::FSSTData + +pub fn vortex_fsst::FSSTData::array_eq(&self, other: &Self, precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_fsst::FSSTData + +pub fn vortex_fsst::FSSTData::array_hash(&self, state: &mut H, precision: vortex_array::hash::Precision) + pub struct vortex_fsst::FSSTMetadata impl vortex_fsst::FSSTMetadata @@ -158,6 +154,18 @@ pub fn vortex_fsst::FSSTMetadata::clear(&mut self) pub fn vortex_fsst::FSSTMetadata::encoded_len(&self) -> usize +pub trait vortex_fsst::FSSTArrayExt: vortex_array::array::typed::TypedArrayRef + +pub fn vortex_fsst::FSSTArrayExt::uncompressed_lengths(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn vortex_fsst::FSSTArrayExt::uncompressed_lengths_dtype(&self) -> &vortex_array::dtype::DType + +impl> vortex_fsst::FSSTArrayExt for T + +pub fn T::uncompressed_lengths(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn T::uncompressed_lengths_dtype(&self) -> &vortex_array::dtype::DType + pub fn vortex_fsst::fsst_compress>(strings: A, len: usize, dtype: &vortex_array::dtype::DType, compressor: &fsst::Compressor) -> vortex_fsst::FSSTArray pub fn vortex_fsst::fsst_compress_iter<'a, I>(iter: I, len: usize, dtype: vortex_array::dtype::DType, compressor: &fsst::Compressor) -> vortex_fsst::FSSTArray where I: core::iter::traits::iterator::Iterator> diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index 417ea7b420e..c7880206ff4 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -3,6 +3,7 @@ use std::fmt::Debug; use std::fmt::Formatter; +use std::hash::Hasher; use std::sync::Arc; use std::sync::LazyLock; @@ -22,8 +23,10 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::TypedArrayRef; use vortex_array::arrays::VarBin; use vortex_array::arrays::VarBinArray; +use vortex_array::arrays::varbin::VarBinArrayExt; use vortex_array::buffer::BufferHandle; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::VarBinViewBuilder; @@ -70,6 +73,28 @@ impl FSSTMetadata { } } +impl ArrayHash for FSSTData { + fn array_hash(&self, state: &mut H, precision: Precision) { + self.symbols.array_hash(state, precision); + self.symbol_lengths.array_hash(state, precision); + self.codes.clone().into_array().array_hash(state, precision); + } +} + +impl ArrayEq for FSSTData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + self.symbols.array_eq(&other.symbols, precision) + && self + .symbol_lengths + .array_eq(&other.symbol_lengths, precision) + && self + .codes + .clone() + .into_array() + .array_eq(&other.codes.clone().into_array(), precision) + } +} + impl VTable for FSST { type ArrayData = FSSTData; type OperationsVTable = Self; @@ -79,34 +104,14 @@ impl VTable for FSST { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate(dtype, len) - } - - fn array_hash(array: &FSSTData, state: &mut H, precision: Precision) { - array.symbols.array_hash(state, precision); - array.symbol_lengths.array_hash(state, precision); - array - .codes - .clone() - .into_array() - .array_hash(state, precision); - array.uncompressed_lengths().array_hash(state, precision); - } - - fn array_eq(array: &FSSTData, other: &FSSTData, precision: Precision) -> bool { - array.symbols.array_eq(&other.symbols, precision) - && array - .symbol_lengths - .array_eq(&other.symbol_lengths, precision) - && array - .codes - .clone() - .into_array() - .array_eq(&other.codes.clone().into_array(), precision) - && array - .uncompressed_lengths() - .array_eq(other.uncompressed_lengths(), precision) + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + data.validate(dtype, len, slots) } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -149,7 +154,7 @@ impl VTable for FSST { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = FSSTMetadata::decode(metadata)?; let symbols = Buffer::::from_byte_buffer(buffers[0].clone().try_to_host_sync()?); let symbol_lengths = Buffer::::from_byte_buffer(buffers[1].clone().try_to_host_sync()?); @@ -178,7 +183,17 @@ impl VTable for FSST { len, )?; - return FSSTData::try_new(symbols, symbol_lengths, codes, uncompressed_lengths, dtype); + FSSTData::validate_parts( + &symbols, + &symbol_lengths, + &codes, + &uncompressed_lengths, + dtype, + len, + )?; + let slots = FSSTData::make_slots(&codes, &uncompressed_lengths); + let data = FSSTData::try_new(symbols, symbol_lengths, codes, dtype)?; + return Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)); } // Check for the current deserialization path. @@ -219,7 +234,17 @@ impl VTable for FSST { codes_validity, )?; - return FSSTData::try_new(symbols, symbol_lengths, codes, uncompressed_lengths, dtype); + FSSTData::validate_parts( + &symbols, + &symbol_lengths, + &codes, + &uncompressed_lengths, + dtype, + len, + )?; + let slots = FSSTData::make_slots(&codes, &uncompressed_lengths); + let data = FSSTData::try_new(symbols, symbol_lengths, codes, dtype)?; + return Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)); } vortex_bail!( @@ -228,43 +253,10 @@ impl VTable for FSST { ); } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "FSSTArray expects {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - - // Rebuild the codes VarBinArray from the slots - let codes_offsets = slots[CODES_OFFSETS_SLOT] - .clone() - .vortex_expect("FSSTArray requires codes_offsets slot"); - let codes_validity = match &slots[CODES_VALIDITY_SLOT] { - Some(v) => Validity::Array(v.clone()), - None => Validity::from(array.codes.dtype().nullability()), - }; - let codes = VarBinArray::try_new( - codes_offsets, - array.codes.bytes().clone(), - array.codes.dtype().clone(), - codes_validity, - )?; - array.codes = codes; - array.codes_array = array.codes.clone().into_array(); - array.slots = slots; - - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { canonicalize_fsst(array.as_view(), ctx).map(ExecutionResult::done) } @@ -287,7 +279,7 @@ impl VTable for FSST { // Decompress the whole block of data into a new buffer, and create some views // from it instead. - let (buffers, views) = fsst_decode_views(&array, builder.completed_block_count(), ctx)?; + let (buffers, views) = fsst_decode_views(array, builder.completed_block_count(), ctx)?; builder.push_buffer_and_adjusted_views(&buffers, &views, array.array().validity_mask()?); Ok(()) @@ -314,8 +306,10 @@ impl VTable for FSST { /// Lengths of the original values before compression, can be compressed. pub(crate) const UNCOMPRESSED_LENGTHS_SLOT: usize = 0; /// The offsets array for the FSST-compressed codes. +#[allow(dead_code, reason = "reserved for back-compat slot numbering")] pub(crate) const CODES_OFFSETS_SLOT: usize = 1; /// The validity bitmap for the compressed codes. +#[allow(dead_code, reason = "reserved for back-compat slot numbering")] pub(crate) const CODES_VALIDITY_SLOT: usize = 2; pub(crate) const NUM_SLOTS: usize = 3; pub(crate) const SLOT_NAMES: [&str; NUM_SLOTS] = @@ -328,8 +322,6 @@ pub struct FSSTData { codes: VarBinArray, /// NOTE(ngates): this === codes, but is stored as an ArrayRef so we can return &ArrayRef! codes_array: ArrayRef, - /// Lengths of the original values before compression, can be compressed. - slots: Vec>, /// Memoized compressor used for push-down of compute by compressing the RHS. compressor: Arc Compressor + Send>>>, @@ -341,7 +333,7 @@ impl Debug for FSSTData { .field("symbols", &self.symbols) .field("symbol_lengths", &self.symbol_lengths) .field("codes", &self.codes) - .field("uncompressed_lengths", self.uncompressed_lengths()) + .field("uncompressed_lengths", &"") .finish() } } @@ -361,8 +353,28 @@ impl FSST { uncompressed_lengths: ArrayRef, ) -> VortexResult { let len = codes.len(); - let data = FSSTData::try_new(symbols, symbol_lengths, codes, uncompressed_lengths, &dtype)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(FSST, dtype, len, data)) }) + FSSTData::validate_parts( + &symbols, + &symbol_lengths, + &codes, + &uncompressed_lengths, + &dtype, + len, + )?; + let slots = vec![ + Some(uncompressed_lengths), + Some(codes.offsets().clone()), + validity_to_child( + &codes + .validity() + .vortex_expect("FSST codes validity should be derivable"), + codes.len(), + ), + ]; + let data = FSSTData::try_new(symbols, symbol_lengths, codes, &dtype)?; + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(FSST, dtype, len, data).with_slots(slots)) + }) } pub(crate) unsafe fn new_unchecked( @@ -373,14 +385,37 @@ impl FSST { uncompressed_lengths: ArrayRef, ) -> FSSTArray { let len = codes.len(); - let data = unsafe { - FSSTData::new_unchecked(symbols, symbol_lengths, codes, uncompressed_lengths) - }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(FSST, dtype, len, data)) } + let slots = vec![ + Some(uncompressed_lengths), + Some(codes.offsets().clone()), + validity_to_child( + &codes + .validity() + .vortex_expect("FSST codes validity should be derivable"), + codes.len(), + ), + ]; + let data = unsafe { FSSTData::new_unchecked(symbols, symbol_lengths, codes) }; + unsafe { + Array::from_parts_unchecked(ArrayParts::new(FSST, dtype, len, data).with_slots(slots)) + } } } impl FSSTData { + fn make_slots(codes: &VarBinArray, uncompressed_lengths: &ArrayRef) -> Vec> { + vec![ + Some(uncompressed_lengths.clone()), + Some(codes.offsets().clone()), + validity_to_child( + &codes + .validity() + .vortex_expect("FSST codes validity should be derivable"), + codes.len(), + ), + ] + } + /// Build an FSST array from a set of `symbols` and `codes`. /// /// Symbols are 8-bytes and can represent short strings, each of which is assigned @@ -393,35 +428,23 @@ impl FSSTData { symbols: Buffer, symbol_lengths: Buffer, codes: VarBinArray, - uncompressed_lengths: ArrayRef, - dtype: &DType, + _dtype: &DType, ) -> VortexResult { - Self::validate_parts( - &symbols, - &symbol_lengths, - &codes, - &uncompressed_lengths, - dtype, - codes.len(), - )?; - // SAFETY: all components validated above - unsafe { - Ok(Self::new_unchecked( - symbols, - symbol_lengths, - codes, - uncompressed_lengths, - )) - } + unsafe { Ok(Self::new_unchecked(symbols, symbol_lengths, codes)) } } - pub fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { + pub fn validate( + &self, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { Self::validate_parts( &self.symbols, &self.symbol_lengths, &self.codes, - self.uncompressed_lengths(), + uncompressed_lengths_from_slots(slots), dtype, len, ) @@ -475,7 +498,6 @@ impl FSSTData { symbols: Buffer, symbol_lengths: Buffer, codes: VarBinArray, - uncompressed_lengths: ArrayRef, ) -> Self { let symbols2 = symbols.clone(); let symbol_lengths2 = symbol_lengths.clone(); @@ -484,24 +506,12 @@ impl FSSTData { }) as Box Compressor + Send>)); let codes_array = codes.clone().into_array(); - let codes_offsets_slot = Some(codes.offsets().clone()); - let codes_validity_slot = validity_to_child( - &codes - .validity() - .vortex_expect("FSST codes validity should be derivable"), - codes.len(), - ); Self { symbols, symbol_lengths, codes, codes_array, - slots: vec![ - Some(uncompressed_lengths), - codes_offsets_slot, - codes_validity_slot, - ], compressor, } } @@ -537,19 +547,6 @@ impl FSSTData { self.codes.dtype() } - /// Get the uncompressed length for each element in the array. - pub fn uncompressed_lengths(&self) -> &ArrayRef { - self.slots[UNCOMPRESSED_LENGTHS_SLOT] - .as_ref() - .vortex_expect("FSSTArray uncompressed_lengths slot") - } - - /// Get the DType of the uncompressed lengths array - #[inline] - pub fn uncompressed_lengths_dtype(&self) -> &DType { - self.uncompressed_lengths().dtype() - } - /// Build a [`Decompressor`][fsst::Decompressor] that can be used to decompress values from /// this array. pub fn decompressor(&self) -> Decompressor<'_> { @@ -562,9 +559,27 @@ impl FSSTData { } } +fn uncompressed_lengths_from_slots(slots: &[Option]) -> &ArrayRef { + slots[UNCOMPRESSED_LENGTHS_SLOT] + .as_ref() + .vortex_expect("FSSTArray uncompressed_lengths slot") +} + +pub trait FSSTArrayExt: TypedArrayRef { + fn uncompressed_lengths(&self) -> &ArrayRef { + uncompressed_lengths_from_slots(self.as_ref().slots()) + } + + fn uncompressed_lengths_dtype(&self) -> &DType { + self.uncompressed_lengths().dtype() + } +} + +impl> FSSTArrayExt for T {} + impl ValidityChild for FSST { - fn validity_child(array: &FSSTData) -> &ArrayRef { - &array.codes_array + fn validity_child(array: ArrayView<'_, FSST>) -> ArrayRef { + array.codes_array.clone() } } @@ -588,6 +603,7 @@ mod test { use vortex_error::VortexError; use crate::FSST; + use crate::array::FSSTArrayExt; use crate::array::FSSTMetadata; use crate::fsst_compress_iter; diff --git a/encodings/fsst/src/canonical.rs b/encodings/fsst/src/canonical.rs index d534ec22173..b37a56f4613 100644 --- a/encodings/fsst/src/canonical.rs +++ b/encodings/fsst/src/canonical.rs @@ -9,6 +9,7 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::varbin::VarBinArrayExt; use vortex_array::arrays::varbinview::build_views::BinaryView; use vortex_array::arrays::varbinview::build_views::MAX_BUFFER_LEN; use vortex_array::arrays::varbinview::build_views::build_views; @@ -19,13 +20,13 @@ use vortex_buffer::ByteBufferMut; use vortex_error::VortexResult; use crate::FSST; -use crate::FSSTData; +use crate::FSSTArrayExt; pub(super) fn canonicalize_fsst( array: ArrayView<'_, FSST>, ctx: &mut ExecutionCtx, ) -> VortexResult { - let (buffers, views) = fsst_decode_views(array.data(), 0, ctx)?; + let (buffers, views) = fsst_decode_views(array, 0, ctx)?; // SAFETY: FSST already validates the bytes for binary/UTF-8. We build views directly on // top of them, so the view pointers will all be valid. Ok(unsafe { @@ -40,7 +41,7 @@ pub(super) fn canonicalize_fsst( } pub(crate) fn fsst_decode_views( - fsst_array: &FSSTData, + fsst_array: ArrayView<'_, FSST>, start_buf_index: u32, ctx: &mut ExecutionCtx, ) -> VortexResult<(Vec, Buffer)> { diff --git a/encodings/fsst/src/compute/cast.rs b/encodings/fsst/src/compute/cast.rs index 82034daab21..de7502550fa 100644 --- a/encodings/fsst/src/compute/cast.rs +++ b/encodings/fsst/src/compute/cast.rs @@ -11,6 +11,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::FSST; +use crate::FSSTArrayExt; impl CastReduce for FSST { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // FSST is a string compression encoding. diff --git a/encodings/fsst/src/compute/compare.rs b/encodings/fsst/src/compute/compare.rs index 14ea2cbe4d6..6acf6a8fa3d 100644 --- a/encodings/fsst/src/compute/compare.rs +++ b/encodings/fsst/src/compute/compare.rs @@ -21,6 +21,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use crate::FSST; +use crate::FSSTArrayExt; impl CompareKernel for FSST { fn compare( lhs: ArrayView<'_, Self>, diff --git a/encodings/fsst/src/compute/filter.rs b/encodings/fsst/src/compute/filter.rs index 0a2aa130d7a..bb450fe9eb1 100644 --- a/encodings/fsst/src/compute/filter.rs +++ b/encodings/fsst/src/compute/filter.rs @@ -12,6 +12,7 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::FSST; +use crate::FSSTArrayExt; impl FilterKernel for FSST { fn filter( @@ -25,7 +26,7 @@ impl FilterKernel for FSST { let filtered_codes_ref = ::filter(codes, mask, ctx)? .vortex_expect("VarBin filter kernel always returns Some"); let filtered_codes = filtered_codes_ref - .try_into::() + .try_downcast::() .ok() .vortex_expect("must be VarBin"); diff --git a/encodings/fsst/src/compute/like.rs b/encodings/fsst/src/compute/like.rs index 45f6a7f9d9d..de4e2a53f6c 100644 --- a/encodings/fsst/src/compute/like.rs +++ b/encodings/fsst/src/compute/like.rs @@ -9,6 +9,7 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::ToCanonical; use vortex_array::arrays::BoolArray; +use vortex_array::arrays::varbin::VarBinArrayExt; use vortex_array::match_each_integer_ptype; use vortex_array::scalar_fn::fns::like::LikeKernel; use vortex_array::scalar_fn::fns::like::LikeOptions; @@ -89,7 +90,7 @@ mod tests { use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::VarBinArray; - use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; + use vortex_array::arrays::scalar_fn::ScalarFnFactoryExt; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; diff --git a/encodings/fsst/src/compute/mod.rs b/encodings/fsst/src/compute/mod.rs index 1bc1df71a29..ac350e6186c 100644 --- a/encodings/fsst/src/compute/mod.rs +++ b/encodings/fsst/src/compute/mod.rs @@ -19,6 +19,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_err; use crate::FSST; +use crate::FSSTArrayExt; impl TakeExecute for FSST { fn take( @@ -40,7 +41,7 @@ impl TakeExecute for FSST { ::take(codes, indices, _ctx)? .vortex_expect("VarBin take kernel always returns Some") } - .try_into::() + .try_downcast::() .map_err(|_| vortex_err!("take for codes must return varbin array"))?, array .uncompressed_lengths() diff --git a/encodings/fsst/src/dfa/tests.rs b/encodings/fsst/src/dfa/tests.rs index 0d61d92090a..caf04adb3c6 100644 --- a/encodings/fsst/src/dfa/tests.rs +++ b/encodings/fsst/src/dfa/tests.rs @@ -13,7 +13,7 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::VarBinArray; -use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnFactoryExt; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; diff --git a/encodings/fsst/src/slice.rs b/encodings/fsst/src/slice.rs index 5e0dd21a9f3..9b7b2833d85 100644 --- a/encodings/fsst/src/slice.rs +++ b/encodings/fsst/src/slice.rs @@ -12,6 +12,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_err; use crate::FSST; +use crate::FSSTArrayExt; impl SliceReduce for FSST { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { @@ -25,7 +26,7 @@ impl SliceReduce for FSST { array .codes() .slice(range.clone())? - .try_into::() + .try_downcast::() .map_err(|_| vortex_err!("cannot fail conversion"))?, array.uncompressed_lengths().slice(range)?, ) diff --git a/encodings/parquet-variant/src/array.rs b/encodings/parquet-variant/src/array.rs index 932274d77f8..60dd4a18794 100644 --- a/encodings/parquet-variant/src/array.rs +++ b/encodings/parquet-variant/src/array.rs @@ -12,6 +12,7 @@ use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::TypedArrayRef; use vortex_array::arrays::VariantArray; use vortex_array::arrow::ArrowArrayExecutor; use vortex_array::arrow::FromArrowArray; @@ -67,10 +68,7 @@ pub(crate) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity", "metadata", "value /// 3. **Typed-value child nullability**: the `typed_value` child carries its own `DType` /// (which includes nullability). #[derive(Clone, Debug)] -pub struct ParquetVariantData { - pub(crate) nullability: Nullability, - pub(crate) slots: Vec>, -} +pub struct ParquetVariantData; impl ParquetVariant { pub fn try_new( @@ -81,24 +79,7 @@ impl ParquetVariant { ) -> VortexResult> { let len = metadata.len(); let dtype = DType::Variant(validity.nullability()); - let data = ParquetVariantData::try_new(validity, metadata, value, typed_value)?; - Ok(unsafe { - Array::from_parts_unchecked(ArrayParts::new(ParquetVariant, dtype, len, data)) - }) - } -} - -impl ParquetVariantData { - /// Creates a Parquet Variant array with explicit outer validity. - pub fn try_new( - validity: Validity, - metadata: ArrayRef, - value: Option, - typed_value: Option, - ) -> VortexResult { - let len = metadata.len(); - let dtype = DType::Variant(validity.nullability()); - Self::validate_parts( + validate_parts( &validity, &metadata, value.as_ref(), @@ -106,29 +87,19 @@ impl ParquetVariantData { &dtype, len, )?; - - let validity_child = validity_to_child(&validity, len); - let slots = vec![validity_child, Some(metadata), value, typed_value]; - - Ok(Self { - nullability: validity.nullability(), - slots, - }) - } - - pub(crate) fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { - let validity = self.validity(); - Self::validate_parts( - &validity, - self.metadata_array(), - self.value_array(), - self.typed_value_array(), - dtype, - len, - ) + let slots = vec![ + validity_to_child(&validity, len), + Some(metadata), + value, + typed_value, + ]; + let data = ParquetVariantData; + Array::try_from_parts(ArrayParts::new(ParquetVariant, dtype, len, data).with_slots(slots)) } +} - fn validate_parts( +impl ParquetVariantData { + pub(crate) fn validate_parts( validity: &Validity, metadata: &ArrayRef, value: Option<&ArrayRef>, @@ -178,27 +149,6 @@ impl ParquetVariantData { Ok(()) } - /// Returns a reference to the metadata child array. - pub fn metadata_array(&self) -> &ArrayRef { - self.slots[METADATA_SLOT] - .as_ref() - .vortex_expect("ParquetVariantArray metadata slot") - } - - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) - } - - /// Returns a reference to the un-shredded value child array, if present. - pub fn value_array(&self) -> Option<&ArrayRef> { - self.slots[VALUE_SLOT].as_ref() - } - - /// Returns a reference to the shredded typed_value child array, if present. - pub fn typed_value_array(&self) -> Option<&ArrayRef> { - self.slots[TYPED_VALUE_SLOT].as_ref() - } - /// Converts an Arrow `parquet_variant_compute::VariantArray` into a Vortex `ArrayRef` /// wrapping `VariantArray(ParquetVariantArray(...))`. pub fn from_arrow_variant(arrow_variant: &ArrowVariantArray) -> VortexResult { @@ -241,9 +191,42 @@ impl ParquetVariantData { let pv = ParquetVariant::try_new(validity, metadata, value, typed_value)?; Ok(VariantArray::new(pv.into_array()).into_array()) } +} - /// Converts this array back to an Arrow [`parquet_variant_compute::VariantArray`]. - pub fn to_arrow(&self, ctx: &mut ExecutionCtx) -> VortexResult { +pub(crate) fn validate_parts( + validity: &Validity, + metadata: &ArrayRef, + value: Option<&ArrayRef>, + typed_value: Option<&ArrayRef>, + dtype: &DType, + len: usize, +) -> VortexResult<()> { + ParquetVariantData::validate_parts(validity, metadata, value, typed_value, dtype, len) +} + +pub trait ParquetVariantArrayExt: TypedArrayRef { + fn metadata_array(&self) -> &ArrayRef { + self.as_ref().slots()[METADATA_SLOT] + .as_ref() + .vortex_expect("ParquetVariantArray metadata slot") + } + + fn validity(&self) -> Validity { + child_to_validity( + &self.as_ref().slots()[VALIDITY_SLOT], + self.as_ref().dtype().nullability(), + ) + } + + fn value_array(&self) -> Option<&ArrayRef> { + self.as_ref().slots()[VALUE_SLOT].as_ref() + } + + fn typed_value_array(&self) -> Option<&ArrayRef> { + self.as_ref().slots()[TYPED_VALUE_SLOT].as_ref() + } + + fn to_arrow(&self, ctx: &mut ExecutionCtx) -> VortexResult { let metadata = self.metadata_array(); let len = metadata.len(); let nulls = to_arrow_null_buffer(self.validity(), len, ctx)?; @@ -284,6 +267,8 @@ impl ParquetVariantData { } } +impl> ParquetVariantArrayExt for T {} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -305,6 +290,7 @@ mod tests { use vortex_array::VortexSessionExecute; use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::Variant; + use vortex_array::arrays::variant::VariantArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::validity::Validity; @@ -313,6 +299,7 @@ mod tests { use crate::ParquetVariant; use crate::ParquetVariantData; + use crate::array::ParquetVariantArrayExt; fn assert_arrow_variant_storage_roundtrip(struct_array: StructArray) -> VortexResult<()> { let arrow_variant = ArrowVariantArray::try_new(&struct_array).unwrap(); @@ -418,8 +405,7 @@ mod tests { fn test_to_arrow_basic() -> VortexResult<()> { let metadata = VarBinViewArray::from_iter_bin([b"\x01\x00", b"\x01\x00"]).into_array(); let value = VarBinViewArray::from_iter_bin([b"\x10", b"\x11"]).into_array(); - let pv_array = - ParquetVariantData::try_new(Validity::NonNullable, metadata, Some(value), None)?; + let pv_array = ParquetVariant::try_new(Validity::NonNullable, metadata, Some(value), None)?; let mut ctx = LEGACY_SESSION.create_execution_ctx(); let variant_arr = pv_array.to_arrow(&mut ctx)?; @@ -436,7 +422,7 @@ mod tests { let metadata = VarBinViewArray::from_iter_bin([b"\x01\x00", b"\x01\x00"]).into_array(); let value = VarBinViewArray::from_iter_bin([b"\x10", b"\x11"]).into_array(); let typed_value = buffer![1i32, 2].into_array(); - let pv_array = ParquetVariantData::try_new( + let pv_array = ParquetVariant::try_new( Validity::NonNullable, metadata, Some(value), diff --git a/encodings/parquet-variant/src/kernel.rs b/encodings/parquet-variant/src/kernel.rs index 6061157c432..d47d4e31267 100644 --- a/encodings/parquet-variant/src/kernel.rs +++ b/encodings/parquet-variant/src/kernel.rs @@ -18,6 +18,7 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::ParquetVariant; +use crate::ParquetVariantArrayExt; pub(crate) static PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ ParentKernelSet::lift(&FilterExecuteAdaptor(ParquetVariant)), diff --git a/encodings/parquet-variant/src/lib.rs b/encodings/parquet-variant/src/lib.rs index 42c97d28870..9ecc9216daa 100644 --- a/encodings/parquet-variant/src/lib.rs +++ b/encodings/parquet-variant/src/lib.rs @@ -30,6 +30,7 @@ mod operations; mod validity; mod vtable; +pub use array::ParquetVariantArrayExt; pub use array::ParquetVariantData; pub use vtable::ParquetVariant; pub use vtable::ParquetVariantArray; diff --git a/encodings/parquet-variant/src/operations.rs b/encodings/parquet-variant/src/operations.rs index 67b6c78016b..2efad44ae6e 100644 --- a/encodings/parquet-variant/src/operations.rs +++ b/encodings/parquet-variant/src/operations.rs @@ -26,6 +26,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_err; +use crate::ParquetVariantArrayExt; use crate::vtable::ParquetVariant; impl OperationsVTable for ParquetVariant { @@ -348,6 +349,7 @@ mod tests { use parquet_variant_compute::VariantArrayBuilder; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Variant; + use vortex_array::arrays::variant::VariantArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; @@ -355,6 +357,7 @@ mod tests { use vortex_error::VortexResult; use crate::ParquetVariant; + use crate::ParquetVariantArrayExt; use crate::ParquetVariantData; use crate::operations::parquet_variant_to_scalar; diff --git a/encodings/parquet-variant/src/validity.rs b/encodings/parquet-variant/src/validity.rs index 50994d8b8a7..b28c7102e22 100644 --- a/encodings/parquet-variant/src/validity.rs +++ b/encodings/parquet-variant/src/validity.rs @@ -6,10 +6,11 @@ use vortex_array::validity::Validity; use vortex_array::vtable::ValidityVTable; use vortex_error::VortexResult; +use crate::ParquetVariantArrayExt; use crate::vtable::ParquetVariant; impl ValidityVTable for ParquetVariant { fn validity(array: ArrayView<'_, ParquetVariant>) -> VortexResult { - Ok(array.data().validity()) + Ok(ParquetVariantArrayExt::validity(&array)) } } diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs index 370a5cfbda0..bc89d9337c8 100644 --- a/encodings/parquet-variant/src/vtable.rs +++ b/encodings/parquet-variant/src/vtable.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::hash::Hash; use std::hash::Hasher; use prost::Message; @@ -9,6 +8,7 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -23,6 +23,8 @@ use vortex_array::serde::ArrayChildren; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; +use vortex_array::vtable::child_to_validity; +use vortex_array::vtable::validity_to_child; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; @@ -30,9 +32,14 @@ use vortex_error::vortex_panic; use vortex_proto::dtype as pb; use vortex_session::VortexSession; -use crate::array::NUM_SLOTS; +use crate::array::METADATA_SLOT; +use crate::array::ParquetVariantArrayExt; use crate::array::ParquetVariantData; use crate::array::SLOT_NAMES; +use crate::array::TYPED_VALUE_SLOT; +use crate::array::VALIDITY_SLOT; +use crate::array::VALUE_SLOT; +use crate::array::validate_parts; use crate::kernel::PARENT_KERNELS; /// VTable for [`ParquetVariantArray`]. @@ -67,51 +74,26 @@ impl VTable for ParquetVariant { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate(dtype, len) - } - - fn array_hash(array: &ParquetVariantData, state: &mut H, precision: Precision) { - array.validity().array_hash(state, precision); - array.metadata_array().array_hash(state, precision); - // Hash discriminators so that (value=Some, typed_value=None) and - // (value=None, typed_value=Some) produce different hashes. - array.value_array().is_some().hash(state); - if let Some(value) = array.value_array() { - value.array_hash(state, precision); - } - array.typed_value_array().is_some().hash(state); - if let Some(typed_value) = array.typed_value_array() { - typed_value.array_hash(state, precision); - } - } - - fn array_eq( - array: &ParquetVariantData, - other: &ParquetVariantData, - precision: Precision, - ) -> bool { - if !array.validity().array_eq(&other.validity(), precision) - || !array - .metadata_array() - .array_eq(other.metadata_array(), precision) - { - return false; - } - match (array.value_array(), other.value_array()) { - (Some(a), Some(b)) => { - if !a.array_eq(b, precision) { - return false; - } - } - (None, None) => {} - _ => return false, - } - match (array.typed_value_array(), other.typed_value_array()) { - (Some(a), Some(b)) => a.array_eq(b, precision), - (None, None) => true, - _ => false, - } + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let _ = data; + let validity = child_to_validity(&slots[VALIDITY_SLOT], dtype.nullability()); + let metadata = slots[METADATA_SLOT] + .as_ref() + .ok_or_else(|| vortex_err!("ParquetVariantArray metadata slot"))?; + validate_parts( + &validity, + metadata, + slots[VALUE_SLOT].as_ref(), + slots[TYPED_VALUE_SLOT].as_ref(), + dtype, + len, + ) } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -126,10 +108,6 @@ impl VTable for ParquetVariant { None } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } @@ -157,7 +135,7 @@ impl VTable for ParquetVariant { buffers: &[BufferHandle], children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { vortex_ensure!( buffers.is_empty(), "ParquetVariantArray expects 0 buffers, got {}", @@ -213,18 +191,22 @@ impl VTable for ParquetVariant { None }; - ParquetVariantData::try_new(validity, variant_metadata, value, typed_value) - } - - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "ParquetVariantArray expects {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) + ParquetVariantData::validate_parts( + &validity, + &variant_metadata, + value.as_ref(), + typed_value.as_ref(), + dtype, + len, + )?; + let slots = vec![ + validity_to_child(&validity, len), + Some(variant_metadata), + value, + typed_value, + ]; + let data = ParquetVariantData; + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { @@ -243,6 +225,16 @@ impl VTable for ParquetVariant { } } +impl ArrayHash for ParquetVariantData { + fn array_hash(&self, _state: &mut H, _precision: Precision) {} +} + +impl ArrayEq for ParquetVariantData { + fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { + true + } +} + #[cfg(test)] mod tests { use vortex_array::ArrayContext; @@ -267,6 +259,7 @@ mod tests { use vortex_session::registry::ReadContext; use crate::ParquetVariant; + use crate::array::ParquetVariantArrayExt; fn roundtrip(array: ArrayRef) -> ArrayRef { let dtype = array.dtype().clone(); let len = array.len(); diff --git a/encodings/pco/public-api.lock b/encodings/pco/public-api.lock index 4513321e259..47e76a743bc 100644 --- a/encodings/pco/public-api.lock +++ b/encodings/pco/public-api.lock @@ -24,15 +24,11 @@ pub type vortex_pco::Pco::OperationsVTable = vortex_pco::Pco pub type vortex_pco::Pco::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromValiditySliceHelper -pub fn vortex_pco::Pco::array_eq(array: &vortex_pco::PcoData, other: &vortex_pco::PcoData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_pco::Pco::array_hash(array: &vortex_pco::PcoData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_pco::Pco::buffer(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_pco::Pco::buffer_name(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_pco::Pco::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_pco::Pco::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_pco::Pco::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -46,11 +42,7 @@ pub fn vortex_pco::Pco::serialize(array: vortex_array::array::view::ArrayView<'_ pub fn vortex_pco::Pco::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_pco::Pco::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_pco::Pco::validate(&self, data: &vortex_pco::PcoData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_pco::Pco::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_pco::Pco::validate(&self, data: &vortex_pco::PcoData, dtype: &vortex_array::dtype::DType, len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_pco::Pco @@ -116,6 +108,14 @@ impl vortex_array::array::vtable::validity::ValiditySliceHelper for vortex_pco:: pub fn vortex_pco::PcoData::unsliced_validity_and_slice(&self) -> (&vortex_array::validity::Validity, usize, usize) +impl vortex_array::hash::ArrayEq for vortex_pco::PcoData + +pub fn vortex_pco::PcoData::array_eq(&self, other: &Self, precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_pco::PcoData + +pub fn vortex_pco::PcoData::array_hash(&self, state: &mut H, precision: vortex_array::hash::Precision) + pub struct vortex_pco::PcoMetadata pub vortex_pco::PcoMetadata::chunks: alloc::vec::Vec diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index f1e352542d3..5cd9c615145 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -4,6 +4,7 @@ use std::cmp; use std::fmt::Debug; use std::hash::Hash; +use std::hash::Hasher; use pco::ChunkConfig; use pco::PagingSpec; @@ -80,58 +81,68 @@ const VALUES_PER_CHUNK: usize = pco::DEFAULT_MAX_PAGE_N; vtable!(Pco, Pco, PcoData); -impl VTable for Pco { - type ArrayData = PcoData; - - type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromValiditySliceHelper; - - fn id(&self) -> ArrayId { - Self::ID - } - - fn validate(&self, data: &PcoData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate(dtype, len) - } - - fn array_hash(array: &PcoData, state: &mut H, precision: Precision) { - array.unsliced_validity.array_hash(state, precision); - array.unsliced_n_rows.hash(state); - array.slice_start.hash(state); - array.slice_stop.hash(state); +impl ArrayHash for PcoData { + fn array_hash(&self, state: &mut H, precision: Precision) { + self.unsliced_validity.array_hash(state, precision); + self.unsliced_n_rows.hash(state); + self.slice_start.hash(state); + self.slice_stop.hash(state); // Hash chunk_metas and pages using pointer-based hashing - for chunk_meta in &array.chunk_metas { + for chunk_meta in &self.chunk_metas { chunk_meta.array_hash(state, precision); } - for page in &array.pages { + for page in &self.pages { page.array_hash(state, precision); } } +} - fn array_eq(array: &PcoData, other: &PcoData, precision: Precision) -> bool { - if !array +impl ArrayEq for PcoData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + if !self .unsliced_validity .array_eq(&other.unsliced_validity, precision) - || array.unsliced_n_rows != other.unsliced_n_rows - || array.slice_start != other.slice_start - || array.slice_stop != other.slice_stop - || array.chunk_metas.len() != other.chunk_metas.len() - || array.pages.len() != other.pages.len() + || self.unsliced_n_rows != other.unsliced_n_rows + || self.slice_start != other.slice_start + || self.slice_stop != other.slice_stop + || self.chunk_metas.len() != other.chunk_metas.len() + || self.pages.len() != other.pages.len() { return false; } - for (a, b) in array.chunk_metas.iter().zip(&other.chunk_metas) { + for (a, b) in self.chunk_metas.iter().zip(&other.chunk_metas) { if !a.array_eq(b, precision) { return false; } } - for (a, b) in array.pages.iter().zip(&other.pages) { + for (a, b) in self.pages.iter().zip(&other.pages) { if !a.array_eq(b, precision) { return false; } } true } +} + +impl VTable for Pco { + type ArrayData = PcoData; + + type OperationsVTable = Self; + type ValidityVTable = ValidityVTableFromValiditySliceHelper; + + fn id(&self) -> ArrayId { + Self::ID + } + + fn validate( + &self, + data: &PcoData, + dtype: &DType, + len: usize, + _slots: &[Option], + ) -> VortexResult<()> { + data.validate(dtype, len) + } fn nbuffers(array: ArrayView<'_, Self>) -> usize { array.chunk_metas.len() + array.pages.len() @@ -166,7 +177,7 @@ impl VTable for Pco { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = PcoMetadata::decode(metadata)?; let validity = if children.is_empty() { Validity::from(dtype.nullability()) @@ -194,39 +205,22 @@ impl VTable for Pco { .sum::(); vortex_ensure!(pages.len() == expected_n_pages); - Ok(PcoData::new( + let slots = vec![validity_to_child(&validity, len)]; + let data = PcoData::new( chunk_metas, pages, dtype.as_ptype(), metadata, len, validity, - )) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + ); + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "PcoArray expects {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.unsliced_validity = match &slots[VALIDITY_SLOT] { - Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.unsliced_validity.nullability()), - }; - array.slots = slots; - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done(array.decompress(ctx)?.into_array())) } @@ -282,7 +276,13 @@ impl Pco { pub(crate) fn try_new(dtype: DType, data: PcoData) -> VortexResult { let len = data.len(); data.validate(&dtype, len)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Pco, dtype, len, data)) }) + let slots = vec![validity_to_child( + &data.unsliced_validity, + data.unsliced_n_rows, + )]; + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(Pco, dtype, len, data).with_slots(slots)) + }) } /// Compress a primitive array using pcodec. @@ -298,7 +298,6 @@ impl Pco { } /// The validity bitmap indicating which elements are non-null. -pub(super) const VALIDITY_SLOT: usize = 0; pub(super) const NUM_SLOTS: usize = 1; pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; @@ -310,7 +309,6 @@ pub struct PcoData { ptype: PType, pub(crate) unsliced_validity: Validity, unsliced_n_rows: usize, - pub(super) slots: Vec>, slice_start: usize, slice_stop: usize, } @@ -377,8 +375,6 @@ impl PcoData { len: usize, validity: Validity, ) -> Self { - let validity_slot = validity_to_child(&validity, len); - Self { chunk_metas, pages, @@ -386,7 +382,6 @@ impl PcoData { ptype, unsliced_validity: validity, unsliced_n_rows: len, - slots: vec![validity_slot], slice_start: 0, slice_stop: len, } @@ -474,7 +469,7 @@ impl PcoData { } pub fn from_array(array: ArrayRef, level: usize, nums_per_page: usize) -> VortexResult { - let parray = array.try_into::().map_err(|a| { + let parray = array.try_downcast::().map_err(|a| { vortex_err!( "Pco can only encode primitive arrays, got {}", a.encoding_id() diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock index 975353acb75..46db17fd0ef 100644 --- a/encodings/runend/public-api.lock +++ b/encodings/runend/public-api.lock @@ -48,15 +48,11 @@ pub type vortex_runend::RunEnd::OperationsVTable = vortex_runend::RunEnd pub type vortex_runend::RunEnd::ValidityVTable = vortex_runend::RunEnd -pub fn vortex_runend::RunEnd::array_eq(array: &vortex_runend::RunEndData, other: &vortex_runend::RunEndData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_runend::RunEnd::array_hash(array: &vortex_runend::RunEndData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_runend::RunEnd::buffer(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_runend::RunEnd::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_runend::RunEnd::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_runend::RunEnd::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_runend::RunEnd::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -72,11 +68,7 @@ pub fn vortex_runend::RunEnd::serialize(array: vortex_array::array::view::ArrayV pub fn vortex_runend::RunEnd::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_runend::RunEnd::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_runend::RunEnd::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_runend::RunEnd::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_runend::RunEnd::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_runend::RunEnd @@ -110,23 +102,13 @@ pub struct vortex_runend::RunEndData impl vortex_runend::RunEndData -pub fn vortex_runend::RunEndData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_runend::RunEndData::encode(array: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult -pub fn vortex_runend::RunEndData::ends(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn vortex_runend::RunEndData::find_physical_index(&self, index: usize) -> vortex_error::VortexResult - -pub fn vortex_runend::RunEndData::into_parts(self) -> vortex_runend::RunEndDataParts +pub fn vortex_runend::RunEndData::into_parts(self, ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef) -> vortex_runend::RunEndDataParts -pub fn vortex_runend::RunEndData::new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef) -> Self +pub fn vortex_runend::RunEndData::new(offset: usize) -> Self -pub unsafe fn vortex_runend::RunEndData::new_unchecked(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, _length: usize) -> Self - -pub fn vortex_runend::RunEndData::offset(&self) -> usize - -pub fn vortex_runend::RunEndData::values(&self) -> &vortex_array::array::erased::ArrayRef +pub unsafe fn vortex_runend::RunEndData::new_unchecked(offset: usize) -> Self impl core::clone::Clone for vortex_runend::RunEndData @@ -136,6 +118,14 @@ impl core::fmt::Debug for vortex_runend::RunEndData pub fn vortex_runend::RunEndData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_runend::RunEndData + +pub fn vortex_runend::RunEndData::array_eq(&self, other: &Self, _precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_runend::RunEndData + +pub fn vortex_runend::RunEndData::array_hash(&self, state: &mut H, _precision: vortex_array::hash::Precision) + impl vortex_array::arrow::FromArrowArray<&arrow_array::array::run_array::RunArray> for vortex_runend::RunEndData where ::Native: vortex_array::dtype::ptype::NativePType pub fn vortex_runend::RunEndData::from_arrow(array: &arrow_array::array::run_array::RunArray, nullable: bool) -> vortex_error::VortexResult @@ -180,6 +170,30 @@ pub fn vortex_runend::RunEndMetadata::clear(&mut self) pub fn vortex_runend::RunEndMetadata::encoded_len(&self) -> usize +pub trait vortex_runend::RunEndArrayExt: vortex_array::array::typed::TypedArrayRef + +pub fn vortex_runend::RunEndArrayExt::dtype(&self) -> &vortex_array::dtype::DType + +pub fn vortex_runend::RunEndArrayExt::ends(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn vortex_runend::RunEndArrayExt::find_physical_index(&self, index: usize) -> vortex_error::VortexResult + +pub fn vortex_runend::RunEndArrayExt::offset(&self) -> usize + +pub fn vortex_runend::RunEndArrayExt::values(&self) -> &vortex_array::array::erased::ArrayRef + +impl> vortex_runend::RunEndArrayExt for T + +pub fn T::dtype(&self) -> &vortex_array::dtype::DType + +pub fn T::ends(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn T::find_physical_index(&self, index: usize) -> vortex_error::VortexResult + +pub fn T::offset(&self) -> usize + +pub fn T::values(&self) -> &vortex_array::array::erased::ArrayRef + pub fn vortex_runend::initialize(session: &vortex_session::VortexSession) pub fn vortex_runend::trimmed_ends_iter(run_ends: &[E], offset: usize, length: usize) -> impl core::iter::traits::iterator::Iterator + use<'_, E> diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index a9fe74e7d36..8705281cab6 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -3,6 +3,7 @@ use std::fmt::Debug; use std::hash::Hash; +use std::hash::Hasher; use prost::Message; use vortex_array::Array; @@ -16,6 +17,7 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::TypedArrayRef; use vortex_array::arrays::Primitive; use vortex_array::arrays::VarBinViewArray; use vortex_array::buffer::BufferHandle; @@ -56,6 +58,18 @@ pub struct RunEndMetadata { pub offset: u64, } +impl ArrayHash for RunEndData { + fn array_hash(&self, state: &mut H, _precision: Precision) { + self.offset.hash(state); + } +} + +impl ArrayEq for RunEndData { + fn array_eq(&self, other: &Self, _precision: Precision) -> bool { + self.offset == other.offset + } +} + impl VTable for RunEnd { type ArrayData = RunEndData; @@ -66,29 +80,29 @@ impl VTable for RunEnd { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - RunEndData::validate(data.ends(), data.values(), data.offset, len)?; + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let ends = slots[ENDS_SLOT] + .as_ref() + .vortex_expect("RunEndArray ends slot"); + let values = slots[VALUES_SLOT] + .as_ref() + .vortex_expect("RunEndArray values slot"); + RunEndData::validate_parts(ends, values, data.offset, len)?; vortex_ensure!( - data.values().dtype() == dtype, + values.dtype() == dtype, "expected dtype {}, got {}", dtype, - data.values().dtype() + values.dtype() ); Ok(()) } - fn array_hash(array: &RunEndData, state: &mut H, precision: Precision) { - array.ends().array_hash(state, precision); - array.values().array_hash(state, precision); - array.offset.hash(state); - } - - fn array_eq(array: &RunEndData, other: &RunEndData, precision: Precision) -> bool { - array.ends().array_eq(other.ends(), precision) - && array.values().array_eq(other.values(), precision) - && array.offset == other.offset - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -121,41 +135,23 @@ impl VTable for RunEnd { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = RunEndMetadata::decode(metadata)?; let ends_dtype = DType::Primitive(metadata.ends_ptype(), Nullability::NonNullable); let runs = usize::try_from(metadata.num_runs).vortex_expect("Must be a valid usize"); let ends = children.get(0, &ends_dtype, runs)?; let values = children.get(1, dtype, runs)?; - - RunEndData::try_new_offset_length( - ends, - values, - usize::try_from(metadata.offset).vortex_expect("Offset must be a valid usize"), - len, - ) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let offset = usize::try_from(metadata.offset).vortex_expect("Offset must be a valid usize"); + let slots = vec![Some(ends), Some(values)]; + let data = RunEndData::new(offset); + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "RunEndArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn reduce_parent( array: ArrayView<'_, Self>, parent: &ArrayRef, @@ -187,7 +183,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["ends", "values"]; #[derive(Clone, Debug)] pub struct RunEndData { - pub(super) slots: Vec>, offset: usize, } @@ -197,6 +192,40 @@ pub struct RunEndDataParts { pub offset: usize, } +pub trait RunEndArrayExt: TypedArrayRef { + fn offset(&self) -> usize { + self.offset + } + + fn ends(&self) -> &ArrayRef { + self.as_ref().slots()[ENDS_SLOT] + .as_ref() + .vortex_expect("RunEndArray ends slot") + } + + fn values(&self) -> &ArrayRef { + self.as_ref().slots()[VALUES_SLOT] + .as_ref() + .vortex_expect("RunEndArray values slot") + } + + fn dtype(&self) -> &DType { + self.values().dtype() + } + + fn find_physical_index(&self, index: usize) -> VortexResult { + Ok(self + .ends() + .as_primitive_typed() + .search_sorted( + &PValue::from(index + self.offset()), + SearchSortedSide::Right, + )? + .to_ends_index(self.ends().len())) + } +} +impl> RunEndArrayExt for T {} + #[derive(Clone, Debug)] pub struct RunEnd; @@ -213,17 +242,25 @@ impl RunEnd { offset: usize, length: usize, ) -> RunEndArray { - let data = unsafe { RunEndData::new_unchecked(ends, values, offset, length) }; - let dtype = data.dtype().clone(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, length, data)) } + let dtype = values.dtype().clone(); + let slots = vec![Some(ends.clone()), Some(values.clone())]; + RunEndData::validate_parts(&ends, &values, offset, length) + .vortex_expect("RunEndArray validation failed"); + let data = unsafe { RunEndData::new_unchecked(offset) }; + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(RunEnd, dtype, length, data).with_slots(slots), + ) + } } /// Build a new [`RunEndArray`] from ends and values. pub fn try_new(ends: ArrayRef, values: ArrayRef) -> VortexResult { let len = RunEndData::logical_len_from_ends(&ends)?; - let data = RunEndData::try_new_offset_length(ends, values, 0, len)?; - let dtype = data.dtype().clone(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, len, data)) }) + let dtype = values.dtype().clone(); + let slots = vec![Some(ends), Some(values)]; + let data = RunEndData::new(0); + Array::try_from_parts(ArrayParts::new(RunEnd, dtype, len, data).with_slots(slots)) } /// Build a new [`RunEndArray`] from ends, values, offset, and length. @@ -233,9 +270,10 @@ impl RunEnd { offset: usize, length: usize, ) -> VortexResult { - let data = RunEndData::try_new_offset_length(ends, values, offset, length)?; - let dtype = data.dtype().clone(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, length, data)) }) + let dtype = values.dtype().clone(); + let slots = vec![Some(ends), Some(values)]; + let data = RunEndData::new(offset); + Array::try_from_parts(ArrayParts::new(RunEnd, dtype, length, data).with_slots(slots)) } /// Build a new [`RunEndArray`] from ends and values (panics on invalid input). @@ -245,10 +283,17 @@ impl RunEnd { /// Run the array through run-end encoding. pub fn encode(array: ArrayRef) -> VortexResult { - let len = array.len(); - let data = RunEndData::encode(array)?; - let dtype = data.dtype().clone(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, len, data)) }) + if let Some(parray) = array.as_opt::() { + let (ends, values) = runend_encode(parray); + let ends = ends.into_array(); + let len = array.len(); + let dtype = values.dtype().clone(); + let slots = vec![Some(ends), Some(values)]; + let data = unsafe { RunEndData::new_unchecked(0) }; + Array::try_from_parts(ArrayParts::new(RunEnd, dtype, len, data).with_slots(slots)) + } else { + vortex_bail!("REE can only encode primitive arrays") + } } } @@ -261,7 +306,7 @@ impl RunEndData { } } - fn validate( + pub(crate) fn validate_parts( ends: &ArrayRef, values: &ArrayRef, offset: usize, @@ -358,35 +403,8 @@ impl RunEndData { /// # Ok(()) /// # } /// ``` - pub fn new(ends: ArrayRef, values: ArrayRef) -> Self { - Self::try_new(ends, values).vortex_expect("RunEndArray new") - } - - /// Build a new `RunEndArray` from components. - /// - /// # Validation - /// - /// The `ends` must be non-nullable unsigned integers. - pub(crate) fn try_new(ends: ArrayRef, values: ArrayRef) -> VortexResult { - let length = Self::logical_len_from_ends(&ends)?; - Self::try_new_offset_length(ends, values, 0, length) - } - - /// Construct a new sliced `RunEndArray` with the provided offset and length. - /// - /// This performs all the same validation as [`RunEnd::try_new_offset_length`]. - pub(crate) fn try_new_offset_length( - ends: ArrayRef, - values: ArrayRef, - offset: usize, - length: usize, - ) -> VortexResult { - Self::validate(&ends, &values, offset, length)?; - - Ok(Self { - slots: vec![Some(ends), Some(values)], - offset, - }) + pub fn new(offset: usize) -> Self { + Self { offset } } /// Build a new `RunEndArray` without validation. @@ -398,92 +416,25 @@ impl RunEndData { /// satisfied before calling this function. /// /// See [`RunEnd::try_new_offset_length`] for the preconditions needed to build a new array. - pub unsafe fn new_unchecked( - ends: ArrayRef, - values: ArrayRef, - offset: usize, - _length: usize, - ) -> Self { - Self { - slots: vec![Some(ends), Some(values)], - offset, - } - } - - /// Convert the given logical index to an index into the `values` array - pub fn find_physical_index(&self, index: usize) -> VortexResult { - Ok(self - .ends() - .as_primitive_typed() - .search_sorted( - &PValue::from(index + self.offset()), - SearchSortedSide::Right, - )? - .to_ends_index(self.ends().len())) + pub unsafe fn new_unchecked(offset: usize) -> Self { + Self { offset } } /// Run the array through run-end encoding. pub fn encode(array: ArrayRef) -> VortexResult { if let Some(parray) = array.as_opt::() { - let (ends, values) = runend_encode(parray); + let (_ends, _values) = runend_encode(parray); // SAFETY: runend_encode handles this - unsafe { - Ok(Self::new_unchecked( - ends.into_array(), - values, - 0, - array.len(), - )) - } + unsafe { Ok(Self::new_unchecked(0)) } } else { vortex_bail!("REE can only encode primitive arrays") } } - /// Returns the logical data type of the array. - #[inline] - pub fn dtype(&self) -> &DType { - self.values().dtype() - } - - /// The offset that the `ends` is relative to. - /// - /// This is generally zero for a "new" array, and non-zero after a slicing operation. - #[inline] - pub fn offset(&self) -> usize { - self.offset - } - - /// The encoded "ends" of value runs. - /// - /// The `i`-th element indicates that there is a run of the same value, beginning - /// at `ends[i]` (inclusive) and terminating at `ends[i+1]` (exclusive). - #[inline] - pub fn ends(&self) -> &ArrayRef { - self.slots[ENDS_SLOT] - .as_ref() - .vortex_expect("RunEndArray ends slot") - } - - /// The scalar values. - /// - /// The `i`-th element is the scalar value for the `i`-th repeated run. The run begins - /// at `ends[i]` (inclusive) and terminates at `ends[i+1]` (exclusive). - #[inline] - pub fn values(&self) -> &ArrayRef { - self.slots[VALUES_SLOT] - .as_ref() - .vortex_expect("RunEndArray values slot") - } - - pub fn into_parts(mut self) -> RunEndDataParts { + pub fn into_parts(self, ends: ArrayRef, values: ArrayRef) -> RunEndDataParts { RunEndDataParts { - ends: self.slots[ENDS_SLOT] - .take() - .vortex_expect("RunEndArray ends slot"), - values: self.slots[VALUES_SLOT] - .take() - .vortex_expect("RunEndArray values slot"), + ends, + values, offset: self.offset, } } diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs index 25c1ef197f7..564c1d06cb2 100644 --- a/encodings/runend/src/arrow.rs +++ b/encodings/runend/src/arrow.rs @@ -6,6 +6,7 @@ use arrow_array::types::RunEndIndexType; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::arrow::FromArrowArray; use vortex_array::dtype::NativePType; use vortex_array::scalar::PValue; @@ -52,7 +53,8 @@ where }; // SAFETY: arrow-rs enforces the RunEndArray invariants, we inherit their guarantees. - Ok(unsafe { RunEndData::new_unchecked(ends_slice, values_slice, offset, len) }) + RunEndData::validate_parts(&ends_slice, &values_slice, offset, len)?; + Ok(unsafe { RunEndData::new_unchecked(offset) }) } } @@ -72,9 +74,11 @@ mod tests { use arrow_schema::DataType; use arrow_schema::Field; use rstest::rstest; + use vortex_array::ArrayRef; use vortex_array::IntoArray as _; use vortex_array::VortexSessionExecute as _; use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::arrow::ArrowArrayExecutor; use vortex_array::arrow::FromArrowArray; use vortex_array::assert_arrays_eq; @@ -82,13 +86,18 @@ mod tests { use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; + use vortex_array::scalar::PValue; + use vortex_array::search_sorted::SearchSorted; + use vortex_array::search_sorted::SearchSortedSide; use vortex_array::session::ArraySession; + use vortex_array::validity::Validity; + use vortex_buffer::Buffer; use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_session::VortexSession; use crate::RunEnd; - use crate::RunEndData; + use crate::ops::find_slice_end_index; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); @@ -100,13 +109,36 @@ mod tests { where R::Native: NativePType, { - let data = RunEndData::from_arrow(array, nullable)?; - RunEnd::try_new_offset_length( - data.ends().clone(), - data.values().clone(), - data.offset(), - array.len(), + let offset = array.run_ends().offset(); + let len = array.run_ends().len(); + let ends_buf = + Buffer::::from_arrow_scalar_buffer(array.run_ends().inner().clone()); + let ends = PrimitiveArray::new(ends_buf, Validity::NonNullable) + .reinterpret_cast(R::Native::PTYPE.to_unsigned()); + let values = ArrayRef::from_arrow(array.values().as_ref(), nullable)?; + + let ends_array = PrimitiveArray::from_buffer_handle( + ends.buffer_handle().clone(), + ends.ptype(), + ends.validity()?, ) + .into_array(); + let (ends_slice, values_slice) = if offset == 0 && len == array.run_ends().max_value() { + (ends_array, values) + } else { + let slice_begin = ends_array + .as_primitive_typed() + .search_sorted(&PValue::from(offset), SearchSortedSide::Right)? + .to_ends_index(ends_array.len()); + let slice_end = find_slice_end_index(&ends_array, offset + len)?; + + ( + ends_array.slice(slice_begin..slice_end)?, + values.slice(slice_begin..slice_end)?, + ) + }; + + RunEnd::try_new_offset_length(ends_slice, values_slice, offset, array.len()) } #[test] @@ -252,10 +284,7 @@ mod tests { ) } - fn execute( - array: vortex_array::ArrayRef, - dt: &DataType, - ) -> VortexResult { + fn execute(array: ArrayRef, dt: &DataType) -> VortexResult { array.execute_arrow(Some(dt), &mut SESSION.create_execution_ctx()) } diff --git a/encodings/runend/src/compress.rs b/encodings/runend/src/compress.rs index b36b549225c..df86d6cc5fb 100644 --- a/encodings/runend/src/compress.rs +++ b/encodings/runend/src/compress.rs @@ -11,6 +11,8 @@ use vortex_array::arrays::ConstantArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::bool::BoolArrayExt; +use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; @@ -288,7 +290,7 @@ pub fn runend_decode_varbinview( ) }); - let parts = values.into_data().into_parts(); + let parts = values.into_data_parts(); let view_handle = BufferHandle::new_host(decoded_views.into_byte_buffer()); // SAFETY: we are expanding views from a valid VarBinViewArray with the same diff --git a/encodings/runend/src/compute/cast.rs b/encodings/runend/src/compute/cast.rs index d9ecb3a0baf..1333a8d076a 100644 --- a/encodings/runend/src/compute/cast.rs +++ b/encodings/runend/src/compute/cast.rs @@ -10,6 +10,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::RunEnd; +use crate::array::RunEndArrayExt; impl CastReduce for RunEnd { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // Cast the values array to the target type diff --git a/encodings/runend/src/compute/compare.rs b/encodings/runend/src/compute/compare.rs index d9926538944..8dd253e453b 100644 --- a/encodings/runend/src/compute/compare.rs +++ b/encodings/runend/src/compute/compare.rs @@ -15,6 +15,7 @@ use vortex_array::scalar_fn::fns::operators::Operator; use vortex_error::VortexResult; use crate::RunEnd; +use crate::array::RunEndArrayExt; use crate::decompress_bool::runend_decode_bools; impl CompareKernel for RunEnd { diff --git a/encodings/runend/src/compute/fill_null.rs b/encodings/runend/src/compute/fill_null.rs index abe0ef44f09..ddce31184eb 100644 --- a/encodings/runend/src/compute/fill_null.rs +++ b/encodings/runend/src/compute/fill_null.rs @@ -10,6 +10,7 @@ use vortex_array::scalar_fn::fns::fill_null::FillNullReduce; use vortex_error::VortexResult; use crate::RunEnd; +use crate::array::RunEndArrayExt; impl FillNullReduce for RunEnd { fn fill_null( diff --git a/encodings/runend/src/compute/filter.rs b/encodings/runend/src/compute/filter.rs index e0ef5382d40..11093d482ac 100644 --- a/encodings/runend/src/compute/filter.rs +++ b/encodings/runend/src/compute/filter.rs @@ -21,6 +21,7 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::RunEnd; +use crate::array::RunEndArrayExt; use crate::compute::take::take_indices_unchecked; const FILTER_TAKE_THRESHOLD: f64 = 0.1; @@ -38,7 +39,7 @@ impl FilterKernel for RunEnd { if runs_ratio < FILTER_TAKE_THRESHOLD || mask_values.true_count() < 25 { Ok(Some(take_indices_unchecked( - &array, + array, mask_values.indices(), &Validity::NonNullable, )?)) diff --git a/encodings/runend/src/compute/is_constant.rs b/encodings/runend/src/compute/is_constant.rs index 89e8711b6eb..53418f3ef50 100644 --- a/encodings/runend/src/compute/is_constant.rs +++ b/encodings/runend/src/compute/is_constant.rs @@ -11,6 +11,7 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::RunEnd; +use crate::array::RunEndArrayExt; /// RunEnd-specific is_constant kernel. /// diff --git a/encodings/runend/src/compute/is_sorted.rs b/encodings/runend/src/compute/is_sorted.rs index e3989b4972c..e26a3d8cb57 100644 --- a/encodings/runend/src/compute/is_sorted.rs +++ b/encodings/runend/src/compute/is_sorted.rs @@ -13,6 +13,7 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::RunEnd; +use crate::array::RunEndArrayExt; /// RunEnd-specific is_sorted kernel. /// diff --git a/encodings/runend/src/compute/min_max.rs b/encodings/runend/src/compute/min_max.rs index c44db02e1b0..2a65d9de2bf 100644 --- a/encodings/runend/src/compute/min_max.rs +++ b/encodings/runend/src/compute/min_max.rs @@ -12,6 +12,7 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::RunEnd; +use crate::array::RunEndArrayExt; /// RunEnd-specific min/max kernel. /// diff --git a/encodings/runend/src/compute/take.rs b/encodings/runend/src/compute/take.rs index 511b473c53b..9bbdbf6784e 100644 --- a/encodings/runend/src/compute/take.rs +++ b/encodings/runend/src/compute/take.rs @@ -20,7 +20,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use crate::RunEnd; -use crate::RunEndData; +use crate::array::RunEndArrayExt; impl TakeExecute for RunEnd { #[expect( @@ -50,13 +50,13 @@ impl TakeExecute for RunEnd { }); let indices_validity = primitive_indices.validity()?; - take_indices_unchecked(&array, &checked_indices, &indices_validity).map(Some) + take_indices_unchecked(array, &checked_indices, &indices_validity).map(Some) } } /// Perform a take operation on a RunEndArray by binary searching for each of the indices. pub fn take_indices_unchecked>( - array: &RunEndData, + array: ArrayView<'_, RunEnd>, indices: &[T], validity: &Validity, ) -> VortexResult { diff --git a/encodings/runend/src/compute/take_from.rs b/encodings/runend/src/compute/take_from.rs index 85d3dcf4fae..497d7dedc23 100644 --- a/encodings/runend/src/compute/take_from.rs +++ b/encodings/runend/src/compute/take_from.rs @@ -6,11 +6,13 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::Dict; +use vortex_array::arrays::dict::DictArrayExt; use vortex_array::dtype::DType; use vortex_array::kernel::ExecuteParentKernel; use vortex_error::VortexResult; use crate::RunEnd; +use crate::array::RunEndArrayExt; #[derive(Debug)] pub(crate) struct RunEndTakeFrom; @@ -63,6 +65,7 @@ mod tests { use crate::RunEnd; use crate::RunEndArray; + use crate::array::RunEndArrayExt; use crate::compute::take_from::RunEndTakeFrom; /// Build a DictArray whose codes are run-end encoded. diff --git a/encodings/runend/src/decompress_bool.rs b/encodings/runend/src/decompress_bool.rs index 407745d9154..6733f143a8c 100644 --- a/encodings/runend/src/decompress_bool.rs +++ b/encodings/runend/src/decompress_bool.rs @@ -12,6 +12,7 @@ use vortex_array::IntoArray; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::match_each_unsigned_integer_ptype; @@ -243,6 +244,7 @@ mod tests { use vortex_array::ToCanonical; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::assert_arrays_eq; use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; diff --git a/encodings/runend/src/kernel.rs b/encodings/runend/src/kernel.rs index 97f17c17857..432453ee609 100644 --- a/encodings/runend/src/kernel.rs +++ b/encodings/runend/src/kernel.rs @@ -17,6 +17,7 @@ use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; use vortex_error::VortexResult; use crate::RunEnd; +use crate::array::RunEndArrayExt; use crate::compute::take_from::RunEndTakeFrom; pub(super) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ diff --git a/encodings/runend/src/ops.rs b/encodings/runend/src/ops.rs index d0d6716d5ad..c27885cc39a 100644 --- a/encodings/runend/src/ops.rs +++ b/encodings/runend/src/ops.rs @@ -13,6 +13,7 @@ use vortex_array::vtable::OperationsVTable; use vortex_error::VortexResult; use crate::RunEnd; +use crate::array::RunEndArrayExt; impl OperationsVTable for RunEnd { fn scalar_at( diff --git a/encodings/runend/src/rules.rs b/encodings/runend/src/rules.rs index 2c8a6607156..73f9ef3cb7a 100644 --- a/encodings/runend/src/rules.rs +++ b/encodings/runend/src/rules.rs @@ -8,6 +8,7 @@ use vortex_array::arrays::Constant; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::ScalarFnArray; use vortex_array::arrays::scalar_fn::AnyScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_array::arrays::scalar_fn::ScalarFnVTable; use vortex_array::dtype::DType; use vortex_array::optimizer::rules::ArrayParentReduceRule; @@ -17,6 +18,7 @@ use vortex_array::scalar_fn::fns::fill_null::FillNullReduceAdaptor; use vortex_error::VortexResult; use crate::RunEnd; +use crate::array::RunEndArrayExt; pub(super) const RULES: ParentRuleSet = ParentRuleSet::new(&[ // CastReduceAdaptor must come before RunEndScalarFnRule so that cast operations are executed diff --git a/encodings/sequence/public-api.lock b/encodings/sequence/public-api.lock index cbbbd492efb..c0f39432ec1 100644 --- a/encodings/sequence/public-api.lock +++ b/encodings/sequence/public-api.lock @@ -26,15 +26,11 @@ pub type vortex_sequence::Sequence::OperationsVTable = vortex_sequence::Sequence pub type vortex_sequence::Sequence::ValidityVTable = vortex_sequence::Sequence -pub fn vortex_sequence::Sequence::array_eq(array: &vortex_sequence::SequenceData, other: &vortex_sequence::SequenceData, _precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_sequence::Sequence::array_hash(array: &vortex_sequence::SequenceData, state: &mut H, _precision: vortex_array::hash::Precision) - pub fn vortex_sequence::Sequence::buffer(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_sequence::Sequence::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_sequence::Sequence::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_sequence::Sequence::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_sequence::Sequence::execute(array: vortex_array::array::typed::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -50,11 +46,7 @@ pub fn vortex_sequence::Sequence::serialize(array: vortex_array::array::view::Ar pub fn vortex_sequence::Sequence::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_sequence::Sequence::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_sequence::Sequence::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_sequence::Sequence::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_sequence::Sequence::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_sequence::Sequence @@ -110,6 +102,14 @@ impl core::fmt::Debug for vortex_sequence::SequenceData pub fn vortex_sequence::SequenceData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_sequence::SequenceData + +pub fn vortex_sequence::SequenceData::array_eq(&self, other: &Self, _precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_sequence::SequenceData + +pub fn vortex_sequence::SequenceData::array_hash(&self, state: &mut H, _precision: vortex_array::hash::Precision) + pub struct vortex_sequence::SequenceDataParts pub vortex_sequence::SequenceDataParts::base: vortex_array::scalar::typed_view::primitive::pvalue::PValue diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index a3d50a420e1..d4b1409deac 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -2,10 +2,13 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::hash::Hash; +use std::hash::Hasher; use num_traits::cast::FromPrimitive; use prost::Message; use vortex_array::Array; +use vortex_array::ArrayEq; +use vortex_array::ArrayHash; use vortex_array::ArrayId; use vortex_array::ArrayParts; use vortex_array::ArrayRef; @@ -63,7 +66,6 @@ pub(super) const SLOT_NAMES: [&str; 0] = []; pub struct SequenceData { base: PValue, multiplier: PValue, - pub(super) slots: Vec>, } pub struct SequenceDataParts { @@ -144,11 +146,7 @@ impl SequenceData { /// - `base` and `multiplier` are both normalized to the same integer `ptype`. /// - they are logically compatible with the outer dtype and len. pub(crate) unsafe fn new_unchecked(base: PValue, multiplier: PValue) -> Self { - Self { - base, - multiplier, - slots: vec![], - } + Self { base, multiplier } } pub fn ptype(&self) -> PType { @@ -205,6 +203,19 @@ impl SequenceData { } } +impl ArrayHash for SequenceData { + fn array_hash(&self, state: &mut H, _precision: Precision) { + self.base.hash(state); + self.multiplier.hash(state); + } +} + +impl ArrayEq for SequenceData { + fn array_eq(&self, other: &Self, _precision: Precision) -> bool { + self.base == other.base && self.multiplier == other.multiplier + } +} + impl VTable for Sequence { type ArrayData = SequenceData; @@ -215,23 +226,16 @@ impl VTable for Sequence { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + _slots: &[Option], + ) -> VortexResult<()> { SequenceData::validate(data.base, data.multiplier, dtype, len) } - fn array_hash( - array: &SequenceData, - state: &mut H, - _precision: Precision, - ) { - array.base.hash(state); - array.multiplier.hash(state); - } - - fn array_eq(array: &SequenceData, other: &SequenceData, _precision: Precision) -> bool { - array.base == other.base && array.multiplier == other.multiplier - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -261,7 +265,7 @@ impl VTable for Sequence { buffers: &[BufferHandle], children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { vortex_ensure!( buffers.is_empty(), "SequenceArray expects 0 buffers, got {}", @@ -301,27 +305,14 @@ impl VTable for Sequence { .pvalue() .vortex_expect("sequence array multiplier should be a non-nullable primitive"); - SequenceData::try_new(base, multiplier, ptype, dtype.nullability(), len) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let data = SequenceData::try_new(base, multiplier, ptype, dtype.nullability(), len)?; + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.is_empty(), - "SequenceArray expects 0 slots, got {}", - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { sequence_decompress(&array).map(ExecutionResult::done) } diff --git a/encodings/sparse/public-api.lock b/encodings/sparse/public-api.lock index b937d3df263..883b4b15f01 100644 --- a/encodings/sparse/public-api.lock +++ b/encodings/sparse/public-api.lock @@ -28,15 +28,11 @@ pub type vortex_sparse::Sparse::OperationsVTable = vortex_sparse::Sparse pub type vortex_sparse::Sparse::ValidityVTable = vortex_sparse::Sparse -pub fn vortex_sparse::Sparse::array_eq(array: &vortex_sparse::SparseData, other: &vortex_sparse::SparseData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_sparse::Sparse::array_hash(array: &vortex_sparse::SparseData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_sparse::Sparse::buffer(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_sparse::Sparse::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_sparse::Sparse::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_sparse::Sparse::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_sparse::Sparse::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -52,11 +48,7 @@ pub fn vortex_sparse::Sparse::serialize(array: vortex_array::array::view::ArrayV pub fn vortex_sparse::Sparse::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_sparse::Sparse::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_sparse::Sparse::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_sparse::Sparse::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_sparse::Sparse::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_sparse::Sparse @@ -104,8 +96,6 @@ pub fn vortex_sparse::SparseData::patches(&self) -> &vortex_array::patches::Patc pub fn vortex_sparse::SparseData::resolved_patches(&self) -> vortex_error::VortexResult -pub fn vortex_sparse::SparseData::try_new(indices: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, len: usize, fill_value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult - pub fn vortex_sparse::SparseData::try_new_from_patches(patches: vortex_array::patches::Patches, fill_value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult pub fn vortex_sparse::SparseData::validate(patches: &vortex_array::patches::Patches, fill_value: &vortex_array::scalar::Scalar, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> @@ -118,6 +108,14 @@ impl core::fmt::Debug for vortex_sparse::SparseData pub fn vortex_sparse::SparseData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_sparse::SparseData + +pub fn vortex_sparse::SparseData::array_eq(&self, other: &Self, precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_sparse::SparseData + +pub fn vortex_sparse::SparseData::array_hash(&self, state: &mut H, precision: vortex_array::hash::Precision) + #[repr(C)] pub struct vortex_sparse::SparseMetadata impl core::clone::Clone for vortex_sparse::SparseMetadata diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index 7afe3ce2b94..1d42ccf5d15 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -5,19 +5,19 @@ use std::sync::Arc; use itertools::Itertools; use num_traits::NumCast; -use vortex_array::Array; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::BoolArray; -use vortex_array::arrays::Decimal; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::ListViewArray; use vortex_array::arrays::NullArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::listview::ListViewArrayExt; +use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::arrays::varbinview::build_views::BinaryView; use vortex_array::buffer::BufferHandle; use vortex_array::builders::ArrayBuilder; @@ -478,10 +478,7 @@ fn execute_sparse_decimal( } } let filled_array = builder.finish_into_decimal(); - let dtype = filled_array.dtype().clone(); - let len = filled_array.len(); - let data = filled_array.into_data().patch(patches, ctx)?; - Ok(Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data))?.into_array()) + Ok(filled_array.patch(patches, ctx)?.into_array()) } fn execute_varbin( @@ -556,6 +553,7 @@ mod test { use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinArray; use vortex_array::arrays::VarBinViewArray; + use vortex_array::arrays::listview::ListViewArrayExt; use vortex_array::arrow::IntoArrowArray as _; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index fc9613a369d..fc42bfafb4c 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -3,6 +3,7 @@ use std::fmt::Debug; use std::hash::Hash; +use std::hash::Hasher; use kernel::PARENT_KERNELS; use prost::Message as _; @@ -19,6 +20,7 @@ use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ToCanonical; use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::buffer::BufferHandle; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; @@ -64,6 +66,19 @@ pub struct SparseMetadata { patches: PatchesMetadata, } +impl ArrayHash for SparseData { + fn array_hash(&self, state: &mut H, precision: Precision) { + self.patches.array_hash(state, precision); + self.fill_value.hash(state); + } +} + +impl ArrayEq for SparseData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + self.patches.array_eq(&other.patches, precision) && self.fill_value == other.fill_value + } +} + impl VTable for Sparse { type ArrayData = SparseData; @@ -74,19 +89,16 @@ impl VTable for Sparse { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + _slots: &[Option], + ) -> VortexResult<()> { SparseData::validate(data.patches(), data.fill_scalar(), dtype, len) } - fn array_hash(array: &SparseData, state: &mut H, precision: Precision) { - array.patches.array_hash(state, precision); - array.fill_value.hash(state); - } - - fn array_eq(array: &SparseData, other: &SparseData, precision: Precision) -> bool { - array.patches.array_eq(&other.patches, precision) && array.fill_value == other.fill_value - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 1 } @@ -125,7 +137,7 @@ impl VTable for Sparse { buffers: &[BufferHandle], children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = SparseMetadata::decode(metadata)?; // Once we have the patches metadata, we need to get the fill value from the buffers. @@ -152,53 +164,22 @@ impl VTable for Sparse { )?; let patch_values = children.get(1, dtype, metadata.patches.len()?)?; - SparseData::try_new_from_patches( - Patches::new( - len, - metadata.patches.offset()?, - patch_indices, - patch_values, - None, - )?, - fill_value, - ) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let patches = Patches::new( + len, + metadata.patches.offset()?, + patch_indices, + patch_values, + None, + )?; + let slots = SparseData::make_slots(&patches); + let data = SparseData::try_new_from_patches(patches, fill_value)?; + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "SparseArray expects {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - - // Reconstruct patches from slots + existing metadata - let indices = slots[PATCH_INDICES_SLOT] - .clone() - .vortex_expect("SparseArray requires patch_indices slot"); - let values = slots[PATCH_VALUES_SLOT] - .clone() - .vortex_expect("SparseArray requires patch_values slot"); - - array.patches = Patches::new( - array.patches.array_len(), - array.patches.offset(), - indices, - values, - slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), - )?; - array.slots = slots; - Ok(()) - } - fn reduce_parent( array: ArrayView<'_, Self>, parent: &ArrayRef, @@ -221,19 +202,12 @@ impl VTable for Sparse { } } -/// The indices of non-fill values in the sparse array. -pub(crate) const PATCH_INDICES_SLOT: usize = 0; -/// The non-fill values at the corresponding patch indices. -pub(crate) const PATCH_VALUES_SLOT: usize = 1; -/// Chunk offsets for the patch indices/values. -pub(crate) const PATCH_CHUNK_OFFSETS_SLOT: usize = 2; pub(crate) const NUM_SLOTS: usize = 3; pub(crate) const SLOT_NAMES: [&str; NUM_SLOTS] = ["patch_indices", "patch_values", "patch_chunk_offsets"]; #[derive(Clone, Debug)] pub struct SparseData { - pub(crate) slots: Vec>, patches: Patches, fill_value: Scalar, } @@ -252,22 +226,32 @@ impl Sparse { fill_value: Scalar, ) -> VortexResult { let dtype = fill_value.dtype().clone(); - let data = SparseData::try_new(indices, values, len, fill_value)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data)) }) + let patches = Patches::new(len, 0, indices, values, None)?; + let slots = SparseData::make_slots(&patches); + let data = SparseData::try_new_from_patches(patches, fill_value)?; + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data).with_slots(slots)) + }) } pub fn try_new_from_patches(patches: Patches, fill_value: Scalar) -> VortexResult { let dtype = fill_value.dtype().clone(); let len = patches.array_len(); + let slots = SparseData::make_slots(&patches); let data = SparseData::try_new_from_patches(patches, fill_value)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data)) }) + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data).with_slots(slots)) + }) } pub(crate) unsafe fn new_unchecked(patches: Patches, fill_value: Scalar) -> SparseArray { let dtype = fill_value.dtype().clone(); let len = patches.array_len(); + let slots = SparseData::make_slots(&patches); let data = unsafe { SparseData::new_unchecked(patches, fill_value) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data)) } + unsafe { + Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data).with_slots(slots)) + } } /// Encode the given array as a [`SparseArray`]. @@ -277,25 +261,6 @@ impl Sparse { } impl SparseData { - fn normalize_values_dtype(values: ArrayRef, fill_value: &Scalar) -> VortexResult { - let fill_dtype = fill_value.dtype(); - let values_dtype = values.dtype(); - - vortex_ensure!( - values_dtype.eq_ignore_nullability(fill_dtype), - "fill value, {:?}, should be instance of values dtype, {} but was {}.", - fill_value, - values_dtype, - fill_dtype, - ); - - if values_dtype == fill_dtype { - Ok(values) - } else { - values.cast(fill_dtype.clone()) - } - } - fn normalize_patches_dtype(patches: Patches, fill_value: &Scalar) -> VortexResult { let fill_dtype = fill_value.dtype(); let values_dtype = patches.values().dtype(); @@ -350,62 +315,17 @@ impl SparseData { ] } - pub fn try_new( - indices: ArrayRef, - values: ArrayRef, - len: usize, - fill_value: Scalar, - ) -> VortexResult { - vortex_ensure!( - indices.len() == values.len(), - "Mismatched indices {} and values {} length", - indices.len(), - values.len() - ); - - if indices.is_host() { - debug_assert_eq!( - indices.statistics().compute_is_strict_sorted(), - Some(true), - "SparseArray: indices must be strict-sorted" - ); - - // Verify the indices are all in the valid range - if !indices.is_empty() { - let last_index = usize::try_from(&indices.scalar_at(indices.len() - 1)?)?; - - vortex_ensure!( - last_index < len, - "Array length was {len} but the last index is {last_index}" - ); - } - } - - let values = Self::normalize_values_dtype(values, &fill_value)?; - - // TODO(0ax1): handle chunk offsets - let patches = Patches::new(len, 0, indices, values, None)?; - Self::try_new_from_patches(patches, fill_value) - } - /// Build a new SparseArray from an existing set of patches. pub fn try_new_from_patches(patches: Patches, fill_value: Scalar) -> VortexResult { let patches = Self::normalize_patches_dtype(patches, &fill_value)?; - - let slots = Self::make_slots(&patches); - Ok(Self { - slots, patches, fill_value, }) } pub(crate) unsafe fn new_unchecked(patches: Patches, fill_value: Scalar) -> Self { - let slots = Self::make_slots(&patches); - Self { - slots, patches, fill_value, } diff --git a/encodings/zigzag/public-api.lock b/encodings/zigzag/public-api.lock index ec8ed62f664..0d8bb4674d5 100644 --- a/encodings/zigzag/public-api.lock +++ b/encodings/zigzag/public-api.lock @@ -24,15 +24,11 @@ pub type vortex_zigzag::ZigZag::OperationsVTable = vortex_zigzag::ZigZag pub type vortex_zigzag::ZigZag::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild -pub fn vortex_zigzag::ZigZag::array_eq(array: &vortex_zigzag::ZigZagData, other: &vortex_zigzag::ZigZagData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_zigzag::ZigZag::array_hash(array: &vortex_zigzag::ZigZagData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_zigzag::ZigZag::buffer(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_zigzag::ZigZag::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_zigzag::ZigZag::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_zigzag::ZigZag::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_zigzag::ZigZag::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -48,11 +44,7 @@ pub fn vortex_zigzag::ZigZag::serialize(_array: vortex_array::array::view::Array pub fn vortex_zigzag::ZigZag::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_zigzag::ZigZag::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_zigzag::ZigZag::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_zigzag::ZigZag::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_zigzag::ZigZag::validate(&self, _data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_zigzag::ZigZag @@ -60,7 +52,7 @@ pub fn vortex_zigzag::ZigZag::scalar_at(array: vortex_array::array::view::ArrayV impl vortex_array::array::vtable::validity::ValidityChild for vortex_zigzag::ZigZag -pub fn vortex_zigzag::ZigZag::validity_child(array: &vortex_zigzag::ZigZagData) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_zigzag::ZigZag::validity_child(array: vortex_array::array::view::ArrayView<'_, vortex_zigzag::ZigZag>) -> vortex_array::array::erased::ArrayRef impl vortex_array::arrays::dict::take::TakeExecute for vortex_zigzag::ZigZag @@ -86,26 +78,42 @@ pub struct vortex_zigzag::ZigZagData impl vortex_zigzag::ZigZagData -pub fn vortex_zigzag::ZigZagData::encoded(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn vortex_zigzag::ZigZagData::is_empty(&self) -> bool +pub fn vortex_zigzag::ZigZagData::new() -> Self -pub fn vortex_zigzag::ZigZagData::len(&self) -> usize - -pub fn vortex_zigzag::ZigZagData::new(encoded: vortex_array::array::erased::ArrayRef) -> Self - -pub fn vortex_zigzag::ZigZagData::ptype(&self) -> vortex_array::dtype::ptype::PType - -pub fn vortex_zigzag::ZigZagData::try_new(encoded: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_zigzag::ZigZagData::try_new(encoded_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult impl core::clone::Clone for vortex_zigzag::ZigZagData pub fn vortex_zigzag::ZigZagData::clone(&self) -> vortex_zigzag::ZigZagData +impl core::default::Default for vortex_zigzag::ZigZagData + +pub fn vortex_zigzag::ZigZagData::default() -> Self + impl core::fmt::Debug for vortex_zigzag::ZigZagData pub fn vortex_zigzag::ZigZagData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_zigzag::ZigZagData + +pub fn vortex_zigzag::ZigZagData::array_eq(&self, _other: &Self, _precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_zigzag::ZigZagData + +pub fn vortex_zigzag::ZigZagData::array_hash(&self, _state: &mut H, _precision: vortex_array::hash::Precision) + +pub trait vortex_zigzag::ZigZagArrayExt: vortex_array::array::typed::TypedArrayRef + +pub fn vortex_zigzag::ZigZagArrayExt::encoded(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn vortex_zigzag::ZigZagArrayExt::ptype(&self) -> vortex_array::dtype::ptype::PType + +impl> vortex_zigzag::ZigZagArrayExt for T + +pub fn T::encoded(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn T::ptype(&self) -> vortex_array::dtype::ptype::PType + pub fn vortex_zigzag::zigzag_decode(parray: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_array::arrays::primitive::vtable::PrimitiveArray pub fn vortex_zigzag::zigzag_encode(parray: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index 896afd67fb3..1f1fff3559a 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::hash::Hasher; + use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; @@ -12,6 +14,7 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::TypedArrayRef; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::PType; @@ -48,28 +51,29 @@ impl VTable for ZigZag { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - let expected_dtype = ZigZagData::dtype_from_encoded_dtype(data.encoded().dtype())?; + fn validate( + &self, + _data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let encoded = slots[ENCODED_SLOT] + .as_ref() + .vortex_expect("ZigZagArray encoded slot"); + let expected_dtype = ZigZagData::dtype_from_encoded_dtype(encoded.dtype())?; vortex_ensure!( dtype == &expected_dtype, "expected dtype {expected_dtype}, got {dtype}" ); vortex_ensure!( - data.encoded().len() == len, + encoded.len() == len, "expected len {len}, got {}", - data.encoded().len() + encoded.len() ); Ok(()) } - fn array_hash(array: &ZigZagData, state: &mut H, precision: Precision) { - array.encoded().array_hash(state, precision); - } - - fn array_eq(array: &ZigZagData, other: &ZigZagData, precision: Precision) -> bool { - array.encoded().array_eq(other.encoded(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -94,7 +98,7 @@ impl VTable for ZigZag { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { if !metadata.is_empty() { vortex_bail!( "ZigZagArray expects empty metadata, got {} bytes", @@ -109,28 +113,15 @@ impl VTable for ZigZag { let encoded_type = DType::Primitive(ptype.to_unsigned(), dtype.nullability()); let encoded = children.get(0, &encoded_type, len)?; - ZigZagData::try_new(encoded) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let slots = vec![Some(encoded.clone())]; + let data = ZigZagData::try_new(encoded.dtype())?; + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "ZigZagArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done( zigzag_decode(array.encoded().clone().execute(ctx)?).into_array(), @@ -155,16 +146,40 @@ impl VTable for ZigZag { } } +impl ArrayHash for ZigZagData { + fn array_hash(&self, _state: &mut H, _precision: Precision) {} +} + +impl ArrayEq for ZigZagData { + fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { + true + } +} + /// The zigzag-encoded values (signed integers mapped to unsigned). pub(super) const ENCODED_SLOT: usize = 0; pub(super) const NUM_SLOTS: usize = 1; pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["encoded"]; #[derive(Clone, Debug)] -pub struct ZigZagData { - pub(super) slots: Vec>, +pub struct ZigZagData {} + +pub trait ZigZagArrayExt: TypedArrayRef { + fn encoded(&self) -> &ArrayRef { + self.as_ref().slots()[ENCODED_SLOT] + .as_ref() + .vortex_expect("ZigZagArray encoded slot") + } + + fn ptype(&self) -> PType { + PType::try_from(self.encoded().dtype()) + .vortex_expect("ZigZagArray encoded dtype") + .to_signed() + } } +impl> ZigZagArrayExt for T {} + #[derive(Clone, Debug)] pub struct ZigZag; @@ -173,10 +188,13 @@ impl ZigZag { /// Construct a new [`ZigZagArray`] from an encoded unsigned integer array. pub fn try_new(encoded: ArrayRef) -> VortexResult { - let data = ZigZagData::try_new(encoded)?; - let dtype = ZigZagData::dtype_from_encoded_dtype(data.encoded().dtype())?; - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(ZigZag, dtype, len, data)) }) + let dtype = ZigZagData::dtype_from_encoded_dtype(encoded.dtype())?; + let len = encoded.len(); + let slots = vec![Some(encoded.clone())]; + let data = ZigZagData::try_new(encoded.dtype())?; + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(ZigZag, dtype, len, data).with_slots(slots)) + }) } } @@ -186,45 +204,24 @@ impl ZigZagData { .with_nullability(encoded_dtype.nullability())) } - pub fn new(encoded: ArrayRef) -> Self { - Self::try_new(encoded).vortex_expect("ZigZagArray new") + pub fn new() -> Self { + Self {} } - pub fn try_new(encoded: ArrayRef) -> VortexResult { - let encoded_dtype = encoded.dtype().clone(); + pub fn try_new(encoded_dtype: &DType) -> VortexResult { if !encoded_dtype.is_unsigned_int() { vortex_bail!(MismatchedTypes: "unsigned int", encoded_dtype); } - Self::dtype_from_encoded_dtype(&encoded_dtype)?; - - Ok(Self { - slots: vec![Some(encoded)], - }) - } - - /// Returns the length of the array. - #[inline] - pub fn len(&self) -> usize { - self.encoded().len() - } + Self::dtype_from_encoded_dtype(encoded_dtype)?; - /// Returns whether the array is empty. - #[inline] - pub fn is_empty(&self) -> bool { - self.encoded().is_empty() - } - - pub fn ptype(&self) -> PType { - PType::try_from(self.encoded().dtype()) - .vortex_expect("ZigZagArray encoded dtype") - .to_signed() + Ok(Self {}) } +} - pub fn encoded(&self) -> &ArrayRef { - self.slots[ENCODED_SLOT] - .as_ref() - .vortex_expect("ZigZagArray encoded slot") +impl Default for ZigZagData { + fn default() -> Self { + Self::new() } } @@ -236,7 +233,7 @@ impl OperationsVTable for ZigZag { ) -> VortexResult { let scalar = array.encoded().scalar_at(index)?; if scalar.is_null() { - return scalar.primitive_reinterpret_cast(array.ptype()); + return scalar.primitive_reinterpret_cast(ZigZagArrayExt::ptype(&array)); } let pscalar = scalar.as_primitive(); @@ -254,8 +251,8 @@ impl OperationsVTable for ZigZag { } impl ValidityChild for ZigZag { - fn validity_child(array: &ZigZagData) -> &ArrayRef { - array.encoded() + fn validity_child(array: ArrayView<'_, ZigZag>) -> ArrayRef { + array.encoded().clone() } } diff --git a/encodings/zigzag/src/compute/cast.rs b/encodings/zigzag/src/compute/cast.rs index 138c6aae736..7f6741d9aa1 100644 --- a/encodings/zigzag/src/compute/cast.rs +++ b/encodings/zigzag/src/compute/cast.rs @@ -10,6 +10,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::ZigZag; +use crate::array::ZigZagArrayExt; impl CastReduce for ZigZag { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { if !dtype.is_signed_int() { diff --git a/encodings/zigzag/src/compute/mod.rs b/encodings/zigzag/src/compute/mod.rs index 11fe580e6bc..71938dbcc18 100644 --- a/encodings/zigzag/src/compute/mod.rs +++ b/encodings/zigzag/src/compute/mod.rs @@ -9,7 +9,7 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::dict::TakeExecute; use vortex_array::arrays::filter::FilterReduce; -use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnFactoryExt; use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::fns::mask::Mask as MaskExpr; use vortex_array::scalar_fn::fns::mask::MaskReduce; @@ -17,6 +17,7 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::ZigZag; +use crate::array::ZigZagArrayExt; impl FilterReduce for ZigZag { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { diff --git a/encodings/zigzag/src/slice.rs b/encodings/zigzag/src/slice.rs index e027cda04cb..6a7128d5111 100644 --- a/encodings/zigzag/src/slice.rs +++ b/encodings/zigzag/src/slice.rs @@ -10,6 +10,7 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::ZigZag; +use crate::array::ZigZagArrayExt; impl SliceReduce for ZigZag { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { diff --git a/encodings/zstd/public-api.lock b/encodings/zstd/public-api.lock index dc7e1e63809..e3ce21c73c1 100644 --- a/encodings/zstd/public-api.lock +++ b/encodings/zstd/public-api.lock @@ -32,15 +32,11 @@ pub type vortex_zstd::Zstd::OperationsVTable = vortex_zstd::Zstd pub type vortex_zstd::Zstd::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromValiditySliceHelper -pub fn vortex_zstd::Zstd::array_eq(array: &vortex_zstd::ZstdData, other: &vortex_zstd::ZstdData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_zstd::Zstd::array_hash(array: &vortex_zstd::ZstdData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_zstd::Zstd::buffer(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_zstd::Zstd::buffer_name(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_zstd::Zstd::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_zstd::Zstd::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_zstd::Zstd::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -54,11 +50,7 @@ pub fn vortex_zstd::Zstd::serialize(array: vortex_array::array::view::ArrayView< pub fn vortex_zstd::Zstd::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_zstd::Zstd::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_zstd::Zstd::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_zstd::Zstd::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_zstd::Zstd::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_zstd::Zstd @@ -138,6 +130,14 @@ impl vortex_array::array::vtable::validity::ValiditySliceHelper for vortex_zstd: pub fn vortex_zstd::ZstdData::unsliced_validity_and_slice(&self) -> (&vortex_array::validity::Validity, usize, usize) +impl vortex_array::hash::ArrayEq for vortex_zstd::ZstdData + +pub fn vortex_zstd::ZstdData::array_eq(&self, other: &Self, precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_zstd::ZstdData + +pub fn vortex_zstd::ZstdData::array_hash(&self, state: &mut H, precision: vortex_array::hash::Precision) + pub struct vortex_zstd::ZstdDataParts pub vortex_zstd::ZstdDataParts::dictionary: core::option::Option diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index 48e25ec1512..f2378e52e5f 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -3,6 +3,7 @@ use std::fmt::Debug; use std::hash::Hash; +use std::hash::Hasher; use std::sync::Arc; use itertools::Itertools as _; @@ -81,22 +82,9 @@ type ViewLen = u32; vtable!(Zstd, Zstd, ZstdData); -impl VTable for Zstd { - type ArrayData = ZstdData; - - type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromValiditySliceHelper; - - fn id(&self) -> ArrayId { - Self::ID - } - - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate(dtype, len) - } - - fn array_hash(array: &ZstdData, state: &mut H, precision: Precision) { - match &array.dictionary { +impl ArrayHash for ZstdData { + fn array_hash(&self, state: &mut H, precision: Precision) { + match &self.dictionary { Some(dict) => { true.hash(state); dict.array_hash(state, precision); @@ -105,37 +93,59 @@ impl VTable for Zstd { false.hash(state); } } - for frame in &array.frames { + for frame in &self.frames { frame.array_hash(state, precision); } - array.unsliced_validity.array_hash(state, precision); - array.unsliced_n_rows.hash(state); - array.slice_start.hash(state); - array.slice_stop.hash(state); + self.unsliced_validity.array_hash(state, precision); + self.unsliced_n_rows.hash(state); + self.slice_start.hash(state); + self.slice_stop.hash(state); } +} - fn array_eq(array: &ZstdData, other: &ZstdData, precision: Precision) -> bool { - if !match (&array.dictionary, &other.dictionary) { +impl ArrayEq for ZstdData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + if !match (&self.dictionary, &other.dictionary) { (Some(d1), Some(d2)) => d1.array_eq(d2, precision), (None, None) => true, _ => false, } { return false; } - if array.frames.len() != other.frames.len() { + if self.frames.len() != other.frames.len() { return false; } - for (a, b) in array.frames.iter().zip(&other.frames) { + for (a, b) in self.frames.iter().zip(&other.frames) { if !a.array_eq(b, precision) { return false; } } - array - .unsliced_validity + self.unsliced_validity .array_eq(&other.unsliced_validity, precision) - && array.unsliced_n_rows == other.unsliced_n_rows - && array.slice_start == other.slice_start - && array.slice_stop == other.slice_stop + && self.unsliced_n_rows == other.unsliced_n_rows + && self.slice_start == other.slice_start + && self.slice_stop == other.slice_stop + } +} + +impl VTable for Zstd { + type ArrayData = ZstdData; + + type OperationsVTable = Self; + type ValidityVTable = ValidityVTableFromValiditySliceHelper; + + fn id(&self) -> ArrayId { + Self::ID + } + + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + _slots: &[Option], + ) -> VortexResult<()> { + data.validate(dtype, len) } fn nbuffers(array: ArrayView<'_, Self>) -> usize { @@ -177,7 +187,7 @@ impl VTable for Zstd { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = ZstdMetadata::decode(metadata)?; let validity = if children.is_empty() { Validity::from(dtype.nullability()) @@ -208,40 +218,21 @@ impl VTable for Zstd { ) }; - Ok(ZstdData::new( + let slots = vec![validity_to_child(&validity, len)]; + let data = ZstdData::new( dictionary_buffer, compressed_buffers, metadata, len, validity, - )) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + ); + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "ZstdArray expects {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - - array.unsliced_validity = match &slots[VALIDITY_SLOT] { - Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.unsliced_validity.nullability()), - }; - - array.slots = slots; - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { Zstd::decompress(&array, ctx)? .execute::(ctx) @@ -266,7 +257,13 @@ impl Zstd { pub fn try_new(dtype: DType, data: ZstdData) -> VortexResult { let len = data.len(); data.validate(&dtype, len)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Zstd, dtype, len, data)) }) + let slots = vec![validity_to_child( + &data.unsliced_validity, + data.unsliced_n_rows, + )]; + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(Zstd, dtype, len, data).with_slots(slots)) + }) } /// Compress a [`VarBinViewArray`] using Zstd without a dictionary. @@ -311,7 +308,6 @@ impl Zstd { } /// The validity bitmap indicating which elements are non-null. -pub(super) const VALIDITY_SLOT: usize = 0; pub(super) const NUM_SLOTS: usize = 1; pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; @@ -322,7 +318,6 @@ pub struct ZstdData { pub(crate) metadata: ZstdMetadata, pub(crate) unsliced_validity: Validity, unsliced_n_rows: usize, - pub(super) slots: Vec>, slice_start: usize, slice_stop: usize, } @@ -443,14 +438,12 @@ impl ZstdData { n_rows: usize, validity: Validity, ) -> Self { - let validity_slot = validity_to_child(&validity, n_rows); Self { dictionary, frames, metadata, unsliced_validity: validity, unsliced_n_rows: n_rows, - slots: vec![validity_slot], slice_start: 0, slice_stop: n_rows, } diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index 1e74a69a9a0..f76cf507706 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -3,6 +3,7 @@ use std::fmt::Debug; use std::hash::Hash; +use std::hash::Hasher; use std::sync::Arc; use prost::Message as _; @@ -79,9 +80,12 @@ impl ZstdBuffers { compressed_buffers, uncompressed_sizes, buffer_alignments, - slots: children.into_iter().map(Some).collect(), }; - let compressed = Self::try_new(array.dtype().clone(), array.len(), data)?; + let slots = children.into_iter().map(Some).collect(); + let compressed = Array::try_from_parts( + ArrayParts::new(ZstdBuffers, array.dtype().clone(), array.len(), data) + .with_slots(slots), + )?; compressed.statistics().inherit_from(array.statistics()); Ok(compressed) } @@ -91,9 +95,23 @@ impl ZstdBuffers { buffer_handles: &[BufferHandle], session: &VortexSession, ) -> VortexResult { - array - .data() - .build_inner(array.dtype(), array.len(), buffer_handles, session) + let registry = session.arrays().registry().clone(); + let inner_vtable = registry + .find(&array.data().inner_encoding_id) + .ok_or_else(|| { + vortex_err!("Unknown inner encoding: {}", array.data().inner_encoding_id) + })?; + + let children: Vec = array.slots().iter().flatten().cloned().collect(); + inner_vtable.build( + array.data().inner_encoding_id.clone(), + array.dtype(), + array.len(), + &array.data().inner_metadata, + buffer_handles, + &children.as_slice(), + session, + ) } fn decompress_and_build_inner( @@ -117,7 +135,6 @@ pub struct ZstdBuffersData { compressed_buffers: Vec, uncompressed_sizes: Vec, buffer_alignments: Vec, - pub(crate) slots: Vec>, } #[derive(Clone, Debug)] @@ -245,42 +262,6 @@ impl ZstdBuffersData { Ok(result) } - fn decompress_and_build_inner( - &self, - dtype: &DType, - len: usize, - session: &VortexSession, - ) -> VortexResult { - let decompressed_buffers = self.decompress_buffers()?; - self.build_inner(dtype, len, &decompressed_buffers, session) - } - - // This is exposed to help non-CPU executors pass uncompressed buffer handles - // to build the inner array. - pub fn build_inner( - &self, - dtype: &DType, - len: usize, - buffer_handles: &[BufferHandle], - session: &VortexSession, - ) -> VortexResult { - let registry = session.arrays().registry().clone(); - let inner_vtable = registry - .find(&self.inner_encoding_id) - .ok_or_else(|| vortex_err!("Unknown inner encoding: {}", self.inner_encoding_id))?; - - let children: Vec = self.slots.iter().flatten().cloned().collect(); - inner_vtable.build( - self.inner_encoding_id.clone(), - dtype, - len, - &self.inner_metadata, - buffer_handles, - &children.as_slice(), - session, - ) - } - pub fn decode_plan(&self) -> VortexResult { // If invariants are somehow broken, device decompression could have UB, so ensure // they still hold. @@ -344,6 +325,33 @@ fn array_id_from_string(s: &str) -> ArrayId { ArrayId::new_arc(Arc::from(s)) } +impl ArrayHash for ZstdBuffersData { + fn array_hash(&self, state: &mut H, precision: Precision) { + self.inner_encoding_id.hash(state); + self.inner_metadata.hash(state); + for buf in &self.compressed_buffers { + buf.array_hash(state, precision); + } + self.uncompressed_sizes.hash(state); + self.buffer_alignments.hash(state); + } +} + +impl ArrayEq for ZstdBuffersData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + self.inner_encoding_id == other.inner_encoding_id + && self.inner_metadata == other.inner_metadata + && self.compressed_buffers.len() == other.compressed_buffers.len() + && self + .compressed_buffers + .iter() + .zip(&other.compressed_buffers) + .all(|(a, b)| a.array_eq(b, precision)) + && self.uncompressed_sizes == other.uncompressed_sizes + && self.buffer_alignments == other.buffer_alignments + } +} + impl VTable for ZstdBuffers { type ArrayData = ZstdBuffersData; type OperationsVTable = Self; @@ -353,47 +361,16 @@ impl VTable for ZstdBuffers { Self::ID } - fn validate(&self, data: &Self::ArrayData, _dtype: &DType, _len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &Self::ArrayData, + _dtype: &DType, + _len: usize, + _slots: &[Option], + ) -> VortexResult<()> { data.validate() } - fn array_hash( - array: &ZstdBuffersData, - state: &mut H, - precision: Precision, - ) { - array.inner_encoding_id.hash(state); - array.inner_metadata.hash(state); - for buf in &array.compressed_buffers { - buf.array_hash(state, precision); - } - array.uncompressed_sizes.hash(state); - array.buffer_alignments.hash(state); - for child in array.slots.iter().flatten() { - child.array_hash(state, precision); - } - } - - fn array_eq(array: &ZstdBuffersData, other: &ZstdBuffersData, precision: Precision) -> bool { - array.inner_encoding_id == other.inner_encoding_id - && array.inner_metadata == other.inner_metadata - && array.compressed_buffers.len() == other.compressed_buffers.len() - && array - .compressed_buffers - .iter() - .zip(&other.compressed_buffers) - .all(|(a, b)| a.array_eq(b, precision)) - && array.uncompressed_sizes == other.uncompressed_sizes - && array.buffer_alignments == other.buffer_alignments - && array.slots.len() == other.slots.len() - && array - .slots - .iter() - .flatten() - .zip(other.slots.iter().flatten()) - .all(|(a, b)| a.array_eq(b, precision)) - } - fn nbuffers(array: ArrayView<'_, Self>) -> usize { array.compressed_buffers.len() } @@ -406,19 +383,10 @@ impl VTable for ZstdBuffers { Some(format!("compressed_{idx}")) } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { format!("child_{idx}") } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - array.slots = slots; - Ok(()) - } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( ZstdBuffersMetadata { @@ -439,11 +407,11 @@ impl VTable for ZstdBuffers { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = ZstdBuffersMetadata::decode(metadata)?; let compressed_buffers: Vec = buffers.to_vec(); - let child_arrays: Vec> = (0..children.len()) + let slots: Vec> = (0..children.len()) .map(|i| children.get(i, dtype, len).map(Some)) .collect::>>()?; @@ -453,11 +421,10 @@ impl VTable for ZstdBuffers { compressed_buffers, uncompressed_sizes: metadata.uncompressed_sizes.clone(), buffer_alignments: metadata.buffer_alignments.clone(), - slots: child_arrays, }; data.validate()?; - Ok(data) + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } // with_slots handles child replacement via the slots mechanism @@ -480,9 +447,8 @@ impl OperationsVTable for ZstdBuffers { // TODO(os): maybe we should not support scalar_at, it is really slow, and adding a cache // layer here is weird. Valid use of zstd buffers array would be by executing it first into // canonical - let inner_array = array.data().decompress_and_build_inner( - array.dtype(), - array.len(), + let inner_array = ZstdBuffers::decompress_and_build_inner( + &array.into_owned(), &vortex_array::LEGACY_SESSION, )?; inner_array.scalar_at(index) @@ -497,9 +463,8 @@ impl ValidityVTable for ZstdBuffers { return Ok(vortex_array::validity::Validity::NonNullable); } - let inner_array = array.data().decompress_and_build_inner( - array.dtype(), - array.len(), + let inner_array = ZstdBuffers::decompress_and_build_inner( + &array.into_owned(), &vortex_array::LEGACY_SESSION, )?; inner_array.validity() diff --git a/fuzz/fuzz_targets/file_io.rs b/fuzz/fuzz_targets/file_io.rs index 2c306561723..7d728905148 100644 --- a/fuzz/fuzz_targets/file_io.rs +++ b/fuzz/fuzz_targets/file_io.rs @@ -11,6 +11,7 @@ use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::ToCanonical; use vortex_array::arrays::ChunkedArray; +use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::StructFields; diff --git a/fuzz/src/array/compare.rs b/fuzz/src/array/compare.rs index df42ee81f64..e6d069d47b1 100644 --- a/fuzz/src/array/compare.rs +++ b/fuzz/src/array/compare.rs @@ -6,6 +6,7 @@ use vortex_array::IntoArray; use vortex_array::ToCanonical; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; +use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::arrays::primitive::NativeValue; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; diff --git a/fuzz/src/array/fill_null.rs b/fuzz/src/array/fill_null.rs index 3c47cb06542..1a66d6f9773 100644 --- a/fuzz/src/array/fill_null.rs +++ b/fuzz/src/array/fill_null.rs @@ -10,6 +10,7 @@ use vortex_array::arrays::ConstantArray; use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; diff --git a/fuzz/src/array/filter.rs b/fuzz/src/array/filter.rs index 9431cac68f6..7e001d20696 100644 --- a/fuzz/src/array/filter.rs +++ b/fuzz/src/array/filter.rs @@ -10,6 +10,8 @@ use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::bool::BoolArrayExt; +use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; use vortex_array::match_each_decimal_value_type; use vortex_array::match_each_native_ptype; diff --git a/fuzz/src/array/mask.rs b/fuzz/src/array/mask.rs index 1155db60926..1a13207830c 100644 --- a/fuzz/src/array/mask.rs +++ b/fuzz/src/array/mask.rs @@ -13,6 +13,11 @@ use vortex_array::arrays::ListViewArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::bool::BoolArrayExt; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::listview::ListViewArrayExt; +use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::Nullability; use vortex_array::match_each_decimal_value_type; use vortex_array::validity::Validity; diff --git a/fuzz/src/array/scalar_at.rs b/fuzz/src/array/scalar_at.rs index 4fb29b1b69e..91898390df6 100644 --- a/fuzz/src/array/scalar_at.rs +++ b/fuzz/src/array/scalar_at.rs @@ -5,6 +5,11 @@ use std::sync::Arc; use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::arrays::bool::BoolArrayExt; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::listview::ListViewArrayExt; +use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::arrays::varbin::varbin_scalar; use vortex_array::dtype::DType; use vortex_array::match_each_decimal_value_type; diff --git a/fuzz/src/array/search_sorted.rs b/fuzz/src/array/search_sorted.rs index 068e5281f73..d22b583a39b 100644 --- a/fuzz/src/array/search_sorted.rs +++ b/fuzz/src/array/search_sorted.rs @@ -7,6 +7,7 @@ use std::fmt::Debug; use vortex_array::ArrayRef; use vortex_array::ToCanonical; use vortex_array::accessor::ArrayAccessor; +use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::match_each_decimal_value_type; diff --git a/fuzz/src/array/slice.rs b/fuzz/src/array/slice.rs index dce0c9c3faa..d4574eea436 100644 --- a/fuzz/src/array/slice.rs +++ b/fuzz/src/array/slice.rs @@ -12,6 +12,10 @@ use vortex_array::arrays::ListViewArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::bool::BoolArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::listview::ListViewArrayExt; +use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; use vortex_array::match_each_decimal_value_type; use vortex_array::match_each_native_ptype; diff --git a/fuzz/src/array/sort.rs b/fuzz/src/array/sort.rs index e1a57de5658..6c07c76c20d 100644 --- a/fuzz/src/array/sort.rs +++ b/fuzz/src/array/sort.rs @@ -11,6 +11,7 @@ use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::match_each_decimal_value_type; diff --git a/fuzz/src/array/take.rs b/fuzz/src/array/take.rs index d807c37c21c..a113829077b 100644 --- a/fuzz/src/array/take.rs +++ b/fuzz/src/array/take.rs @@ -10,6 +10,8 @@ use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::bool::BoolArrayExt; +use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::builders::builder_with_capacity; use vortex_array::dtype::DType; use vortex_array::dtype::DecimalDType; diff --git a/fuzz/src/fsst_like.rs b/fuzz/src/fsst_like.rs index 8c1cc108fd8..b962c4244c8 100644 --- a/fuzz/src/fsst_like.rs +++ b/fuzz/src/fsst_like.rs @@ -16,6 +16,7 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::VarBinArray; +use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar_fn::fns::like::Like; @@ -158,7 +159,7 @@ fn run_like_on_array( len: usize, opts: LikeOptions, ) -> VortexResult { - use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; + use vortex_array::arrays::scalar_fn::ScalarFnFactoryExt; let pattern_arr = ConstantArray::new(pattern, len).into_array(); let result = Like diff --git a/rust-toolchain.toml b/rust-toolchain.toml index e6c94d091a7..c8eba8175ae 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] channel = "1.90" components = ["rust-src", "rustfmt", "clippy", "rust-analyzer"] -profile = "minimal" +profile = "minimal" \ No newline at end of file diff --git a/vortex-array/benches/dict_unreferenced_mask.rs b/vortex-array/benches/dict_unreferenced_mask.rs index 7a4b3a93c6e..24cf366f692 100644 --- a/vortex-array/benches/dict_unreferenced_mask.rs +++ b/vortex-array/benches/dict_unreferenced_mask.rs @@ -10,6 +10,7 @@ use rand::rngs::StdRng; use vortex_array::IntoArray; use vortex_array::arrays::DictArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::dict::DictArrayExt; fn main() { divan::main(); diff --git a/vortex-array/benches/listview_rebuild.rs b/vortex-array/benches/listview_rebuild.rs index 90e1560e8af..6affedca45f 100644 --- a/vortex-array/benches/listview_rebuild.rs +++ b/vortex-array/benches/listview_rebuild.rs @@ -14,6 +14,7 @@ use vortex_array::arrays::ListViewArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::listview::ListViewArrayExt; use vortex_array::arrays::listview::ListViewRebuildMode; use vortex_array::dtype::FieldNames; use vortex_array::validity::Validity; diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 7ef7bdb5c83..283d750b1e2 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -976,10 +976,6 @@ pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool pub fn vortex_array::arrays::Bool::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Bool::array_eq(array: &vortex_array::arrays::bool::BoolData, other: &vortex_array::arrays::bool::BoolData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Bool::array_hash(array: &vortex_array::arrays::bool::BoolData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -988,7 +984,7 @@ pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -1010,11 +1006,7 @@ pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Bool @@ -1054,27 +1046,7 @@ pub struct vortex_array::arrays::bool::BoolData impl vortex_array::arrays::bool::BoolData -pub fn vortex_array::arrays::bool::BoolData::dtype(&self) -> vortex_array::dtype::DType - -pub fn vortex_array::arrays::bool::BoolData::into_bit_buffer(self) -> vortex_buffer::bit::buf::BitBuffer - -pub fn vortex_array::arrays::bool::BoolData::into_parts(self) -> vortex_array::arrays::bool::BoolDataParts - -pub fn vortex_array::arrays::bool::BoolData::is_empty(&self) -> bool - -pub fn vortex_array::arrays::bool::BoolData::len(&self) -> usize - -pub fn vortex_array::arrays::bool::BoolData::maybe_to_mask(&self) -> vortex_error::VortexResult> - -pub fn vortex_array::arrays::bool::BoolData::to_bit_buffer(&self) -> vortex_buffer::bit::buf::BitBuffer - -pub fn vortex_array::arrays::bool::BoolData::to_mask(&self) -> vortex_mask::Mask - -pub fn vortex_array::arrays::bool::BoolData::to_mask_fill_null_false(&self) -> vortex_mask::Mask - -pub fn vortex_array::arrays::bool::BoolData::validity(&self) -> vortex_array::validity::Validity - -pub fn vortex_array::arrays::bool::BoolData::validity_mask(&self) -> vortex_mask::Mask +pub fn vortex_array::arrays::bool::BoolData::into_parts(self, len: usize) -> vortex_array::arrays::bool::BoolDataParts impl core::clone::Clone for vortex_array::arrays::bool::BoolData @@ -1084,6 +1056,14 @@ impl core::fmt::Debug for vortex_array::arrays::bool::BoolData pub fn vortex_array::arrays::bool::BoolData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::ArrayEq for vortex_array::arrays::bool::BoolData + +pub fn vortex_array::arrays::bool::BoolData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayHash for vortex_array::arrays::bool::BoolData + +pub fn vortex_array::arrays::bool::BoolData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) + pub struct vortex_array::arrays::bool::BoolDataParts pub vortex_array::arrays::bool::BoolDataParts::bits: vortex_array::buffer::BufferHandle @@ -1092,8 +1072,6 @@ pub vortex_array::arrays::bool::BoolDataParts::len: usize pub vortex_array::arrays::bool::BoolDataParts::offset: usize -pub vortex_array::arrays::bool::BoolDataParts::validity: vortex_array::validity::Validity - pub struct vortex_array::arrays::bool::BoolMaskedValidityRule impl core::default::Default for vortex_array::arrays::bool::BoolMaskedValidityRule @@ -1110,6 +1088,38 @@ pub type vortex_array::arrays::bool::BoolMaskedValidityRule::Parent = vortex_arr pub fn vortex_array::arrays::bool::BoolMaskedValidityRule::reduce_parent(&self, array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, parent: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, child_idx: usize) -> vortex_error::VortexResult> +pub trait vortex_array::arrays::bool::BoolArrayExt: vortex_array::TypedArrayRef + +pub fn vortex_array::arrays::bool::BoolArrayExt::bool_validity_mask(&self) -> vortex_mask::Mask + +pub fn vortex_array::arrays::bool::BoolArrayExt::maybe_to_mask(&self) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::bool::BoolArrayExt::nullability(&self) -> vortex_array::dtype::Nullability + +pub fn vortex_array::arrays::bool::BoolArrayExt::to_bit_buffer(&self) -> vortex_buffer::bit::buf::BitBuffer + +pub fn vortex_array::arrays::bool::BoolArrayExt::to_mask(&self) -> vortex_mask::Mask + +pub fn vortex_array::arrays::bool::BoolArrayExt::to_mask_fill_null_false(&self) -> vortex_mask::Mask + +pub fn vortex_array::arrays::bool::BoolArrayExt::validity(&self) -> vortex_array::validity::Validity + +impl> vortex_array::arrays::bool::BoolArrayExt for T + +pub fn T::bool_validity_mask(&self) -> vortex_mask::Mask + +pub fn T::maybe_to_mask(&self) -> vortex_error::VortexResult> + +pub fn T::nullability(&self) -> vortex_array::dtype::Nullability + +pub fn T::to_bit_buffer(&self) -> vortex_buffer::bit::buf::BitBuffer + +pub fn T::to_mask(&self) -> vortex_mask::Mask + +pub fn T::to_mask_fill_null_false(&self) -> vortex_mask::Mask + +pub fn T::validity(&self) -> vortex_array::validity::Validity + pub type vortex_array::arrays::bool::BoolArray = vortex_array::Array pub mod vortex_array::arrays::chunked @@ -1142,10 +1152,6 @@ pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::C pub fn vortex_array::arrays::Chunked::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Chunked::array_eq(array: &vortex_array::arrays::chunked::ChunkedData, other: &vortex_array::arrays::chunked::ChunkedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Chunked::array_hash(array: &vortex_array::arrays::chunked::ChunkedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -1154,7 +1160,7 @@ pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -1176,11 +1182,7 @@ pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Chunked @@ -1218,43 +1220,67 @@ pub struct vortex_array::arrays::chunked::ChunkedData impl vortex_array::arrays::chunked::ChunkedData -pub fn vortex_array::arrays::chunked::ChunkedData::array_iterator(&self) -> impl vortex_array::iter::ArrayIterator + '_ +pub fn vortex_array::arrays::chunked::ChunkedData::validate(chunks: &[vortex_array::ArrayRef], dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult<()> + +impl core::clone::Clone for vortex_array::arrays::chunked::ChunkedData -pub fn vortex_array::arrays::chunked::ChunkedData::array_stream(&self) -> impl vortex_array::stream::ArrayStream + '_ +pub fn vortex_array::arrays::chunked::ChunkedData::clone(&self) -> vortex_array::arrays::chunked::ChunkedData -pub fn vortex_array::arrays::chunked::ChunkedData::chunk(&self, idx: usize) -> &vortex_array::ArrayRef +impl core::fmt::Debug for vortex_array::arrays::chunked::ChunkedData -pub fn vortex_array::arrays::chunked::ChunkedData::chunk_offsets(&self) -> vortex_buffer::buffer::Buffer +pub fn vortex_array::arrays::chunked::ChunkedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::chunked::ChunkedData::chunks(&self) -> alloc::vec::Vec +impl vortex_array::ArrayEq for vortex_array::arrays::chunked::ChunkedData -pub fn vortex_array::arrays::chunked::ChunkedData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::chunked::ChunkedData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -pub fn vortex_array::arrays::chunked::ChunkedData::is_empty(&self) -> bool +impl vortex_array::ArrayHash for vortex_array::arrays::chunked::ChunkedData -pub fn vortex_array::arrays::chunked::ChunkedData::iter_chunks(&self) -> impl core::iter::traits::iterator::Iterator + '_ +pub fn vortex_array::arrays::chunked::ChunkedData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) -pub fn vortex_array::arrays::chunked::ChunkedData::len(&self) -> usize +pub trait vortex_array::arrays::chunked::ChunkedArrayExt: vortex_array::TypedArrayRef -pub fn vortex_array::arrays::chunked::ChunkedData::nchunks(&self) -> usize +pub fn vortex_array::arrays::chunked::ChunkedArrayExt::array_iterator(&self) -> impl vortex_array::iter::ArrayIterator + '_ -pub unsafe fn vortex_array::arrays::chunked::ChunkedData::new_unchecked(chunks: alloc::vec::Vec, dtype: vortex_array::dtype::DType) -> Self +pub fn vortex_array::arrays::chunked::ChunkedArrayExt::array_stream(&self) -> impl vortex_array::stream::ArrayStream + '_ -pub fn vortex_array::arrays::chunked::ChunkedData::non_empty_chunks(&self) -> impl core::iter::traits::iterator::Iterator + '_ +pub fn vortex_array::arrays::chunked::ChunkedArrayExt::chunk(&self, idx: usize) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::chunked::ChunkedData::rechunk(&self, target_bytesize: u64, target_rowsize: usize) -> vortex_error::VortexResult +pub fn vortex_array::arrays::chunked::ChunkedArrayExt::chunk_offsets(&self) -> &[usize] -pub fn vortex_array::arrays::chunked::ChunkedData::try_new(chunks: alloc::vec::Vec, dtype: vortex_array::dtype::DType) -> vortex_error::VortexResult +pub fn vortex_array::arrays::chunked::ChunkedArrayExt::chunk_offsets_array(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::chunked::ChunkedData::validate(chunks: &[vortex_array::ArrayRef], dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::chunked::ChunkedArrayExt::chunks(&self) -> alloc::vec::Vec -impl core::clone::Clone for vortex_array::arrays::chunked::ChunkedData +pub fn vortex_array::arrays::chunked::ChunkedArrayExt::find_chunk_idx(&self, index: usize) -> vortex_error::VortexResult<(usize, usize)> -pub fn vortex_array::arrays::chunked::ChunkedData::clone(&self) -> vortex_array::arrays::chunked::ChunkedData +pub fn vortex_array::arrays::chunked::ChunkedArrayExt::iter_chunks<'a>(&'a self) -> alloc::boxed::Box<(dyn core::iter::traits::iterator::Iterator + 'a)> -impl core::fmt::Debug for vortex_array::arrays::chunked::ChunkedData +pub fn vortex_array::arrays::chunked::ChunkedArrayExt::nchunks(&self) -> usize -pub fn vortex_array::arrays::chunked::ChunkedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::arrays::chunked::ChunkedArrayExt::non_empty_chunks<'a>(&'a self) -> alloc::boxed::Box<(dyn core::iter::traits::iterator::Iterator + 'a)> + +impl> vortex_array::arrays::chunked::ChunkedArrayExt for T + +pub fn T::array_iterator(&self) -> impl vortex_array::iter::ArrayIterator + '_ + +pub fn T::array_stream(&self) -> impl vortex_array::stream::ArrayStream + '_ + +pub fn T::chunk(&self, idx: usize) -> &vortex_array::ArrayRef + +pub fn T::chunk_offsets(&self) -> &[usize] + +pub fn T::chunk_offsets_array(&self) -> &vortex_array::ArrayRef + +pub fn T::chunks(&self) -> alloc::vec::Vec + +pub fn T::find_chunk_idx(&self, index: usize) -> vortex_error::VortexResult<(usize, usize)> + +pub fn T::iter_chunks<'a>(&'a self) -> alloc::boxed::Box<(dyn core::iter::traits::iterator::Iterator + 'a)> + +pub fn T::nchunks(&self) -> usize + +pub fn T::non_empty_chunks<'a>(&'a self) -> alloc::boxed::Box<(dyn core::iter::traits::iterator::Iterator + 'a)> pub type vortex_array::arrays::chunked::ChunkedArray = vortex_array::Array @@ -1292,10 +1318,6 @@ pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays:: pub fn vortex_array::arrays::Constant::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Constant::array_eq(array: &vortex_array::arrays::constant::ConstantData, other: &vortex_array::arrays::constant::ConstantData, _precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Constant::array_hash(array: &vortex_array::arrays::constant::ConstantData, state: &mut H, _precision: vortex_array::Precision) - pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -1304,7 +1326,7 @@ pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -1326,11 +1348,7 @@ pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Constant @@ -1382,6 +1400,14 @@ impl core::fmt::Debug for vortex_array::arrays::constant::ConstantData pub fn vortex_array::arrays::constant::ConstantData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::ArrayEq for vortex_array::arrays::constant::ConstantData + +pub fn vortex_array::arrays::constant::ConstantData::array_eq(&self, other: &Self, _precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayHash for vortex_array::arrays::constant::ConstantData + +pub fn vortex_array::arrays::constant::ConstantData::array_hash(&self, state: &mut H, _precision: vortex_array::Precision) + pub type vortex_array::arrays::constant::ConstantArray = vortex_array::Array pub mod vortex_array::arrays::datetime @@ -1474,10 +1500,6 @@ pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::D pub fn vortex_array::arrays::Decimal::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Decimal::array_eq(array: &vortex_array::arrays::decimal::DecimalData, other: &vortex_array::arrays::decimal::DecimalData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Decimal::array_hash(array: &vortex_array::arrays::decimal::DecimalData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -1486,7 +1508,7 @@ pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -1508,11 +1530,7 @@ pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Decimal @@ -1558,39 +1576,27 @@ pub fn vortex_array::arrays::decimal::DecimalData::buffer_handle(&self) -> &vort pub fn vortex_array::arrays::decimal::DecimalData::decimal_dtype(&self) -> vortex_array::dtype::DecimalDType -pub fn vortex_array::arrays::decimal::DecimalData::dtype(&self) -> vortex_array::dtype::DType - -pub fn vortex_array::arrays::decimal::DecimalData::from_iter>(iter: I, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self - -pub fn vortex_array::arrays::decimal::DecimalData::from_option_iter>>(iter: I, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self - -pub fn vortex_array::arrays::decimal::DecimalData::into_parts(self) -> vortex_array::arrays::decimal::DecimalDataParts - pub fn vortex_array::arrays::decimal::DecimalData::is_empty(&self) -> bool pub fn vortex_array::arrays::decimal::DecimalData::len(&self) -> usize -pub fn vortex_array::arrays::decimal::DecimalData::new(buffer: vortex_buffer::buffer::Buffer, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::arrays::decimal::DecimalData::new(buffer: vortex_buffer::buffer::Buffer, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self -pub fn vortex_array::arrays::decimal::DecimalData::new_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::arrays::decimal::DecimalData::new_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self -pub unsafe fn vortex_array::arrays::decimal::DecimalData::new_unchecked(buffer: vortex_buffer::buffer::Buffer, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> Self +pub unsafe fn vortex_array::arrays::decimal::DecimalData::new_unchecked(buffer: vortex_buffer::buffer::Buffer, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self -pub unsafe fn vortex_array::arrays::decimal::DecimalData::new_unchecked_from_byte_buffer(byte_buffer: vortex_buffer::ByteBuffer, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> Self +pub unsafe fn vortex_array::arrays::decimal::DecimalData::new_unchecked_from_byte_buffer(byte_buffer: vortex_buffer::ByteBuffer, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self -pub unsafe fn vortex_array::arrays::decimal::DecimalData::new_unchecked_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> Self - -pub fn vortex_array::arrays::decimal::DecimalData::patch(self, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub unsafe fn vortex_array::arrays::decimal::DecimalData::new_unchecked_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self pub fn vortex_array::arrays::decimal::DecimalData::precision(&self) -> u8 pub fn vortex_array::arrays::decimal::DecimalData::scale(&self) -> i8 -pub fn vortex_array::arrays::decimal::DecimalData::try_new(buffer: vortex_buffer::buffer::Buffer, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::decimal::DecimalData::try_new_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult +pub fn vortex_array::arrays::decimal::DecimalData::try_new(buffer: vortex_buffer::buffer::Buffer, decimal_dtype: vortex_array::dtype::DecimalDType) -> vortex_error::VortexResult -pub fn vortex_array::arrays::decimal::DecimalData::validity(&self) -> vortex_array::validity::Validity +pub fn vortex_array::arrays::decimal::DecimalData::try_new_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType) -> vortex_error::VortexResult pub fn vortex_array::arrays::decimal::DecimalData::values_type(&self) -> vortex_array::dtype::DecimalType @@ -1602,6 +1608,14 @@ impl core::fmt::Debug for vortex_array::arrays::decimal::DecimalData pub fn vortex_array::arrays::decimal::DecimalData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::ArrayEq for vortex_array::arrays::decimal::DecimalData + +pub fn vortex_array::arrays::decimal::DecimalData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayHash for vortex_array::arrays::decimal::DecimalData + +pub fn vortex_array::arrays::decimal::DecimalData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) + pub struct vortex_array::arrays::decimal::DecimalDataParts pub vortex_array::arrays::decimal::DecimalDataParts::decimal_dtype: vortex_array::dtype::DecimalDType @@ -1628,6 +1642,46 @@ pub type vortex_array::arrays::decimal::DecimalMaskedValidityRule::Parent = vort pub fn vortex_array::arrays::decimal::DecimalMaskedValidityRule::reduce_parent(&self, array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, parent: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, _child_idx: usize) -> vortex_error::VortexResult> +pub trait vortex_array::arrays::decimal::DecimalArrayExt: vortex_array::TypedArrayRef + +pub fn vortex_array::arrays::decimal::DecimalArrayExt::buffer(&self) -> vortex_buffer::buffer::Buffer + +pub fn vortex_array::arrays::decimal::DecimalArrayExt::buffer_handle(&self) -> &vortex_array::buffer::BufferHandle + +pub fn vortex_array::arrays::decimal::DecimalArrayExt::decimal_dtype(&self) -> vortex_array::dtype::DecimalDType + +pub fn vortex_array::arrays::decimal::DecimalArrayExt::nullability(&self) -> vortex_array::dtype::Nullability + +pub fn vortex_array::arrays::decimal::DecimalArrayExt::precision(&self) -> u8 + +pub fn vortex_array::arrays::decimal::DecimalArrayExt::scale(&self) -> i8 + +pub fn vortex_array::arrays::decimal::DecimalArrayExt::validity(&self) -> vortex_array::validity::Validity + +pub fn vortex_array::arrays::decimal::DecimalArrayExt::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> + +pub fn vortex_array::arrays::decimal::DecimalArrayExt::values_type(&self) -> vortex_array::dtype::DecimalType + +impl> vortex_array::arrays::decimal::DecimalArrayExt for T + +pub fn T::buffer(&self) -> vortex_buffer::buffer::Buffer + +pub fn T::buffer_handle(&self) -> &vortex_array::buffer::BufferHandle + +pub fn T::decimal_dtype(&self) -> vortex_array::dtype::DecimalDType + +pub fn T::nullability(&self) -> vortex_array::dtype::Nullability + +pub fn T::precision(&self) -> u8 + +pub fn T::scale(&self) -> i8 + +pub fn T::validity(&self) -> vortex_array::validity::Validity + +pub fn T::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> + +pub fn T::values_type(&self) -> vortex_array::dtype::DecimalType + pub fn vortex_array::arrays::decimal::narrowed_decimal(decimal_array: vortex_array::arrays::DecimalArray) -> vortex_array::arrays::DecimalArray pub type vortex_array::arrays::decimal::DecimalArray = vortex_array::Array @@ -1664,10 +1718,6 @@ pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::dict::Dict::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::dict::Dict::array_eq(array: &vortex_array::arrays::dict::DictData, other: &vortex_array::arrays::dict::DictData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::dict::Dict::array_hash(array: &vortex_array::arrays::dict::DictData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -1676,7 +1726,7 @@ pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -1698,11 +1748,7 @@ pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayVie pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::dict::Dict @@ -1770,10 +1816,6 @@ pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::dict::Dict::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::dict::Dict::array_eq(array: &vortex_array::arrays::dict::DictData, other: &vortex_array::arrays::dict::DictData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::dict::Dict::array_hash(array: &vortex_array::arrays::dict::DictData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -1782,7 +1824,7 @@ pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -1804,11 +1846,7 @@ pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayVie pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::dict::Dict @@ -1850,32 +1888,12 @@ pub struct vortex_array::arrays::dict::DictData impl vortex_array::arrays::dict::DictData -pub fn vortex_array::arrays::dict::DictData::codes(&self) -> &vortex_array::ArrayRef - -pub fn vortex_array::arrays::dict::DictData::dtype(&self) -> vortex_array::dtype::DType - -pub fn vortex_array::arrays::dict::DictData::has_all_values_referenced(&self) -> bool +pub fn vortex_array::arrays::dict::DictData::new(codes_dtype: &vortex_array::dtype::DType) -> Self -pub fn vortex_array::arrays::dict::DictData::into_parts(self) -> vortex_array::arrays::dict::DictDataParts - -pub fn vortex_array::arrays::dict::DictData::is_empty(&self) -> bool - -pub fn vortex_array::arrays::dict::DictData::len(&self) -> usize - -pub fn vortex_array::arrays::dict::DictData::new(codes: vortex_array::ArrayRef, values: vortex_array::ArrayRef) -> Self - -pub unsafe fn vortex_array::arrays::dict::DictData::new_unchecked(codes: vortex_array::ArrayRef, values: vortex_array::ArrayRef) -> Self +pub unsafe fn vortex_array::arrays::dict::DictData::new_unchecked() -> Self pub unsafe fn vortex_array::arrays::dict::DictData::set_all_values_referenced(self, all_values_referenced: bool) -> Self -pub fn vortex_array::arrays::dict::DictData::validate_all_values_referenced(&self) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::dict::DictData::values(&self) -> &vortex_array::ArrayRef - -impl vortex_array::arrays::dict::DictData - -pub fn vortex_array::arrays::dict::DictData::compute_referenced_values_mask(&self, referenced: bool) -> vortex_error::VortexResult - impl core::clone::Clone for vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::clone(&self) -> vortex_array::arrays::dict::DictData @@ -1884,11 +1902,13 @@ impl core::fmt::Debug for vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct vortex_array::arrays::dict::DictDataParts +impl vortex_array::ArrayEq for vortex_array::arrays::dict::DictData -pub vortex_array::arrays::dict::DictDataParts::codes: vortex_array::ArrayRef +pub fn vortex_array::arrays::dict::DictData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -pub vortex_array::arrays::dict::DictDataParts::values: vortex_array::ArrayRef +impl vortex_array::ArrayHash for vortex_array::arrays::dict::DictData + +pub fn vortex_array::arrays::dict::DictData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) pub struct vortex_array::arrays::dict::DictMetadata @@ -1952,6 +1972,30 @@ pub type vortex_array::arrays::dict::TakeReduceAdaptor::Parent = vortex_array pub fn vortex_array::arrays::dict::TakeReduceAdaptor::reduce_parent(&self, array: vortex_array::ArrayView<'_, V>, parent: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, child_idx: usize) -> vortex_error::VortexResult> +pub trait vortex_array::arrays::dict::DictArrayExt: vortex_array::TypedArrayRef + +pub fn vortex_array::arrays::dict::DictArrayExt::codes(&self) -> &vortex_array::ArrayRef + +pub fn vortex_array::arrays::dict::DictArrayExt::compute_referenced_values_mask(&self, referenced: bool) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::dict::DictArrayExt::has_all_values_referenced(&self) -> bool + +pub fn vortex_array::arrays::dict::DictArrayExt::validate_all_values_referenced(&self) -> vortex_error::VortexResult<()> + +pub fn vortex_array::arrays::dict::DictArrayExt::values(&self) -> &vortex_array::ArrayRef + +impl> vortex_array::arrays::dict::DictArrayExt for T + +pub fn T::codes(&self) -> &vortex_array::ArrayRef + +pub fn T::compute_referenced_values_mask(&self, referenced: bool) -> vortex_error::VortexResult + +pub fn T::has_all_values_referenced(&self) -> bool + +pub fn T::validate_all_values_referenced(&self) -> vortex_error::VortexResult<()> + +pub fn T::values(&self) -> &vortex_array::ArrayRef + pub trait vortex_array::arrays::dict::TakeExecute: vortex_array::VTable pub fn vortex_array::arrays::dict::TakeExecute::take(array: vortex_array::ArrayView<'_, Self>, indices: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2058,10 +2102,6 @@ pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::Validit pub fn vortex_array::arrays::Extension::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Extension::array_eq(array: &vortex_array::arrays::extension::ExtensionData, other: &vortex_array::arrays::extension::ExtensionData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Extension::array_hash(array: &vortex_array::arrays::extension::ExtensionData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -2070,7 +2110,7 @@ pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2092,15 +2132,11 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityChild for vortex_array::arrays::Extension -pub fn vortex_array::arrays::Extension::validity_child(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::Extension::validity_child(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>) -> vortex_array::ArrayRef impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::Extension @@ -2130,21 +2166,13 @@ pub struct vortex_array::arrays::extension::ExtensionData impl vortex_array::arrays::extension::ExtensionData -pub fn vortex_array::arrays::extension::ExtensionData::dtype(&self) -> vortex_array::dtype::DType - pub fn vortex_array::arrays::extension::ExtensionData::ext_dtype(&self) -> &vortex_array::dtype::extension::ExtDTypeRef -pub fn vortex_array::arrays::extension::ExtensionData::is_empty(&self) -> bool +pub fn vortex_array::arrays::extension::ExtensionData::new(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_dtype: &vortex_array::dtype::DType) -> Self -pub fn vortex_array::arrays::extension::ExtensionData::len(&self) -> usize +pub unsafe fn vortex_array::arrays::extension::ExtensionData::new_unchecked(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_dtype: &vortex_array::dtype::DType) -> Self -pub fn vortex_array::arrays::extension::ExtensionData::new(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_array: vortex_array::ArrayRef) -> Self - -pub unsafe fn vortex_array::arrays::extension::ExtensionData::new_unchecked(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_array: vortex_array::ArrayRef) -> Self - -pub fn vortex_array::arrays::extension::ExtensionData::storage_array(&self) -> &vortex_array::ArrayRef - -pub fn vortex_array::arrays::extension::ExtensionData::try_new(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_array: vortex_array::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_array::arrays::extension::ExtensionData::try_new(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult impl core::clone::Clone for vortex_array::arrays::extension::ExtensionData @@ -2154,6 +2182,22 @@ impl core::fmt::Debug for vortex_array::arrays::extension::ExtensionData pub fn vortex_array::arrays::extension::ExtensionData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::ArrayEq for vortex_array::arrays::extension::ExtensionData + +pub fn vortex_array::arrays::extension::ExtensionData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayHash for vortex_array::arrays::extension::ExtensionData + +pub fn vortex_array::arrays::extension::ExtensionData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) + +pub trait vortex_array::arrays::extension::ExtensionArrayExt: vortex_array::TypedArrayRef + +pub fn vortex_array::arrays::extension::ExtensionArrayExt::storage_array(&self) -> &vortex_array::ArrayRef + +impl> vortex_array::arrays::extension::ExtensionArrayExt for T + +pub fn T::storage_array(&self) -> &vortex_array::ArrayRef + pub type vortex_array::arrays::extension::ExtensionArray = vortex_array::Array pub mod vortex_array::arrays::filter @@ -2186,10 +2230,6 @@ pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Fi pub fn vortex_array::arrays::Filter::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Filter::array_eq(array: &vortex_array::arrays::filter::FilterData, other: &vortex_array::arrays::filter::FilterData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Filter::array_hash(array: &vortex_array::arrays::filter::FilterData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -2198,7 +2238,7 @@ pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2220,11 +2260,7 @@ pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Filter @@ -2234,10 +2270,6 @@ pub struct vortex_array::arrays::filter::FilterData impl vortex_array::arrays::filter::FilterData -pub fn vortex_array::arrays::filter::FilterData::child(&self) -> &vortex_array::ArrayRef - -pub fn vortex_array::arrays::filter::FilterData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_array::arrays::filter::FilterData::filter_mask(&self) -> &vortex_mask::Mask pub fn vortex_array::arrays::filter::FilterData::into_parts(self) -> vortex_array::arrays::filter::FilterDataParts @@ -2246,7 +2278,7 @@ pub fn vortex_array::arrays::filter::FilterData::is_empty(&self) -> bool pub fn vortex_array::arrays::filter::FilterData::len(&self) -> usize -pub fn vortex_array::arrays::filter::FilterData::new(array: vortex_array::ArrayRef, mask: vortex_mask::Mask) -> Self +pub fn vortex_array::arrays::filter::FilterData::new(mask: vortex_mask::Mask) -> Self impl core::clone::Clone for vortex_array::arrays::filter::FilterData @@ -2256,9 +2288,15 @@ impl core::fmt::Debug for vortex_array::arrays::filter::FilterData pub fn vortex_array::arrays::filter::FilterData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct vortex_array::arrays::filter::FilterDataParts +impl vortex_array::ArrayEq for vortex_array::arrays::filter::FilterData + +pub fn vortex_array::arrays::filter::FilterData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool -pub vortex_array::arrays::filter::FilterDataParts::child: vortex_array::ArrayRef +impl vortex_array::ArrayHash for vortex_array::arrays::filter::FilterData + +pub fn vortex_array::arrays::filter::FilterData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) + +pub struct vortex_array::arrays::filter::FilterDataParts pub vortex_array::arrays::filter::FilterDataParts::mask: vortex_mask::Mask @@ -2294,6 +2332,14 @@ pub type vortex_array::arrays::filter::FilterReduceAdaptor::Parent = vortex_a pub fn vortex_array::arrays::filter::FilterReduceAdaptor::reduce_parent(&self, array: vortex_array::ArrayView<'_, V>, parent: vortex_array::ArrayView<'_, vortex_array::arrays::Filter>, child_idx: usize) -> vortex_error::VortexResult> +pub trait vortex_array::arrays::filter::FilterArrayExt: vortex_array::TypedArrayRef + +pub fn vortex_array::arrays::filter::FilterArrayExt::child(&self) -> &vortex_array::ArrayRef + +impl> vortex_array::arrays::filter::FilterArrayExt for T + +pub fn T::child(&self) -> &vortex_array::ArrayRef + pub trait vortex_array::arrays::filter::FilterKernel: vortex_array::VTable pub fn vortex_array::arrays::filter::FilterKernel::filter(array: vortex_array::ArrayView<'_, Self>, mask: &vortex_mask::Mask, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2374,10 +2420,6 @@ pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arr pub fn vortex_array::arrays::FixedSizeList::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::FixedSizeList::array_eq(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData, other: &vortex_array::arrays::fixed_size_list::FixedSizeListData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::FixedSizeList::array_hash(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -2386,7 +2428,7 @@ pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2408,11 +2450,7 @@ pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::Arra pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::FixedSizeList @@ -2438,39 +2476,63 @@ pub struct vortex_array::arrays::fixed_size_list::FixedSizeListData impl vortex_array::arrays::fixed_size_list::FixedSizeListData -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::build(elements: vortex_array::ArrayRef, list_size: u32, validity: vortex_array::validity::Validity, len: usize) -> Self -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::elements(&self) -> &vortex_array::ArrayRef +pub unsafe fn vortex_array::arrays::fixed_size_list::FixedSizeListData::new_unchecked(list_size: u32, len: usize) -> Self -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::into_parts(self) -> (vortex_array::ArrayRef, vortex_array::validity::Validity, vortex_array::dtype::DType) +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::validate(elements: &vortex_array::ArrayRef, len: usize, list_size: u32, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::is_empty(&self) -> bool +impl core::clone::Clone for vortex_array::arrays::fixed_size_list::FixedSizeListData -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::len(&self) -> usize +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::clone(&self) -> vortex_array::arrays::fixed_size_list::FixedSizeListData -pub const fn vortex_array::arrays::fixed_size_list::FixedSizeListData::list_size(&self) -> u32 +impl core::fmt::Debug for vortex_array::arrays::fixed_size_list::FixedSizeListData -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::new(elements: vortex_array::ArrayRef, list_size: u32, validity: vortex_array::validity::Validity, len: usize) -> Self +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub unsafe fn vortex_array::arrays::fixed_size_list::FixedSizeListData::new_unchecked(elements: vortex_array::ArrayRef, list_size: u32, validity: vortex_array::validity::Validity, len: usize) -> Self +impl vortex_array::ArrayEq for vortex_array::arrays::fixed_size_list::FixedSizeListData -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::validate(elements: &vortex_array::ArrayRef, len: usize, list_size: u32, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::validity(&self) -> vortex_array::validity::Validity +impl vortex_array::ArrayHash for vortex_array::arrays::fixed_size_list::FixedSizeListData -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::validity_mask(&self) -> vortex_mask::Mask +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) -impl vortex_array::arrays::fixed_size_list::FixedSizeListData +pub struct vortex_array::arrays::fixed_size_list::FixedSizeListDataParts -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::fixed_size_list_elements_at(&self, index: usize) -> vortex_error::VortexResult +pub vortex_array::arrays::fixed_size_list::FixedSizeListDataParts::dtype: vortex_array::dtype::DType -impl core::clone::Clone for vortex_array::arrays::fixed_size_list::FixedSizeListData +pub vortex_array::arrays::fixed_size_list::FixedSizeListDataParts::elements: vortex_array::ArrayRef -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::clone(&self) -> vortex_array::arrays::fixed_size_list::FixedSizeListData +pub vortex_array::arrays::fixed_size_list::FixedSizeListDataParts::validity: vortex_array::validity::Validity -impl core::fmt::Debug for vortex_array::arrays::fixed_size_list::FixedSizeListData +pub trait vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt: vortex_array::TypedArrayRef -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::dtype_parts(&self) -> (&vortex_array::dtype::DType, u32, vortex_array::dtype::Nullability) + +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::elements(&self) -> &vortex_array::ArrayRef + +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::fixed_size_list_elements_at(&self, index: usize) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::fixed_size_list_validity(&self) -> vortex_array::validity::Validity + +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::fixed_size_list_validity_mask(&self) -> vortex_mask::Mask + +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::list_size(&self) -> u32 + +impl> vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt for T + +pub fn T::dtype_parts(&self) -> (&vortex_array::dtype::DType, u32, vortex_array::dtype::Nullability) + +pub fn T::elements(&self) -> &vortex_array::ArrayRef + +pub fn T::fixed_size_list_elements_at(&self, index: usize) -> vortex_error::VortexResult + +pub fn T::fixed_size_list_validity(&self) -> vortex_array::validity::Validity + +pub fn T::fixed_size_list_validity_mask(&self) -> vortex_mask::Mask + +pub fn T::list_size(&self) -> u32 pub type vortex_array::arrays::fixed_size_list::FixedSizeListArray = vortex_array::Array @@ -2504,10 +2566,6 @@ pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List pub fn vortex_array::arrays::List::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::List::array_eq(array: &vortex_array::arrays::list::ListData, other: &vortex_array::arrays::list::ListData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::List::array_hash(array: &vortex_array::arrays::list::ListData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -2516,7 +2574,7 @@ pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2538,11 +2596,7 @@ pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::validate(&self, _data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::List @@ -2572,57 +2626,85 @@ pub struct vortex_array::arrays::list::ListData impl vortex_array::arrays::list::ListData -pub fn vortex_array::arrays::list::ListData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::list::ListData::build(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> Self -pub fn vortex_array::arrays::list::ListData::element_dtype(&self) -> &vortex_array::dtype::DType +pub unsafe fn vortex_array::arrays::list::ListData::new_unchecked() -> Self -pub fn vortex_array::arrays::list::ListData::elements(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::list::ListData::validate(elements: &vortex_array::ArrayRef, offsets: &vortex_array::ArrayRef, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::list::ListData::into_parts(self) -> vortex_array::arrays::list::ListDataParts +impl core::clone::Clone for vortex_array::arrays::list::ListData -pub fn vortex_array::arrays::list::ListData::is_empty(&self) -> bool +pub fn vortex_array::arrays::list::ListData::clone(&self) -> vortex_array::arrays::list::ListData -pub fn vortex_array::arrays::list::ListData::len(&self) -> usize +impl core::default::Default for vortex_array::arrays::list::ListData -pub fn vortex_array::arrays::list::ListData::list_elements_at(&self, index: usize) -> vortex_error::VortexResult +pub fn vortex_array::arrays::list::ListData::default() -> vortex_array::arrays::list::ListData -pub fn vortex_array::arrays::list::ListData::new(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> Self +impl core::fmt::Debug for vortex_array::arrays::list::ListData -pub unsafe fn vortex_array::arrays::list::ListData::new_unchecked(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::arrays::list::ListData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::list::ListData::offset_at(&self, index: usize) -> vortex_error::VortexResult +impl vortex_array::ArrayEq for vortex_array::arrays::list::ListData -pub fn vortex_array::arrays::list::ListData::offsets(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::list::ListData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -pub fn vortex_array::arrays::list::ListData::sliced_elements(&self) -> vortex_error::VortexResult +impl vortex_array::ArrayHash for vortex_array::arrays::list::ListData -pub fn vortex_array::arrays::list::ListData::validate(elements: &vortex_array::ArrayRef, offsets: &vortex_array::ArrayRef, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::list::ListData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) -pub fn vortex_array::arrays::list::ListData::validity(&self) -> vortex_array::validity::Validity +pub struct vortex_array::arrays::list::ListDataParts -pub fn vortex_array::arrays::list::ListData::validity_mask(&self) -> vortex_mask::Mask +pub vortex_array::arrays::list::ListDataParts::dtype: vortex_array::dtype::DType -impl vortex_array::arrays::list::ListData +pub vortex_array::arrays::list::ListDataParts::elements: vortex_array::ArrayRef -pub fn vortex_array::arrays::list::ListData::reset_offsets(&self, recurse: bool) -> vortex_error::VortexResult +pub vortex_array::arrays::list::ListDataParts::offsets: vortex_array::ArrayRef -impl core::clone::Clone for vortex_array::arrays::list::ListData +pub vortex_array::arrays::list::ListDataParts::validity: vortex_array::validity::Validity -pub fn vortex_array::arrays::list::ListData::clone(&self) -> vortex_array::arrays::list::ListData +pub trait vortex_array::arrays::list::ListArrayExt: vortex_array::TypedArrayRef -impl core::fmt::Debug for vortex_array::arrays::list::ListData +pub fn vortex_array::arrays::list::ListArrayExt::element_dtype(&self) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::list::ListData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::arrays::list::ListArrayExt::elements(&self) -> &vortex_array::ArrayRef -pub struct vortex_array::arrays::list::ListDataParts +pub fn vortex_array::arrays::list::ListArrayExt::list_elements_at(&self, index: usize) -> vortex_error::VortexResult -pub vortex_array::arrays::list::ListDataParts::dtype: vortex_array::dtype::DType +pub fn vortex_array::arrays::list::ListArrayExt::list_validity(&self) -> vortex_array::validity::Validity -pub vortex_array::arrays::list::ListDataParts::elements: vortex_array::ArrayRef +pub fn vortex_array::arrays::list::ListArrayExt::list_validity_mask(&self) -> vortex_mask::Mask -pub vortex_array::arrays::list::ListDataParts::offsets: vortex_array::ArrayRef +pub fn vortex_array::arrays::list::ListArrayExt::nullability(&self) -> vortex_array::dtype::Nullability -pub vortex_array::arrays::list::ListDataParts::validity: vortex_array::validity::Validity +pub fn vortex_array::arrays::list::ListArrayExt::offset_at(&self, index: usize) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::list::ListArrayExt::offsets(&self) -> &vortex_array::ArrayRef + +pub fn vortex_array::arrays::list::ListArrayExt::reset_offsets(&self, recurse: bool) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::list::ListArrayExt::sliced_elements(&self) -> vortex_error::VortexResult + +impl> vortex_array::arrays::list::ListArrayExt for T + +pub fn T::element_dtype(&self) -> &vortex_array::dtype::DType + +pub fn T::elements(&self) -> &vortex_array::ArrayRef + +pub fn T::list_elements_at(&self, index: usize) -> vortex_error::VortexResult + +pub fn T::list_validity(&self) -> vortex_array::validity::Validity + +pub fn T::list_validity_mask(&self) -> vortex_mask::Mask + +pub fn T::nullability(&self) -> vortex_array::dtype::Nullability + +pub fn T::offset_at(&self, index: usize) -> vortex_error::VortexResult + +pub fn T::offsets(&self) -> &vortex_array::ArrayRef + +pub fn T::reset_offsets(&self, recurse: bool) -> vortex_error::VortexResult> + +pub fn T::sliced_elements(&self) -> vortex_error::VortexResult pub type vortex_array::arrays::list::ListArray = vortex_array::Array @@ -2666,10 +2748,6 @@ pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays:: pub fn vortex_array::arrays::ListView::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::ListView::array_eq(array: &vortex_array::arrays::listview::ListViewData, other: &vortex_array::arrays::listview::ListViewData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::ListView::array_hash(array: &vortex_array::arrays::listview::ListViewData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -2678,7 +2756,7 @@ pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2700,11 +2778,7 @@ pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView< pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::validate(&self, _data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::ListView @@ -2730,52 +2804,38 @@ pub struct vortex_array::arrays::listview::ListViewData impl vortex_array::arrays::listview::ListViewData -pub fn vortex_array::arrays::listview::ListViewData::dtype(&self) -> vortex_array::dtype::DType - -pub fn vortex_array::arrays::listview::ListViewData::elements(&self) -> &vortex_array::ArrayRef - -pub fn vortex_array::arrays::listview::ListViewData::into_parts(self) -> vortex_array::arrays::listview::ListViewDataParts - -pub fn vortex_array::arrays::listview::ListViewData::is_empty(&self) -> bool - pub fn vortex_array::arrays::listview::ListViewData::is_zero_copy_to_list(&self) -> bool -pub fn vortex_array::arrays::listview::ListViewData::len(&self) -> usize - -pub fn vortex_array::arrays::listview::ListViewData::list_elements_at(&self, index: usize) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::listview::ListViewData::new(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, sizes: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> Self - -pub unsafe fn vortex_array::arrays::listview::ListViewData::new_unchecked(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, sizes: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> Self - -pub fn vortex_array::arrays::listview::ListViewData::offset_at(&self, index: usize) -> usize - -pub fn vortex_array::arrays::listview::ListViewData::offsets(&self) -> &vortex_array::ArrayRef - -pub fn vortex_array::arrays::listview::ListViewData::size_at(&self, index: usize) -> usize +pub fn vortex_array::arrays::listview::ListViewData::new() -> Self -pub fn vortex_array::arrays::listview::ListViewData::sizes(&self) -> &vortex_array::ArrayRef +pub unsafe fn vortex_array::arrays::listview::ListViewData::new_unchecked() -> Self -pub fn vortex_array::arrays::listview::ListViewData::try_new(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, sizes: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult +pub fn vortex_array::arrays::listview::ListViewData::try_new() -> vortex_error::VortexResult pub fn vortex_array::arrays::listview::ListViewData::validate(elements: &vortex_array::ArrayRef, offsets: &vortex_array::ArrayRef, sizes: &vortex_array::ArrayRef, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::listview::ListViewData::validity(&self) -> vortex_array::validity::Validity - -pub fn vortex_array::arrays::listview::ListViewData::validity_mask(&self) -> vortex_mask::Mask - -pub fn vortex_array::arrays::listview::ListViewData::verify_is_zero_copy_to_list(&self) -> bool - pub unsafe fn vortex_array::arrays::listview::ListViewData::with_zero_copy_to_list(self, is_zctl: bool) -> Self impl core::clone::Clone for vortex_array::arrays::listview::ListViewData pub fn vortex_array::arrays::listview::ListViewData::clone(&self) -> vortex_array::arrays::listview::ListViewData +impl core::default::Default for vortex_array::arrays::listview::ListViewData + +pub fn vortex_array::arrays::listview::ListViewData::default() -> Self + impl core::fmt::Debug for vortex_array::arrays::listview::ListViewData pub fn vortex_array::arrays::listview::ListViewData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::ArrayEq for vortex_array::arrays::listview::ListViewData + +pub fn vortex_array::arrays::listview::ListViewData::array_eq(&self, other: &Self, _precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayHash for vortex_array::arrays::listview::ListViewData + +pub fn vortex_array::arrays::listview::ListViewData::array_hash(&self, state: &mut H, _precision: vortex_array::Precision) + pub struct vortex_array::arrays::listview::ListViewDataParts pub vortex_array::arrays::listview::ListViewDataParts::elements: vortex_array::ArrayRef @@ -2788,6 +2848,50 @@ pub vortex_array::arrays::listview::ListViewDataParts::sizes: vortex_array::Arra pub vortex_array::arrays::listview::ListViewDataParts::validity: vortex_array::validity::Validity +pub trait vortex_array::arrays::listview::ListViewArrayExt: vortex_array::TypedArrayRef + +pub fn vortex_array::arrays::listview::ListViewArrayExt::elements(&self) -> &vortex_array::ArrayRef + +pub fn vortex_array::arrays::listview::ListViewArrayExt::list_elements_at(&self, index: usize) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::listview::ListViewArrayExt::listview_validity(&self) -> vortex_array::validity::Validity + +pub fn vortex_array::arrays::listview::ListViewArrayExt::listview_validity_mask(&self) -> vortex_mask::Mask + +pub fn vortex_array::arrays::listview::ListViewArrayExt::nullability(&self) -> vortex_array::dtype::Nullability + +pub fn vortex_array::arrays::listview::ListViewArrayExt::offset_at(&self, index: usize) -> usize + +pub fn vortex_array::arrays::listview::ListViewArrayExt::offsets(&self) -> &vortex_array::ArrayRef + +pub fn vortex_array::arrays::listview::ListViewArrayExt::size_at(&self, index: usize) -> usize + +pub fn vortex_array::arrays::listview::ListViewArrayExt::sizes(&self) -> &vortex_array::ArrayRef + +pub fn vortex_array::arrays::listview::ListViewArrayExt::verify_is_zero_copy_to_list(&self) -> bool + +impl> vortex_array::arrays::listview::ListViewArrayExt for T + +pub fn T::elements(&self) -> &vortex_array::ArrayRef + +pub fn T::list_elements_at(&self, index: usize) -> vortex_error::VortexResult + +pub fn T::listview_validity(&self) -> vortex_array::validity::Validity + +pub fn T::listview_validity_mask(&self) -> vortex_mask::Mask + +pub fn T::nullability(&self) -> vortex_array::dtype::Nullability + +pub fn T::offset_at(&self, index: usize) -> usize + +pub fn T::offsets(&self) -> &vortex_array::ArrayRef + +pub fn T::size_at(&self, index: usize) -> usize + +pub fn T::sizes(&self) -> &vortex_array::ArrayRef + +pub fn T::verify_is_zero_copy_to_list(&self) -> bool + pub fn vortex_array::arrays::listview::list_from_list_view(list_view: vortex_array::arrays::ListViewArray) -> vortex_error::VortexResult pub fn vortex_array::arrays::listview::list_view_from_list(list: vortex_array::arrays::ListArray, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2826,10 +2930,6 @@ pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Ma pub fn vortex_array::arrays::Masked::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Masked::array_eq(array: &vortex_array::arrays::masked::MaskedData, other: &vortex_array::arrays::masked::MaskedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Masked::array_hash(array: &vortex_array::arrays::masked::MaskedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -2838,7 +2938,7 @@ pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2860,11 +2960,7 @@ pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::validate(&self, _data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Masked @@ -2888,27 +2984,41 @@ pub fn vortex_array::arrays::Masked::mask(array: vortex_array::ArrayView<'_, vor pub struct vortex_array::arrays::masked::MaskedData -impl vortex_array::arrays::masked::MaskedData +impl core::clone::Clone for vortex_array::arrays::masked::MaskedData -pub fn vortex_array::arrays::masked::MaskedData::child(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::masked::MaskedData::clone(&self) -> vortex_array::arrays::masked::MaskedData -pub fn vortex_array::arrays::masked::MaskedData::dtype(&self) -> vortex_array::dtype::DType +impl core::fmt::Debug for vortex_array::arrays::masked::MaskedData -pub fn vortex_array::arrays::masked::MaskedData::is_empty(&self) -> bool +pub fn vortex_array::arrays::masked::MaskedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::masked::MaskedData::len(&self) -> usize +impl vortex_array::ArrayEq for vortex_array::arrays::masked::MaskedData -pub fn vortex_array::arrays::masked::MaskedData::validity(&self) -> vortex_array::validity::Validity +pub fn vortex_array::arrays::masked::MaskedData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -pub fn vortex_array::arrays::masked::MaskedData::validity_mask(&self) -> vortex_mask::Mask +impl vortex_array::ArrayHash for vortex_array::arrays::masked::MaskedData -impl core::clone::Clone for vortex_array::arrays::masked::MaskedData +pub fn vortex_array::arrays::masked::MaskedData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) -pub fn vortex_array::arrays::masked::MaskedData::clone(&self) -> vortex_array::arrays::masked::MaskedData +pub trait vortex_array::arrays::masked::MaskedArrayExt: vortex_array::TypedArrayRef -impl core::fmt::Debug for vortex_array::arrays::masked::MaskedData +pub fn vortex_array::arrays::masked::MaskedArrayExt::child(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::masked::MaskedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::arrays::masked::MaskedArrayExt::masked_validity(&self) -> vortex_array::validity::Validity + +pub fn vortex_array::arrays::masked::MaskedArrayExt::masked_validity_mask(&self) -> vortex_mask::Mask + +pub fn vortex_array::arrays::masked::MaskedArrayExt::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> + +impl> vortex_array::arrays::masked::MaskedArrayExt for T + +pub fn T::child(&self) -> &vortex_array::ArrayRef + +pub fn T::masked_validity(&self) -> vortex_array::validity::Validity + +pub fn T::masked_validity_mask(&self) -> vortex_mask::Mask + +pub fn T::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> pub fn vortex_array::arrays::masked::mask_validity_canonical(canonical: vortex_array::Canonical, validity_mask: &vortex_mask::Mask, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2948,10 +3058,6 @@ pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::null::Null::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::null::Null::array_eq(_array: &vortex_array::arrays::null::NullData, _other: &vortex_array::arrays::null::NullData, _precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::null::Null::array_hash(_array: &vortex_array::arrays::null::NullData, _state: &mut H, _precision: vortex_array::Precision) - pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -2960,7 +3066,7 @@ pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2982,11 +3088,7 @@ pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::null::Null @@ -3030,12 +3132,24 @@ impl core::fmt::Debug for vortex_array::arrays::null::NullData pub fn vortex_array::arrays::null::NullData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::ArrayEq for vortex_array::arrays::null::NullData + +pub fn vortex_array::arrays::null::NullData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayHash for vortex_array::arrays::null::NullData + +pub fn vortex_array::arrays::null::NullData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) + pub type vortex_array::arrays::null::NullArray = vortex_array::Array pub mod vortex_array::arrays::patched pub struct vortex_array::arrays::patched::Patched +impl vortex_array::arrays::patched::Patched + +pub fn vortex_array::arrays::patched::Patched::from_array_and_patches(inner: vortex_array::ArrayRef, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl core::clone::Clone for vortex_array::arrays::patched::Patched pub fn vortex_array::arrays::patched::Patched::clone(&self) -> vortex_array::arrays::patched::Patched @@ -3050,7 +3164,7 @@ pub fn vortex_array::arrays::patched::Patched::scalar_at(array: vortex_array::Ar impl vortex_array::VTable for vortex_array::arrays::patched::Patched -pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray +pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedData pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched @@ -3058,10 +3172,6 @@ pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array:: pub fn vortex_array::arrays::patched::Patched::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::patched::Patched::array_eq(array: &vortex_array::arrays::patched::PatchedArray, other: &vortex_array::arrays::patched::PatchedArray, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::patched::Patched::array_hash(array: &vortex_array::arrays::patched::PatchedArray, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -3070,7 +3180,7 @@ pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayV pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3092,15 +3202,11 @@ pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::Ar pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityChild for vortex_array::arrays::patched::Patched -pub fn vortex_array::arrays::patched::Patched::validity_child(array: &vortex_array::arrays::patched::PatchedArray) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::Patched::validity_child(array: vortex_array::ArrayView<'_, vortex_array::arrays::patched::Patched>) -> vortex_array::ArrayRef impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::patched::Patched @@ -3118,65 +3224,81 @@ impl vortex_array::scalar_fn::fns::binary::CompareKernel for vortex_array::array pub fn vortex_array::arrays::patched::Patched::compare(lhs: vortex_array::ArrayView<'_, Self>, rhs: &vortex_array::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::patched::PatchedArray +pub struct vortex_array::arrays::patched::PatchedData -impl vortex_array::arrays::patched::PatchedArray +impl core::clone::Clone for vortex_array::arrays::patched::PatchedData -pub fn vortex_array::arrays::patched::PatchedArray::base_array(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedData::clone(&self) -> vortex_array::arrays::patched::PatchedData -pub fn vortex_array::arrays::patched::PatchedArray::is_empty(&self) -> bool +impl core::fmt::Debug for vortex_array::arrays::patched::PatchedData -pub fn vortex_array::arrays::patched::PatchedArray::lane_offsets(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::patched::PatchedArray::len(&self) -> usize +impl vortex_array::ArrayEq for vortex_array::arrays::patched::PatchedData -pub fn vortex_array::arrays::patched::PatchedArray::patch_indices(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedData::array_eq(&self, other: &Self, _precision: vortex_array::Precision) -> bool -pub fn vortex_array::arrays::patched::PatchedArray::patch_values(&self) -> &vortex_array::ArrayRef +impl vortex_array::ArrayHash for vortex_array::arrays::patched::PatchedData -impl vortex_array::arrays::patched::PatchedArray +pub fn vortex_array::arrays::patched::PatchedData::array_hash(&self, state: &mut H, _precision: vortex_array::Precision) -pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner: vortex_array::ArrayRef, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub struct vortex_array::arrays::patched::PatchedMetadata -impl vortex_array::arrays::patched::PatchedArray +impl core::clone::Clone for vortex_array::arrays::patched::PatchedMetadata -pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef where Self: core::clone::Clone + vortex_array::IntoArray +pub fn vortex_array::arrays::patched::PatchedMetadata::clone(&self) -> vortex_array::arrays::patched::PatchedMetadata -impl core::clone::Clone for vortex_array::arrays::patched::PatchedArray +impl core::default::Default for vortex_array::arrays::patched::PatchedMetadata -pub fn vortex_array::arrays::patched::PatchedArray::clone(&self) -> vortex_array::arrays::patched::PatchedArray +pub fn vortex_array::arrays::patched::PatchedMetadata::default() -> Self -impl core::convert::From for vortex_array::ArrayRef +impl core::fmt::Debug for vortex_array::arrays::patched::PatchedMetadata -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> Self +pub fn vortex_array::arrays::patched::PatchedMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl core::fmt::Debug for vortex_array::arrays::patched::PatchedArray +impl prost::message::Message for vortex_array::arrays::patched::PatchedMetadata -pub fn vortex_array::arrays::patched::PatchedArray::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::arrays::patched::PatchedMetadata::clear(&mut self) -impl vortex_array::IntoArray for vortex_array::arrays::patched::PatchedArray +pub fn vortex_array::arrays::patched::PatchedMetadata::encoded_len(&self) -> usize -pub fn vortex_array::arrays::patched::PatchedArray::into_array(self) -> vortex_array::ArrayRef +pub trait vortex_array::arrays::patched::PatchedArrayExt: vortex_array::TypedArrayRef -pub struct vortex_array::arrays::patched::PatchedMetadata +pub fn vortex_array::arrays::patched::PatchedArrayExt::base_array(&self) -> &vortex_array::ArrayRef -impl core::clone::Clone for vortex_array::arrays::patched::PatchedMetadata +pub fn vortex_array::arrays::patched::PatchedArrayExt::lane_offsets(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::patched::PatchedMetadata::clone(&self) -> vortex_array::arrays::patched::PatchedMetadata +pub fn vortex_array::arrays::patched::PatchedArrayExt::lane_range(&self, chunk: usize, lane: usize) -> vortex_error::VortexResult> -impl core::default::Default for vortex_array::arrays::patched::PatchedMetadata +pub fn vortex_array::arrays::patched::PatchedArrayExt::n_lanes(&self) -> usize -pub fn vortex_array::arrays::patched::PatchedMetadata::default() -> Self +pub fn vortex_array::arrays::patched::PatchedArrayExt::offset(&self) -> usize -impl core::fmt::Debug for vortex_array::arrays::patched::PatchedMetadata +pub fn vortex_array::arrays::patched::PatchedArrayExt::patch_indices(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::patched::PatchedMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::arrays::patched::PatchedArrayExt::patch_values(&self) -> &vortex_array::ArrayRef -impl prost::message::Message for vortex_array::arrays::patched::PatchedMetadata +pub fn vortex_array::arrays::patched::PatchedArrayExt::slice_chunks(&self, chunks: core::ops::range::Range) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::PatchedMetadata::clear(&mut self) +impl> vortex_array::arrays::patched::PatchedArrayExt for T -pub fn vortex_array::arrays::patched::PatchedMetadata::encoded_len(&self) -> usize +pub fn T::base_array(&self) -> &vortex_array::ArrayRef + +pub fn T::lane_offsets(&self) -> &vortex_array::ArrayRef + +pub fn T::lane_range(&self, chunk: usize, lane: usize) -> vortex_error::VortexResult> + +pub fn T::n_lanes(&self) -> usize + +pub fn T::offset(&self) -> usize + +pub fn T::patch_indices(&self) -> &vortex_array::ArrayRef + +pub fn T::patch_values(&self) -> &vortex_array::ArrayRef + +pub fn T::slice_chunks(&self, chunks: core::ops::range::Range) -> vortex_error::VortexResult> + +pub type vortex_array::arrays::patched::PatchedArray = vortex_array::Array pub mod vortex_array::arrays::primitive @@ -3274,10 +3396,6 @@ pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays: pub fn vortex_array::arrays::Primitive::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Primitive::array_eq(array: &vortex_array::arrays::primitive::PrimitiveData, other: &vortex_array::arrays::primitive::PrimitiveData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Primitive::array_hash(array: &vortex_array::arrays::primitive::PrimitiveData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -3286,7 +3404,7 @@ pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3308,11 +3426,7 @@ pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Primitive @@ -3354,15 +3468,11 @@ impl vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::as_slice(&self) -> &[T] -pub fn vortex_array::arrays::primitive::PrimitiveData::reinterpret_cast(&self, ptype: vortex_array::dtype::PType) -> Self - impl vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::buffer_handle(&self) -> &vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::primitive::PrimitiveData::dtype(&self) -> vortex_array::dtype::DType - -pub fn vortex_array::arrays::primitive::PrimitiveData::from_buffer_handle(handle: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::PType, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::arrays::primitive::PrimitiveData::from_buffer_handle(handle: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::PType, _validity: vortex_array::validity::Validity) -> Self pub fn vortex_array::arrays::primitive::PrimitiveData::from_byte_buffer(buffer: vortex_buffer::ByteBuffer, ptype: vortex_array::dtype::PType, validity: vortex_array::validity::Validity) -> Self @@ -3376,38 +3486,26 @@ pub fn vortex_array::arrays::primitive::PrimitiveData::is_empty(&self) -> bool pub fn vortex_array::arrays::primitive::PrimitiveData::len(&self) -> usize -pub fn vortex_array::arrays::primitive::PrimitiveData::map_each(self, f: F) -> Self where T: vortex_array::dtype::NativePType, R: vortex_array::dtype::NativePType, F: core::ops::function::FnMut(T) -> R - -pub fn vortex_array::arrays::primitive::PrimitiveData::map_each_with_validity(self, f: F) -> vortex_error::VortexResult where T: vortex_array::dtype::NativePType, R: vortex_array::dtype::NativePType, F: core::ops::function::FnMut((T, bool)) -> R - pub fn vortex_array::arrays::primitive::PrimitiveData::ptype(&self) -> vortex_array::dtype::PType pub fn vortex_array::arrays::primitive::PrimitiveData::to_buffer(&self) -> vortex_buffer::buffer::Buffer pub fn vortex_array::arrays::primitive::PrimitiveData::try_into_buffer_mut(self) -> core::result::Result, vortex_buffer::buffer::Buffer> -pub fn vortex_array::arrays::primitive::PrimitiveData::validity(&self) -> vortex_array::validity::Validity - -pub fn vortex_array::arrays::primitive::PrimitiveData::validity_mask(&self) -> vortex_mask::Mask - impl vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::empty(nullability: vortex_array::dtype::Nullability) -> Self pub fn vortex_array::arrays::primitive::PrimitiveData::new(buffer: impl core::convert::Into>, validity: vortex_array::validity::Validity) -> Self -pub unsafe fn vortex_array::arrays::primitive::PrimitiveData::new_unchecked(buffer: vortex_buffer::buffer::Buffer, validity: vortex_array::validity::Validity) -> Self +pub unsafe fn vortex_array::arrays::primitive::PrimitiveData::new_unchecked(buffer: vortex_buffer::buffer::Buffer, _validity: vortex_array::validity::Validity) -> Self -pub unsafe fn vortex_array::arrays::primitive::PrimitiveData::new_unchecked_from_handle(handle: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::PType, validity: vortex_array::validity::Validity) -> Self +pub unsafe fn vortex_array::arrays::primitive::PrimitiveData::new_unchecked_from_handle(handle: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::PType, _validity: vortex_array::validity::Validity) -> Self pub fn vortex_array::arrays::primitive::PrimitiveData::try_new(buffer: vortex_buffer::buffer::Buffer, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult pub fn vortex_array::arrays::primitive::PrimitiveData::validate(buffer: &vortex_buffer::buffer::Buffer, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> -impl vortex_array::arrays::primitive::PrimitiveData - -pub fn vortex_array::arrays::primitive::PrimitiveData::into_parts(self) -> vortex_array::arrays::primitive::PrimitiveDataParts - impl core::clone::Clone for vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::clone(&self) -> vortex_array::arrays::primitive::PrimitiveData @@ -3416,6 +3514,14 @@ impl core::fmt::Debug for vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::ArrayEq for vortex_array::arrays::primitive::PrimitiveData + +pub fn vortex_array::arrays::primitive::PrimitiveData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayHash for vortex_array::arrays::primitive::PrimitiveData + +pub fn vortex_array::arrays::primitive::PrimitiveData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) + pub struct vortex_array::arrays::primitive::PrimitiveDataParts pub vortex_array::arrays::primitive::PrimitiveDataParts::buffer: vortex_array::buffer::BufferHandle @@ -3440,6 +3546,42 @@ pub type vortex_array::arrays::primitive::PrimitiveMaskedValidityRule::Parent = pub fn vortex_array::arrays::primitive::PrimitiveMaskedValidityRule::reduce_parent(&self, array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, parent: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, _child_idx: usize) -> vortex_error::VortexResult> +pub trait vortex_array::arrays::primitive::PrimitiveArrayExt: vortex_array::TypedArrayRef + +pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::buffer_handle(&self) -> &vortex_array::buffer::BufferHandle + +pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::narrow(&self) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::nullability(&self) -> vortex_array::dtype::Nullability + +pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::ptype(&self) -> vortex_array::dtype::PType + +pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::reinterpret_cast(&self, ptype: vortex_array::dtype::PType) -> vortex_array::arrays::PrimitiveArray + +pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::validity(&self) -> vortex_array::validity::Validity + +pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> + +pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::validity_mask(&self) -> vortex_mask::Mask + +impl> vortex_array::arrays::primitive::PrimitiveArrayExt for T + +pub fn T::buffer_handle(&self) -> &vortex_array::buffer::BufferHandle + +pub fn T::narrow(&self) -> vortex_error::VortexResult + +pub fn T::nullability(&self) -> vortex_array::dtype::Nullability + +pub fn T::ptype(&self) -> vortex_array::dtype::PType + +pub fn T::reinterpret_cast(&self, ptype: vortex_array::dtype::PType) -> vortex_array::arrays::PrimitiveArray + +pub fn T::validity(&self) -> vortex_array::validity::Validity + +pub fn T::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> + +pub fn T::validity_mask(&self) -> vortex_mask::Mask + pub fn vortex_array::arrays::primitive::chunk_range(chunk_idx: usize, offset: usize, array_len: usize) -> core::ops::range::Range pub fn vortex_array::arrays::primitive::patch_chunk(decoded_values: &mut [T], patches_indices: &[I], patches_values: &[T], patches_offset: usize, chunk_offsets_slice: &[C], chunk_idx: usize, offset_within_chunk: usize) where T: vortex_array::dtype::NativePType, I: vortex_array::dtype::UnsignedPType, C: vortex_array::dtype::UnsignedPType @@ -3496,18 +3638,10 @@ pub struct vortex_array::arrays::scalar_fn::ScalarFnData impl vortex_array::arrays::scalar_fn::ScalarFnData -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::children(&self) -> alloc::vec::Vec - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::get_child(&self, idx: usize) -> &vortex_array::ArrayRef - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::iter_children(&self) -> impl core::iter::traits::iterator::Iterator + '_ - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::nchildren(&self) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFnData::build(scalar_fn: vortex_array::scalar_fn::ScalarFnRef, children: alloc::vec::Vec, len: usize) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnData::scalar_fn(&self) -> &vortex_array::scalar_fn::ScalarFnRef -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::try_new(scalar_fn: vortex_array::scalar_fn::ScalarFnRef, children: alloc::vec::Vec, len: usize) -> vortex_error::VortexResult - impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFnData pub fn vortex_array::arrays::scalar_fn::ScalarFnData::clone(&self) -> vortex_array::arrays::scalar_fn::ScalarFnData @@ -3516,6 +3650,14 @@ impl core::fmt::Debug for vortex_array::arrays::scalar_fn::ScalarFnData pub fn vortex_array::arrays::scalar_fn::ScalarFnData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::ArrayEq for vortex_array::arrays::scalar_fn::ScalarFnData + +pub fn vortex_array::arrays::scalar_fn::ScalarFnData::array_eq(&self, other: &Self, _precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayHash for vortex_array::arrays::scalar_fn::ScalarFnData + +pub fn vortex_array::arrays::scalar_fn::ScalarFnData::array_hash(&self, state: &mut H, _precision: vortex_array::Precision) + pub struct vortex_array::arrays::scalar_fn::ScalarFnVTable impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -3540,10 +3682,6 @@ pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vorte pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_eq(array: &vortex_array::arrays::scalar_fn::ScalarFnData, other: &vortex_array::arrays::scalar_fn::ScalarFnData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_hash(array: &vortex_array::arrays::scalar_fn::ScalarFnData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -3552,7 +3690,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_arra pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3574,21 +3712,57 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>) -> vortex_error::VortexResult -pub trait vortex_array::arrays::scalar_fn::ScalarFnArrayExt: vortex_array::scalar_fn::ScalarFnVTable +pub trait vortex_array::arrays::scalar_fn::ScalarFnArrayExt: vortex_array::TypedArrayRef + +pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayExt::child_at(&self, idx: usize) -> &vortex_array::ArrayRef + +pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayExt::child_count(&self) -> usize + +pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayExt::children(&self) -> alloc::vec::Vec + +pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayExt::get_child(&self, idx: usize) -> &vortex_array::ArrayRef + +pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayExt::iter_children(&self) -> impl core::iter::traits::iterator::Iterator + '_ + +pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayExt::nchildren(&self) -> usize + +pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayExt::scalar_fn(&self) -> &vortex_array::scalar_fn::ScalarFnRef + +impl> vortex_array::arrays::scalar_fn::ScalarFnArrayExt for T + +pub fn T::child_at(&self, idx: usize) -> &vortex_array::ArrayRef + +pub fn T::child_count(&self) -> usize + +pub fn T::children(&self) -> alloc::vec::Vec + +pub fn T::get_child(&self, idx: usize) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayExt::try_new_array(&self, len: usize, options: Self::Options, children: impl core::convert::Into>) -> vortex_error::VortexResult +pub fn T::iter_children(&self) -> impl core::iter::traits::iterator::Iterator + '_ -impl vortex_array::arrays::scalar_fn::ScalarFnArrayExt for V +pub fn T::nchildren(&self) -> usize + +pub fn T::scalar_fn(&self) -> &vortex_array::scalar_fn::ScalarFnRef + +pub trait vortex_array::arrays::scalar_fn::ScalarFnFactoryExt: vortex_array::scalar_fn::ScalarFnVTable + +pub trait vortex_array::arrays::scalar_fn::ScalarFnFactoryExt: vortex_array::scalar_fn::ScalarFnVTable + +pub fn vortex_array::arrays::scalar_fn::ScalarFnFactoryExt::try_new_array(&self, len: usize, options: Self::Options, children: impl core::convert::Into>) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::scalar_fn::ScalarFnFactoryExt::try_new_array(&self, len: usize, options: Self::Options, children: impl core::convert::Into>) -> vortex_error::VortexResult + +impl vortex_array::arrays::scalar_fn::ScalarFnFactoryExt for V + +impl vortex_array::arrays::scalar_fn::ScalarFnFactoryExt for V + +pub fn V::try_new_array(&self, len: usize, options: Self::Options, children: impl core::convert::Into>) -> vortex_error::VortexResult pub fn V::try_new_array(&self, len: usize, options: Self::Options, children: impl core::convert::Into>) -> vortex_error::VortexResult @@ -3624,10 +3798,6 @@ pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Sh pub fn vortex_array::arrays::Shared::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Shared::array_eq(array: &vortex_array::arrays::shared::SharedData, other: &vortex_array::arrays::shared::SharedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Shared::array_hash(array: &vortex_array::arrays::shared::SharedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -3636,7 +3806,7 @@ pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3658,39 +3828,57 @@ pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] +pub fn vortex_array::arrays::Shared::validate(&self, _data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> + +impl vortex_array::ValidityVTable for vortex_array::arrays::Shared + +pub fn vortex_array::arrays::Shared::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::Shared>) -> vortex_error::VortexResult + +pub struct vortex_array::arrays::shared::SharedData + +impl vortex_array::arrays::shared::SharedData + +pub fn vortex_array::arrays::shared::SharedData::new() -> Self + +impl core::clone::Clone for vortex_array::arrays::shared::SharedData + +pub fn vortex_array::arrays::shared::SharedData::clone(&self) -> vortex_array::arrays::shared::SharedData + +impl core::default::Default for vortex_array::arrays::shared::SharedData + +pub fn vortex_array::arrays::shared::SharedData::default() -> Self -pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +impl core::fmt::Debug for vortex_array::arrays::shared::SharedData -pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::shared::SharedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::ValidityVTable for vortex_array::arrays::Shared +impl vortex_array::ArrayEq for vortex_array::arrays::shared::SharedData -pub fn vortex_array::arrays::Shared::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::Shared>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::shared::SharedData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -pub struct vortex_array::arrays::shared::SharedData +impl vortex_array::ArrayHash for vortex_array::arrays::shared::SharedData -impl vortex_array::arrays::shared::SharedData +pub fn vortex_array::arrays::shared::SharedData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) -pub fn vortex_array::arrays::shared::SharedData::dtype(&self) -> &vortex_array::dtype::DType +pub trait vortex_array::arrays::shared::SharedArrayExt: vortex_array::TypedArrayRef -pub fn vortex_array::arrays::shared::SharedData::get_or_compute(&self, f: impl core::ops::function::FnOnce(&vortex_array::ArrayRef) -> vortex_error::VortexResult) -> vortex_error::VortexResult +pub fn vortex_array::arrays::shared::SharedArrayExt::current_array_ref(&self) -> &vortex_array::ArrayRef -pub async fn vortex_array::arrays::shared::SharedData::get_or_compute_async(&self, f: F) -> vortex_error::VortexResult where F: core::ops::function::FnOnce(vortex_array::ArrayRef) -> Fut, Fut: core::future::future::Future> +pub fn vortex_array::arrays::shared::SharedArrayExt::get_or_compute(&self, f: impl core::ops::function::FnOnce(&vortex_array::ArrayRef) -> vortex_error::VortexResult) -> vortex_error::VortexResult -pub fn vortex_array::arrays::shared::SharedData::is_empty(&self) -> bool +pub async fn vortex_array::arrays::shared::SharedArrayExt::get_or_compute_async(&self, f: F) -> vortex_error::VortexResult where F: core::ops::function::FnOnce(vortex_array::ArrayRef) -> Fut, Fut: core::future::future::Future> -pub fn vortex_array::arrays::shared::SharedData::len(&self) -> usize +pub fn vortex_array::arrays::shared::SharedArrayExt::source(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::shared::SharedData::new(source: vortex_array::ArrayRef) -> Self +impl> vortex_array::arrays::shared::SharedArrayExt for T -impl core::clone::Clone for vortex_array::arrays::shared::SharedData +pub fn T::current_array_ref(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::shared::SharedData::clone(&self) -> vortex_array::arrays::shared::SharedData +pub fn T::get_or_compute(&self, f: impl core::ops::function::FnOnce(&vortex_array::ArrayRef) -> vortex_error::VortexResult) -> vortex_error::VortexResult -impl core::fmt::Debug for vortex_array::arrays::shared::SharedData +pub async fn T::get_or_compute_async(&self, f: F) -> vortex_error::VortexResult where F: core::ops::function::FnOnce(vortex_array::ArrayRef) -> Fut, Fut: core::future::future::Future> -pub fn vortex_array::arrays::shared::SharedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn T::source(&self) -> &vortex_array::ArrayRef pub type vortex_array::arrays::shared::SharedArray = vortex_array::Array @@ -3724,10 +3912,6 @@ pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arra pub fn vortex_array::arrays::slice::Slice::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::slice::Slice::array_eq(array: &vortex_array::arrays::slice::SliceData, other: &vortex_array::arrays::slice::SliceData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::slice::Slice::array_hash(array: &vortex_array::arrays::slice::SliceData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -3736,7 +3920,7 @@ pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView< pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3758,11 +3942,7 @@ pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::Array pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::slice::Slice @@ -3776,17 +3956,13 @@ pub struct vortex_array::arrays::slice::SliceData impl vortex_array::arrays::slice::SliceData -pub fn vortex_array::arrays::slice::SliceData::child(&self) -> &vortex_array::ArrayRef - -pub fn vortex_array::arrays::slice::SliceData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_array::arrays::slice::SliceData::into_parts(self) -> vortex_array::arrays::slice::SliceDataParts pub fn vortex_array::arrays::slice::SliceData::is_empty(&self) -> bool pub fn vortex_array::arrays::slice::SliceData::len(&self) -> usize -pub fn vortex_array::arrays::slice::SliceData::new(child: vortex_array::ArrayRef, range: core::ops::range::Range) -> Self +pub fn vortex_array::arrays::slice::SliceData::new(range: core::ops::range::Range) -> Self pub fn vortex_array::arrays::slice::SliceData::slice_range(&self) -> &core::ops::range::Range @@ -3798,9 +3974,15 @@ impl core::fmt::Debug for vortex_array::arrays::slice::SliceData pub fn vortex_array::arrays::slice::SliceData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct vortex_array::arrays::slice::SliceDataParts +impl vortex_array::ArrayEq for vortex_array::arrays::slice::SliceData + +pub fn vortex_array::arrays::slice::SliceData::array_eq(&self, other: &Self, _precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayHash for vortex_array::arrays::slice::SliceData + +pub fn vortex_array::arrays::slice::SliceData::array_hash(&self, state: &mut H, _precision: vortex_array::Precision) -pub vortex_array::arrays::slice::SliceDataParts::child: vortex_array::ArrayRef +pub struct vortex_array::arrays::slice::SliceDataParts pub vortex_array::arrays::slice::SliceDataParts::range: core::ops::range::Range @@ -3842,6 +4024,14 @@ pub type vortex_array::arrays::slice::SliceReduceAdaptor::Parent = vortex_arr pub fn vortex_array::arrays::slice::SliceReduceAdaptor::reduce_parent(&self, array: vortex_array::ArrayView<'_, V>, parent: ::Match, child_idx: usize) -> vortex_error::VortexResult> +pub trait vortex_array::arrays::slice::SliceArrayExt: vortex_array::TypedArrayRef + +pub fn vortex_array::arrays::slice::SliceArrayExt::child(&self) -> &vortex_array::ArrayRef + +impl> vortex_array::arrays::slice::SliceArrayExt for T + +pub fn T::child(&self) -> &vortex_array::ArrayRef + pub trait vortex_array::arrays::slice::SliceKernel: vortex_array::VTable pub fn vortex_array::arrays::slice::SliceKernel::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -3950,10 +4140,6 @@ pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::St pub fn vortex_array::arrays::Struct::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Struct::array_eq(array: &vortex_array::arrays::struct_::StructData, other: &vortex_array::arrays::struct_::StructData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Struct::array_hash(array: &vortex_array::arrays::struct_::StructData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -3962,7 +4148,7 @@ pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3984,11 +4170,7 @@ pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Struct @@ -4018,67 +4200,79 @@ pub struct vortex_array::arrays::struct_::StructData impl vortex_array::arrays::struct_::StructData -pub fn vortex_array::arrays::struct_::StructData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::struct_::StructData::build(names: vortex_array::dtype::FieldNames, fields: impl core::convert::Into>, length: usize, validity: vortex_array::validity::Validity) -> Self + +pub fn vortex_array::arrays::struct_::StructData::names(&self) -> &vortex_array::dtype::FieldNames + +pub fn vortex_array::arrays::struct_::StructData::new_fieldless_with_len(len: usize) -> Self -pub fn vortex_array::arrays::struct_::StructData::from_fields>(items: &[(N, vortex_array::ArrayRef)]) -> vortex_error::VortexResult +pub unsafe fn vortex_array::arrays::struct_::StructData::new_unchecked(dtype: vortex_array::dtype::StructFields, fieldless_len: core::option::Option) -> Self -pub fn vortex_array::arrays::struct_::StructData::into_fields(self) -> alloc::vec::Vec +pub fn vortex_array::arrays::struct_::StructData::validate(fields: &[vortex_array::ArrayRef], dtype: &vortex_array::dtype::StructFields, length: usize, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::struct_::StructData::into_parts(self) -> vortex_array::arrays::struct_::StructDataParts +impl core::clone::Clone for vortex_array::arrays::struct_::StructData -pub fn vortex_array::arrays::struct_::StructData::is_empty(&self) -> bool +pub fn vortex_array::arrays::struct_::StructData::clone(&self) -> vortex_array::arrays::struct_::StructData -pub fn vortex_array::arrays::struct_::StructData::iter_unmasked_fields(&self) -> impl core::iter::traits::iterator::Iterator + '_ +impl core::fmt::Debug for vortex_array::arrays::struct_::StructData -pub fn vortex_array::arrays::struct_::StructData::len(&self) -> usize +pub fn vortex_array::arrays::struct_::StructData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::struct_::StructData::names(&self) -> &vortex_array::dtype::FieldNames +impl vortex_array::ArrayEq for vortex_array::arrays::struct_::StructData -pub fn vortex_array::arrays::struct_::StructData::new(names: vortex_array::dtype::FieldNames, fields: impl core::convert::Into>, length: usize, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::arrays::struct_::StructData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -pub fn vortex_array::arrays::struct_::StructData::new_fieldless_with_len(len: usize) -> Self +impl vortex_array::ArrayHash for vortex_array::arrays::struct_::StructData -pub unsafe fn vortex_array::arrays::struct_::StructData::new_unchecked(fields: impl core::convert::Into>, dtype: vortex_array::dtype::StructFields, length: usize, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::arrays::struct_::StructData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) -pub fn vortex_array::arrays::struct_::StructData::remove_column(&mut self, name: impl core::convert::Into) -> core::option::Option +pub struct vortex_array::arrays::struct_::StructDataParts -pub fn vortex_array::arrays::struct_::StructData::struct_fields(&self) -> vortex_array::dtype::StructFields +pub vortex_array::arrays::struct_::StructDataParts::fields: alloc::sync::Arc<[vortex_array::ArrayRef]> -pub fn vortex_array::arrays::struct_::StructData::try_from_iter, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T) -> vortex_error::VortexResult +pub vortex_array::arrays::struct_::StructDataParts::struct_fields: vortex_array::dtype::StructFields -pub fn vortex_array::arrays::struct_::StructData::try_from_iter_with_validity, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult +pub vortex_array::arrays::struct_::StructDataParts::validity: vortex_array::validity::Validity -pub fn vortex_array::arrays::struct_::StructData::unmasked_field(&self, idx: usize) -> &vortex_array::ArrayRef +pub trait vortex_array::arrays::struct_::StructArrayExt: vortex_array::TypedArrayRef -pub fn vortex_array::arrays::struct_::StructData::unmasked_field_by_name(&self, name: impl core::convert::AsRef) -> vortex_error::VortexResult<&vortex_array::ArrayRef> +pub fn vortex_array::arrays::struct_::StructArrayExt::iter_unmasked_fields(&self) -> impl core::iter::traits::iterator::Iterator + '_ -pub fn vortex_array::arrays::struct_::StructData::unmasked_field_by_name_opt(&self, name: impl core::convert::AsRef) -> core::option::Option<&vortex_array::ArrayRef> +pub fn vortex_array::arrays::struct_::StructArrayExt::names(&self) -> &vortex_array::dtype::FieldNames -pub fn vortex_array::arrays::struct_::StructData::unmasked_fields(&self) -> alloc::sync::Arc<[vortex_array::ArrayRef]> +pub fn vortex_array::arrays::struct_::StructArrayExt::nullability(&self) -> vortex_array::dtype::Nullability -pub fn vortex_array::arrays::struct_::StructData::validate(fields: &[vortex_array::ArrayRef], dtype: &vortex_array::dtype::StructFields, length: usize, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::struct_::StructArrayExt::struct_fields(&self) -> vortex_array::dtype::StructFields -pub fn vortex_array::arrays::struct_::StructData::validity(&self) -> vortex_array::validity::Validity +pub fn vortex_array::arrays::struct_::StructArrayExt::struct_validity(&self) -> vortex_array::validity::Validity -impl vortex_array::arrays::struct_::StructData +pub fn vortex_array::arrays::struct_::StructArrayExt::unmasked_field(&self, idx: usize) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::struct_::StructData::with_column(&self, name: impl core::convert::Into, array: vortex_array::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_array::arrays::struct_::StructArrayExt::unmasked_field_by_name(&self, name: impl core::convert::AsRef) -> vortex_error::VortexResult<&vortex_array::ArrayRef> -impl core::clone::Clone for vortex_array::arrays::struct_::StructData +pub fn vortex_array::arrays::struct_::StructArrayExt::unmasked_field_by_name_opt(&self, name: impl core::convert::AsRef) -> core::option::Option<&vortex_array::ArrayRef> -pub fn vortex_array::arrays::struct_::StructData::clone(&self) -> vortex_array::arrays::struct_::StructData +pub fn vortex_array::arrays::struct_::StructArrayExt::unmasked_fields(&self) -> alloc::sync::Arc<[vortex_array::ArrayRef]> -impl core::fmt::Debug for vortex_array::arrays::struct_::StructData +impl> vortex_array::arrays::struct_::StructArrayExt for T -pub fn vortex_array::arrays::struct_::StructData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn T::iter_unmasked_fields(&self) -> impl core::iter::traits::iterator::Iterator + '_ -pub struct vortex_array::arrays::struct_::StructDataParts +pub fn T::names(&self) -> &vortex_array::dtype::FieldNames -pub vortex_array::arrays::struct_::StructDataParts::fields: alloc::sync::Arc<[vortex_array::ArrayRef]> +pub fn T::nullability(&self) -> vortex_array::dtype::Nullability -pub vortex_array::arrays::struct_::StructDataParts::struct_fields: vortex_array::dtype::StructFields +pub fn T::struct_fields(&self) -> vortex_array::dtype::StructFields -pub vortex_array::arrays::struct_::StructDataParts::validity: vortex_array::validity::Validity +pub fn T::struct_validity(&self) -> vortex_array::validity::Validity + +pub fn T::unmasked_field(&self, idx: usize) -> &vortex_array::ArrayRef + +pub fn T::unmasked_field_by_name(&self, name: impl core::convert::AsRef) -> vortex_error::VortexResult<&vortex_array::ArrayRef> + +pub fn T::unmasked_field_by_name_opt(&self, name: impl core::convert::AsRef) -> core::option::Option<&vortex_array::ArrayRef> + +pub fn T::unmasked_fields(&self) -> alloc::sync::Arc<[vortex_array::ArrayRef]> pub type vortex_array::arrays::struct_::StructArray = vortex_array::Array @@ -4142,10 +4336,6 @@ pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::Va pub fn vortex_array::arrays::VarBin::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::VarBin::array_eq(array: &vortex_array::arrays::varbin::VarBinData, other: &vortex_array::arrays::varbin::VarBinData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::VarBin::array_hash(array: &vortex_array::arrays::varbin::VarBinData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -4154,7 +4344,7 @@ pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -4176,11 +4366,7 @@ pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::validate(&self, _data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::VarBin @@ -4214,107 +4400,93 @@ pub struct vortex_array::arrays::varbin::VarBinData impl vortex_array::arrays::varbin::VarBinData -pub fn vortex_array::arrays::varbin::VarBinData::bytes(&self) -> &vortex_buffer::ByteBuffer - -pub fn vortex_array::arrays::varbin::VarBinData::bytes_handle(&self) -> &vortex_array::buffer::BufferHandle - -pub fn vortex_array::arrays::varbin::VarBinData::dtype(&self) -> vortex_array::dtype::DType - -pub fn vortex_array::arrays::varbin::VarBinData::from_iter, I: core::iter::traits::collect::IntoIterator>>(iter: I, dtype: vortex_array::dtype::DType) -> Self - -pub fn vortex_array::arrays::varbin::VarBinData::from_iter_nonnull, I: core::iter::traits::collect::IntoIterator>(iter: I, dtype: vortex_array::dtype::DType) -> Self - -pub fn vortex_array::arrays::varbin::VarBinData::from_vec>(vec: alloc::vec::Vec, dtype: vortex_array::dtype::DType) -> Self - -pub fn vortex_array::arrays::varbin::VarBinData::is_empty(&self) -> bool - -pub fn vortex_array::arrays::varbin::VarBinData::len(&self) -> usize - -pub fn vortex_array::arrays::varbin::VarBinData::new(offsets: vortex_array::ArrayRef, bytes: vortex_buffer::ByteBuffer, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::arrays::varbin::VarBinData::build(offsets: vortex_array::ArrayRef, bytes: vortex_buffer::ByteBuffer, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self -pub fn vortex_array::arrays::varbin::VarBinData::new_from_handle(offset: vortex_array::ArrayRef, bytes: vortex_array::buffer::BufferHandle, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::arrays::varbin::VarBinData::build_from_handle(offset: vortex_array::ArrayRef, bytes: vortex_array::buffer::BufferHandle, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self -pub unsafe fn vortex_array::arrays::varbin::VarBinData::new_unchecked(offsets: vortex_array::ArrayRef, bytes: vortex_buffer::ByteBuffer, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::arrays::varbin::VarBinData::bytes(&self) -> &vortex_buffer::ByteBuffer -pub unsafe fn vortex_array::arrays::varbin::VarBinData::new_unchecked_from_handle(offsets: vortex_array::ArrayRef, bytes: vortex_array::buffer::BufferHandle, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::arrays::varbin::VarBinData::bytes_handle(&self) -> &vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::varbin::VarBinData::offsets(&self) -> &vortex_array::ArrayRef +pub unsafe fn vortex_array::arrays::varbin::VarBinData::new_unchecked(bytes: vortex_buffer::ByteBuffer) -> Self -pub fn vortex_array::arrays::varbin::VarBinData::sliced_bytes(&self) -> vortex_buffer::ByteBuffer +pub unsafe fn vortex_array::arrays::varbin::VarBinData::new_unchecked_from_handle(bytes: vortex_array::buffer::BufferHandle) -> Self -pub fn vortex_array::arrays::varbin::VarBinData::try_new(offsets: vortex_array::ArrayRef, bytes: vortex_buffer::ByteBuffer, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult +pub fn vortex_array::arrays::varbin::VarBinData::try_build(offsets: vortex_array::ArrayRef, bytes: vortex_buffer::ByteBuffer, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult -pub fn vortex_array::arrays::varbin::VarBinData::try_new_from_handle(offsets: vortex_array::ArrayRef, bytes: vortex_array::buffer::BufferHandle, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult +pub fn vortex_array::arrays::varbin::VarBinData::try_build_from_handle(offsets: vortex_array::ArrayRef, bytes: vortex_array::buffer::BufferHandle, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult pub fn vortex_array::arrays::varbin::VarBinData::validate(offsets: &vortex_array::ArrayRef, bytes: &vortex_array::buffer::BufferHandle, dtype: &vortex_array::dtype::DType, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::varbin::VarBinData::validity(&self) -> vortex_array::validity::Validity +impl core::clone::Clone for vortex_array::arrays::varbin::VarBinData -pub fn vortex_array::arrays::varbin::VarBinData::validity_mask(&self) -> vortex_mask::Mask +pub fn vortex_array::arrays::varbin::VarBinData::clone(&self) -> vortex_array::arrays::varbin::VarBinData -impl vortex_array::arrays::varbin::VarBinData +impl core::fmt::Debug for vortex_array::arrays::varbin::VarBinData -pub fn vortex_array::arrays::varbin::VarBinData::bytes_at(&self, index: usize) -> vortex_buffer::ByteBuffer +pub fn vortex_array::arrays::varbin::VarBinData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::varbin::VarBinData::into_parts(self) -> (vortex_array::dtype::DType, vortex_array::buffer::BufferHandle, vortex_array::ArrayRef, vortex_array::validity::Validity) +impl vortex_array::ArrayEq for vortex_array::arrays::varbin::VarBinData -pub fn vortex_array::arrays::varbin::VarBinData::offset_at(&self, index: usize) -> usize +pub fn vortex_array::arrays::varbin::VarBinData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool -impl core::clone::Clone for vortex_array::arrays::varbin::VarBinData +impl vortex_array::ArrayHash for vortex_array::arrays::varbin::VarBinData -pub fn vortex_array::arrays::varbin::VarBinData::clone(&self) -> vortex_array::arrays::varbin::VarBinData +pub fn vortex_array::arrays::varbin::VarBinData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) -impl core::convert::From> for vortex_array::arrays::varbin::VarBinData +pub struct vortex_array::arrays::varbin::VarBinDataParts -pub fn vortex_array::arrays::varbin::VarBinData::from(value: alloc::vec::Vec<&[u8]>) -> Self +pub vortex_array::arrays::varbin::VarBinDataParts::bytes: vortex_array::buffer::BufferHandle -impl core::convert::From> for vortex_array::arrays::varbin::VarBinData +pub vortex_array::arrays::varbin::VarBinDataParts::dtype: vortex_array::dtype::DType -pub fn vortex_array::arrays::varbin::VarBinData::from(value: alloc::vec::Vec<&str>) -> Self +pub vortex_array::arrays::varbin::VarBinDataParts::offsets: vortex_array::ArrayRef -impl core::convert::From> for vortex_array::arrays::varbin::VarBinData +pub vortex_array::arrays::varbin::VarBinDataParts::validity: vortex_array::validity::Validity -pub fn vortex_array::arrays::varbin::VarBinData::from(value: alloc::vec::Vec) -> Self +pub trait vortex_array::arrays::varbin::VarBinArrayExt: vortex_array::TypedArrayRef -impl core::convert::From>> for vortex_array::arrays::varbin::VarBinData +pub fn vortex_array::arrays::varbin::VarBinArrayExt::bytes_at(&self, index: usize) -> vortex_buffer::ByteBuffer -pub fn vortex_array::arrays::varbin::VarBinData::from(value: alloc::vec::Vec>) -> Self +pub fn vortex_array::arrays::varbin::VarBinArrayExt::dtype_parts(&self) -> (bool, vortex_array::dtype::Nullability) -impl core::convert::From>> for vortex_array::arrays::varbin::VarBinData +pub fn vortex_array::arrays::varbin::VarBinArrayExt::is_utf8(&self) -> bool -pub fn vortex_array::arrays::varbin::VarBinData::from(value: alloc::vec::Vec>) -> Self +pub fn vortex_array::arrays::varbin::VarBinArrayExt::nullability(&self) -> vortex_array::dtype::Nullability -impl core::convert::From>> for vortex_array::arrays::varbin::VarBinData +pub fn vortex_array::arrays::varbin::VarBinArrayExt::offset_at(&self, index: usize) -> usize -pub fn vortex_array::arrays::varbin::VarBinData::from(value: alloc::vec::Vec>) -> Self +pub fn vortex_array::arrays::varbin::VarBinArrayExt::offsets(&self) -> &vortex_array::ArrayRef -impl core::convert::From>> for vortex_array::arrays::varbin::VarBinData +pub fn vortex_array::arrays::varbin::VarBinArrayExt::sliced_bytes(&self) -> vortex_buffer::ByteBuffer -pub fn vortex_array::arrays::varbin::VarBinData::from(value: alloc::vec::Vec>) -> Self +pub fn vortex_array::arrays::varbin::VarBinArrayExt::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> -impl core::convert::From>>> for vortex_array::arrays::varbin::VarBinData +pub fn vortex_array::arrays::varbin::VarBinArrayExt::varbin_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::varbin::VarBinData::from(value: alloc::vec::Vec>>) -> Self +pub fn vortex_array::arrays::varbin::VarBinArrayExt::varbin_validity_mask(&self) -> vortex_mask::Mask -impl core::fmt::Debug for vortex_array::arrays::varbin::VarBinData +impl> vortex_array::arrays::varbin::VarBinArrayExt for T -pub fn vortex_array::arrays::varbin::VarBinData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn T::bytes_at(&self, index: usize) -> vortex_buffer::ByteBuffer + +pub fn T::dtype_parts(&self) -> (bool, vortex_array::dtype::Nullability) -impl core::iter::traits::collect::FromIterator> for vortex_array::arrays::varbin::VarBinData +pub fn T::is_utf8(&self) -> bool -pub fn vortex_array::arrays::varbin::VarBinData::from_iter>>(iter: T) -> Self +pub fn T::nullability(&self) -> vortex_array::dtype::Nullability -impl core::iter::traits::collect::FromIterator>> for vortex_array::arrays::varbin::VarBinData +pub fn T::offset_at(&self, index: usize) -> usize -pub fn vortex_array::arrays::varbin::VarBinData::from_iter>>>(iter: T) -> Self +pub fn T::offsets(&self) -> &vortex_array::ArrayRef -impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::arrays::varbin::VarBinData +pub fn T::sliced_bytes(&self) -> vortex_buffer::ByteBuffer -pub fn vortex_array::arrays::varbin::VarBinData::from_iter>>(iter: T) -> Self +pub fn T::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> -impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::arrays::varbin::VarBinData +pub fn T::varbin_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::varbin::VarBinData::from_iter>>(iter: T) -> Self +pub fn T::varbin_validity_mask(&self) -> vortex_mask::Mask pub fn vortex_array::arrays::varbin::varbin_scalar(value: vortex_buffer::ByteBuffer, dtype: &vortex_array::dtype::DType) -> vortex_array::scalar::Scalar @@ -4532,10 +4704,6 @@ pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::VarBinView::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::VarBinView::array_eq(array: &vortex_array::arrays::varbinview::VarBinViewData, other: &vortex_array::arrays::varbinview::VarBinViewData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::VarBinView::array_hash(array: &vortex_array::arrays::varbinview::VarBinViewData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -4544,7 +4712,7 @@ pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -4566,11 +4734,7 @@ pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::VarBinView @@ -4606,8 +4770,6 @@ pub fn vortex_array::arrays::varbinview::VarBinViewData::bytes_at(&self, index: pub fn vortex_array::arrays::varbinview::VarBinViewData::data_buffers(&self) -> &alloc::sync::Arc<[vortex_array::buffer::BufferHandle]> -pub fn vortex_array::arrays::varbinview::VarBinViewData::dtype(&self) -> vortex_array::dtype::DType - pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter, I: core::iter::traits::collect::IntoIterator>>(iter: I, dtype: vortex_array::dtype::DType) -> Self pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter_bin, I: core::iter::traits::collect::IntoIterator>(iter: I) -> Self @@ -4618,8 +4780,6 @@ pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter_nullable_str< pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter_str, I: core::iter::traits::collect::IntoIterator>(iter: I) -> Self -pub fn vortex_array::arrays::varbinview::VarBinViewData::into_parts(self) -> vortex_array::arrays::varbinview::VarBinViewDataParts - pub fn vortex_array::arrays::varbinview::VarBinViewData::is_empty(&self) -> bool pub fn vortex_array::arrays::varbinview::VarBinViewData::len(&self) -> usize @@ -4628,7 +4788,7 @@ pub fn vortex_array::arrays::varbinview::VarBinViewData::new(views: vortex_buffe pub fn vortex_array::arrays::varbinview::VarBinViewData::new_handle(views: vortex_array::buffer::BufferHandle, buffers: alloc::sync::Arc<[vortex_array::buffer::BufferHandle]>, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self -pub unsafe fn vortex_array::arrays::varbinview::VarBinViewData::new_handle_unchecked(views: vortex_array::buffer::BufferHandle, buffers: alloc::sync::Arc<[vortex_array::buffer::BufferHandle]>, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self +pub unsafe fn vortex_array::arrays::varbinview::VarBinViewData::new_handle_unchecked(views: vortex_array::buffer::BufferHandle, buffers: alloc::sync::Arc<[vortex_array::buffer::BufferHandle]>, dtype: vortex_array::dtype::DType, _validity: vortex_array::validity::Validity) -> Self pub unsafe fn vortex_array::arrays::varbinview::VarBinViewData::new_unchecked(views: vortex_buffer::buffer::Buffer, buffers: alloc::sync::Arc<[vortex_buffer::ByteBuffer]>, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self @@ -4638,10 +4798,6 @@ pub fn vortex_array::arrays::varbinview::VarBinViewData::try_new_handle(views: v pub fn vortex_array::arrays::varbinview::VarBinViewData::validate(views: &vortex_buffer::buffer::Buffer, buffers: &alloc::sync::Arc<[vortex_buffer::ByteBuffer]>, dtype: &vortex_array::dtype::DType, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::varbinview::VarBinViewData::validity(&self) -> vortex_array::validity::Validity - -pub fn vortex_array::arrays::varbinview::VarBinViewData::validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::varbinview::VarBinViewData::views(&self) -> &[vortex_array::arrays::varbinview::BinaryView] pub fn vortex_array::arrays::varbinview::VarBinViewData::views_handle(&self) -> &vortex_array::buffer::BufferHandle @@ -4662,6 +4818,14 @@ impl core::iter::traits::collect::FromIterator>>>(iter: T) -> Self +impl vortex_array::ArrayEq for vortex_array::arrays::varbinview::VarBinViewData + +pub fn vortex_array::arrays::varbinview::VarBinViewData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayHash for vortex_array::arrays::varbinview::VarBinViewData + +pub fn vortex_array::arrays::varbinview::VarBinViewData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) + impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::arrays::varbinview::VarBinViewData pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter>>(iter: T) -> Self @@ -4680,6 +4844,22 @@ pub vortex_array::arrays::varbinview::VarBinViewDataParts::validity: vortex_arra pub vortex_array::arrays::varbinview::VarBinViewDataParts::views: vortex_array::buffer::BufferHandle +pub trait vortex_array::arrays::varbinview::VarBinViewArrayExt: vortex_array::TypedArrayRef + +pub fn vortex_array::arrays::varbinview::VarBinViewArrayExt::dtype_parts(&self) -> (bool, vortex_array::dtype::Nullability) + +pub fn vortex_array::arrays::varbinview::VarBinViewArrayExt::varbinview_validity(&self) -> vortex_array::validity::Validity + +pub fn vortex_array::arrays::varbinview::VarBinViewArrayExt::varbinview_validity_mask(&self) -> vortex_mask::Mask + +impl> vortex_array::arrays::varbinview::VarBinViewArrayExt for T + +pub fn T::dtype_parts(&self) -> (bool, vortex_array::dtype::Nullability) + +pub fn T::varbinview_validity(&self) -> vortex_array::validity::Validity + +pub fn T::varbinview_validity_mask(&self) -> vortex_mask::Mask + pub type vortex_array::arrays::varbinview::VarBinViewArray = vortex_array::Array pub mod vortex_array::arrays::variant @@ -4712,10 +4892,6 @@ pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::V pub fn vortex_array::arrays::Variant::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Variant::array_eq(array: &vortex_array::arrays::variant::VariantData, other: &vortex_array::arrays::variant::VariantData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Variant::array_hash(array: &vortex_array::arrays::variant::VariantData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -4724,7 +4900,7 @@ pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -4746,11 +4922,7 @@ pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Variant @@ -4758,25 +4930,29 @@ pub fn vortex_array::arrays::Variant::validity(array: vortex_array::ArrayView<'_ pub struct vortex_array::arrays::variant::VariantData -impl vortex_array::arrays::variant::VariantData +impl core::clone::Clone for vortex_array::arrays::variant::VariantData + +pub fn vortex_array::arrays::variant::VariantData::clone(&self) -> vortex_array::arrays::variant::VariantData + +impl core::fmt::Debug for vortex_array::arrays::variant::VariantData -pub fn vortex_array::arrays::variant::VariantData::child(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::variant::VariantData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::variant::VariantData::dtype(&self) -> vortex_array::dtype::DType +impl vortex_array::ArrayEq for vortex_array::arrays::variant::VariantData -pub fn vortex_array::arrays::variant::VariantData::is_empty(&self) -> bool +pub fn vortex_array::arrays::variant::VariantData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -pub fn vortex_array::arrays::variant::VariantData::len(&self) -> usize +impl vortex_array::ArrayHash for vortex_array::arrays::variant::VariantData -pub fn vortex_array::arrays::variant::VariantData::new(child: vortex_array::ArrayRef) -> Self +pub fn vortex_array::arrays::variant::VariantData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) -impl core::clone::Clone for vortex_array::arrays::variant::VariantData +pub trait vortex_array::arrays::variant::VariantArrayExt: vortex_array::TypedArrayRef -pub fn vortex_array::arrays::variant::VariantData::clone(&self) -> vortex_array::arrays::variant::VariantData +pub fn vortex_array::arrays::variant::VariantArrayExt::child(&self) -> &vortex_array::ArrayRef -impl core::fmt::Debug for vortex_array::arrays::variant::VariantData +impl> vortex_array::arrays::variant::VariantArrayExt for T -pub fn vortex_array::arrays::variant::VariantData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn T::child(&self) -> &vortex_array::ArrayRef pub type vortex_array::arrays::variant::VariantArray = vortex_array::Array @@ -4808,10 +4984,6 @@ pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool pub fn vortex_array::arrays::Bool::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Bool::array_eq(array: &vortex_array::arrays::bool::BoolData, other: &vortex_array::arrays::bool::BoolData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Bool::array_hash(array: &vortex_array::arrays::bool::BoolData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -4820,7 +4992,7 @@ pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -4842,11 +5014,7 @@ pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Bool @@ -4910,10 +5078,6 @@ pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::C pub fn vortex_array::arrays::Chunked::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Chunked::array_eq(array: &vortex_array::arrays::chunked::ChunkedData, other: &vortex_array::arrays::chunked::ChunkedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Chunked::array_hash(array: &vortex_array::arrays::chunked::ChunkedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -4922,7 +5086,7 @@ pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -4944,11 +5108,7 @@ pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Chunked @@ -5014,10 +5174,6 @@ pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays:: pub fn vortex_array::arrays::Constant::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Constant::array_eq(array: &vortex_array::arrays::constant::ConstantData, other: &vortex_array::arrays::constant::ConstantData, _precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Constant::array_hash(array: &vortex_array::arrays::constant::ConstantData, state: &mut H, _precision: vortex_array::Precision) - pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -5026,7 +5182,7 @@ pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5048,11 +5204,7 @@ pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Constant @@ -5114,10 +5266,6 @@ pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::D pub fn vortex_array::arrays::Decimal::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Decimal::array_eq(array: &vortex_array::arrays::decimal::DecimalData, other: &vortex_array::arrays::decimal::DecimalData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Decimal::array_hash(array: &vortex_array::arrays::decimal::DecimalData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -5126,7 +5274,7 @@ pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5148,11 +5296,7 @@ pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Decimal @@ -5216,10 +5360,6 @@ pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::dict::Dict::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::dict::Dict::array_eq(array: &vortex_array::arrays::dict::DictData, other: &vortex_array::arrays::dict::DictData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::dict::Dict::array_hash(array: &vortex_array::arrays::dict::DictData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -5228,7 +5368,7 @@ pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5250,11 +5390,7 @@ pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayVie pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::dict::Dict @@ -5320,10 +5456,6 @@ pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::Validit pub fn vortex_array::arrays::Extension::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Extension::array_eq(array: &vortex_array::arrays::extension::ExtensionData, other: &vortex_array::arrays::extension::ExtensionData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Extension::array_hash(array: &vortex_array::arrays::extension::ExtensionData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -5332,7 +5464,7 @@ pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5354,15 +5486,11 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityChild for vortex_array::arrays::Extension -pub fn vortex_array::arrays::Extension::validity_child(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::Extension::validity_child(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>) -> vortex_array::ArrayRef impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::Extension @@ -5416,10 +5544,6 @@ pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Fi pub fn vortex_array::arrays::Filter::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Filter::array_eq(array: &vortex_array::arrays::filter::FilterData, other: &vortex_array::arrays::filter::FilterData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Filter::array_hash(array: &vortex_array::arrays::filter::FilterData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -5428,7 +5552,7 @@ pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5450,11 +5574,7 @@ pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Filter @@ -5488,10 +5608,6 @@ pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arr pub fn vortex_array::arrays::FixedSizeList::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::FixedSizeList::array_eq(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData, other: &vortex_array::arrays::fixed_size_list::FixedSizeListData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::FixedSizeList::array_hash(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -5500,7 +5616,7 @@ pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5522,11 +5638,7 @@ pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::Arra pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::FixedSizeList @@ -5576,10 +5688,6 @@ pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List pub fn vortex_array::arrays::List::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::List::array_eq(array: &vortex_array::arrays::list::ListData, other: &vortex_array::arrays::list::ListData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::List::array_hash(array: &vortex_array::arrays::list::ListData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -5588,7 +5696,7 @@ pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5610,11 +5718,7 @@ pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::validate(&self, _data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::List @@ -5668,10 +5772,6 @@ pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays:: pub fn vortex_array::arrays::ListView::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::ListView::array_eq(array: &vortex_array::arrays::listview::ListViewData, other: &vortex_array::arrays::listview::ListViewData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::ListView::array_hash(array: &vortex_array::arrays::listview::ListViewData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -5680,7 +5780,7 @@ pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5702,11 +5802,7 @@ pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView< pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::validate(&self, _data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::ListView @@ -5756,10 +5852,6 @@ pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Ma pub fn vortex_array::arrays::Masked::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Masked::array_eq(array: &vortex_array::arrays::masked::MaskedData, other: &vortex_array::arrays::masked::MaskedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Masked::array_hash(array: &vortex_array::arrays::masked::MaskedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -5768,7 +5860,7 @@ pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5790,11 +5882,7 @@ pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::validate(&self, _data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Masked @@ -5848,10 +5936,6 @@ pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::null::Null::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::null::Null::array_eq(_array: &vortex_array::arrays::null::NullData, _other: &vortex_array::arrays::null::NullData, _precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::null::Null::array_hash(_array: &vortex_array::arrays::null::NullData, _state: &mut H, _precision: vortex_array::Precision) - pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -5860,7 +5944,7 @@ pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5882,11 +5966,7 @@ pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::null::Null @@ -5914,6 +5994,10 @@ pub fn vortex_array::arrays::null::Null::mask(array: vortex_array::ArrayView<'_, pub struct vortex_array::arrays::Patched +impl vortex_array::arrays::patched::Patched + +pub fn vortex_array::arrays::patched::Patched::from_array_and_patches(inner: vortex_array::ArrayRef, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl core::clone::Clone for vortex_array::arrays::patched::Patched pub fn vortex_array::arrays::patched::Patched::clone(&self) -> vortex_array::arrays::patched::Patched @@ -5928,7 +6012,7 @@ pub fn vortex_array::arrays::patched::Patched::scalar_at(array: vortex_array::Ar impl vortex_array::VTable for vortex_array::arrays::patched::Patched -pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray +pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedData pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched @@ -5936,10 +6020,6 @@ pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array:: pub fn vortex_array::arrays::patched::Patched::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::patched::Patched::array_eq(array: &vortex_array::arrays::patched::PatchedArray, other: &vortex_array::arrays::patched::PatchedArray, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::patched::Patched::array_hash(array: &vortex_array::arrays::patched::PatchedArray, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -5948,7 +6028,7 @@ pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayV pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5970,15 +6050,11 @@ pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::Ar pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityChild for vortex_array::arrays::patched::Patched -pub fn vortex_array::arrays::patched::Patched::validity_child(array: &vortex_array::arrays::patched::PatchedArray) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::Patched::validity_child(array: vortex_array::ArrayView<'_, vortex_array::arrays::patched::Patched>) -> vortex_array::ArrayRef impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::patched::Patched @@ -5996,46 +6072,6 @@ impl vortex_array::scalar_fn::fns::binary::CompareKernel for vortex_array::array pub fn vortex_array::arrays::patched::Patched::compare(lhs: vortex_array::ArrayView<'_, Self>, rhs: &vortex_array::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::PatchedArray - -impl vortex_array::arrays::patched::PatchedArray - -pub fn vortex_array::arrays::patched::PatchedArray::base_array(&self) -> &vortex_array::ArrayRef - -pub fn vortex_array::arrays::patched::PatchedArray::is_empty(&self) -> bool - -pub fn vortex_array::arrays::patched::PatchedArray::lane_offsets(&self) -> &vortex_array::ArrayRef - -pub fn vortex_array::arrays::patched::PatchedArray::len(&self) -> usize - -pub fn vortex_array::arrays::patched::PatchedArray::patch_indices(&self) -> &vortex_array::ArrayRef - -pub fn vortex_array::arrays::patched::PatchedArray::patch_values(&self) -> &vortex_array::ArrayRef - -impl vortex_array::arrays::patched::PatchedArray - -pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner: vortex_array::ArrayRef, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult - -impl vortex_array::arrays::patched::PatchedArray - -pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef where Self: core::clone::Clone + vortex_array::IntoArray - -impl core::clone::Clone for vortex_array::arrays::patched::PatchedArray - -pub fn vortex_array::arrays::patched::PatchedArray::clone(&self) -> vortex_array::arrays::patched::PatchedArray - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> Self - -impl core::fmt::Debug for vortex_array::arrays::patched::PatchedArray - -pub fn vortex_array::arrays::patched::PatchedArray::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl vortex_array::IntoArray for vortex_array::arrays::patched::PatchedArray - -pub fn vortex_array::arrays::patched::PatchedArray::into_array(self) -> vortex_array::ArrayRef - pub struct vortex_array::arrays::Primitive impl vortex_array::arrays::Primitive @@ -6064,10 +6100,6 @@ pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays: pub fn vortex_array::arrays::Primitive::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Primitive::array_eq(array: &vortex_array::arrays::primitive::PrimitiveData, other: &vortex_array::arrays::primitive::PrimitiveData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Primitive::array_hash(array: &vortex_array::arrays::primitive::PrimitiveData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -6076,7 +6108,7 @@ pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6098,11 +6130,7 @@ pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Primitive @@ -6162,10 +6190,6 @@ pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vorte pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_eq(array: &vortex_array::arrays::scalar_fn::ScalarFnData, other: &vortex_array::arrays::scalar_fn::ScalarFnData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_hash(array: &vortex_array::arrays::scalar_fn::ScalarFnData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -6174,7 +6198,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_arra pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6196,11 +6220,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -6234,10 +6254,6 @@ pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Sh pub fn vortex_array::arrays::Shared::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Shared::array_eq(array: &vortex_array::arrays::shared::SharedData, other: &vortex_array::arrays::shared::SharedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Shared::array_hash(array: &vortex_array::arrays::shared::SharedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -6246,7 +6262,7 @@ pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6268,11 +6284,7 @@ pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::validate(&self, _data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Shared @@ -6306,10 +6318,6 @@ pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arra pub fn vortex_array::arrays::slice::Slice::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::slice::Slice::array_eq(array: &vortex_array::arrays::slice::SliceData, other: &vortex_array::arrays::slice::SliceData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::slice::Slice::array_hash(array: &vortex_array::arrays::slice::SliceData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -6318,7 +6326,7 @@ pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView< pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6340,11 +6348,7 @@ pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::Array pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::slice::Slice @@ -6382,10 +6386,6 @@ pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::St pub fn vortex_array::arrays::Struct::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Struct::array_eq(array: &vortex_array::arrays::struct_::StructData, other: &vortex_array::arrays::struct_::StructData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Struct::array_hash(array: &vortex_array::arrays::struct_::StructData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -6394,7 +6394,7 @@ pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6416,11 +6416,7 @@ pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Struct @@ -6478,10 +6474,6 @@ pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::Va pub fn vortex_array::arrays::VarBin::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::VarBin::array_eq(array: &vortex_array::arrays::varbin::VarBinData, other: &vortex_array::arrays::varbin::VarBinData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::VarBin::array_hash(array: &vortex_array::arrays::varbin::VarBinData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -6490,7 +6482,7 @@ pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6512,11 +6504,7 @@ pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::validate(&self, _data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::VarBin @@ -6574,10 +6562,6 @@ pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::VarBinView::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::VarBinView::array_eq(array: &vortex_array::arrays::varbinview::VarBinViewData, other: &vortex_array::arrays::varbinview::VarBinViewData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::VarBinView::array_hash(array: &vortex_array::arrays::varbinview::VarBinViewData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -6586,7 +6570,7 @@ pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6608,11 +6592,7 @@ pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::VarBinView @@ -6666,10 +6646,6 @@ pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::V pub fn vortex_array::arrays::Variant::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Variant::array_eq(array: &vortex_array::arrays::variant::VariantData, other: &vortex_array::arrays::variant::VariantData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Variant::array_hash(array: &vortex_array::arrays::variant::VariantData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -6678,7 +6654,7 @@ pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6700,11 +6676,7 @@ pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::Variant @@ -6734,6 +6706,8 @@ pub type vortex_array::arrays::MaskedArray = vortex_array::Array +pub type vortex_array::arrays::PatchedArray = vortex_array::Array + pub type vortex_array::arrays::PrimitiveArray = vortex_array::Array pub type vortex_array::arrays::ScalarFnArray = vortex_array::Array @@ -18888,7 +18862,7 @@ pub fn vortex_array::ValidityVTableFromValiditySliceHelper::validity(array: vort pub trait vortex_array::vtable::ArrayVTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::vtable::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug +pub type vortex_array::vtable::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::ArrayHash + vortex_array::ArrayEq pub type vortex_array::vtable::ArrayVTable::OperationsVTable: vortex_array::OperationsVTable @@ -18896,10 +18870,6 @@ pub type vortex_array::vtable::ArrayVTable::ValidityVTable: vortex_array::Validi pub fn vortex_array::vtable::ArrayVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::vtable::ArrayVTable::array_eq(array: &Self::ArrayData, other: &Self::ArrayData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::vtable::ArrayVTable::array_hash(array: &Self::ArrayData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::vtable::ArrayVTable::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::vtable::ArrayVTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -18908,7 +18878,7 @@ pub fn vortex_array::vtable::ArrayVTable::child(array: vortex_array::ArrayView<' pub fn vortex_array::vtable::ArrayVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::vtable::ArrayVTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::vtable::ArrayVTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::vtable::ArrayVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -18930,11 +18900,7 @@ pub fn vortex_array::vtable::ArrayVTable::serialize(array: vortex_array::ArrayVi pub fn vortex_array::vtable::ArrayVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::vtable::ArrayVTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] - -pub fn vortex_array::vtable::ArrayVTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::vtable::ArrayVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::vtable::ArrayVTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Bool @@ -18946,10 +18912,6 @@ pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool pub fn vortex_array::arrays::Bool::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Bool::array_eq(array: &vortex_array::arrays::bool::BoolData, other: &vortex_array::arrays::bool::BoolData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Bool::array_hash(array: &vortex_array::arrays::bool::BoolData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -18958,7 +18920,7 @@ pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -18980,11 +18942,7 @@ pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Chunked @@ -18996,10 +18954,6 @@ pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::C pub fn vortex_array::arrays::Chunked::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Chunked::array_eq(array: &vortex_array::arrays::chunked::ChunkedData, other: &vortex_array::arrays::chunked::ChunkedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Chunked::array_hash(array: &vortex_array::arrays::chunked::ChunkedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -19008,7 +18962,7 @@ pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19030,11 +18984,7 @@ pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Constant @@ -19046,10 +18996,6 @@ pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays:: pub fn vortex_array::arrays::Constant::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Constant::array_eq(array: &vortex_array::arrays::constant::ConstantData, other: &vortex_array::arrays::constant::ConstantData, _precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Constant::array_hash(array: &vortex_array::arrays::constant::ConstantData, state: &mut H, _precision: vortex_array::Precision) - pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -19058,7 +19004,7 @@ pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19080,11 +19026,7 @@ pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Decimal @@ -19096,10 +19038,6 @@ pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::D pub fn vortex_array::arrays::Decimal::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Decimal::array_eq(array: &vortex_array::arrays::decimal::DecimalData, other: &vortex_array::arrays::decimal::DecimalData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Decimal::array_hash(array: &vortex_array::arrays::decimal::DecimalData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -19108,7 +19046,7 @@ pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19130,11 +19068,7 @@ pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Extension @@ -19146,10 +19080,6 @@ pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::Validit pub fn vortex_array::arrays::Extension::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Extension::array_eq(array: &vortex_array::arrays::extension::ExtensionData, other: &vortex_array::arrays::extension::ExtensionData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Extension::array_hash(array: &vortex_array::arrays::extension::ExtensionData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -19158,7 +19088,7 @@ pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19180,11 +19110,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Filter @@ -19196,10 +19122,6 @@ pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Fi pub fn vortex_array::arrays::Filter::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Filter::array_eq(array: &vortex_array::arrays::filter::FilterData, other: &vortex_array::arrays::filter::FilterData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Filter::array_hash(array: &vortex_array::arrays::filter::FilterData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -19208,7 +19130,7 @@ pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19230,11 +19152,7 @@ pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::FixedSizeList @@ -19244,11 +19162,7 @@ pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::a pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList -pub fn vortex_array::arrays::FixedSizeList::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::FixedSizeList::array_eq(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData, other: &vortex_array::arrays::fixed_size_list::FixedSizeListData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::FixedSizeList::array_hash(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData, state: &mut H, precision: vortex_array::Precision) +pub fn vortex_array::arrays::FixedSizeList::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle @@ -19258,7 +19172,7 @@ pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19280,11 +19194,7 @@ pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::Arra pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::List @@ -19296,10 +19206,6 @@ pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List pub fn vortex_array::arrays::List::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::List::array_eq(array: &vortex_array::arrays::list::ListData, other: &vortex_array::arrays::list::ListData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::List::array_hash(array: &vortex_array::arrays::list::ListData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -19308,7 +19214,7 @@ pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19330,11 +19236,7 @@ pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::validate(&self, _data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::ListView @@ -19346,10 +19248,6 @@ pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays:: pub fn vortex_array::arrays::ListView::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::ListView::array_eq(array: &vortex_array::arrays::listview::ListViewData, other: &vortex_array::arrays::listview::ListViewData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::ListView::array_hash(array: &vortex_array::arrays::listview::ListViewData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -19358,7 +19256,7 @@ pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19380,11 +19278,7 @@ pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView< pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::validate(&self, _data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Masked @@ -19396,10 +19290,6 @@ pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Ma pub fn vortex_array::arrays::Masked::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Masked::array_eq(array: &vortex_array::arrays::masked::MaskedData, other: &vortex_array::arrays::masked::MaskedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Masked::array_hash(array: &vortex_array::arrays::masked::MaskedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -19408,7 +19298,7 @@ pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19430,11 +19320,7 @@ pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::validate(&self, _data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Primitive @@ -19446,10 +19332,6 @@ pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays: pub fn vortex_array::arrays::Primitive::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Primitive::array_eq(array: &vortex_array::arrays::primitive::PrimitiveData, other: &vortex_array::arrays::primitive::PrimitiveData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Primitive::array_hash(array: &vortex_array::arrays::primitive::PrimitiveData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -19458,7 +19340,7 @@ pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19480,11 +19362,7 @@ pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Shared @@ -19496,10 +19374,6 @@ pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Sh pub fn vortex_array::arrays::Shared::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Shared::array_eq(array: &vortex_array::arrays::shared::SharedData, other: &vortex_array::arrays::shared::SharedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Shared::array_hash(array: &vortex_array::arrays::shared::SharedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -19508,7 +19382,7 @@ pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19530,11 +19404,7 @@ pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::validate(&self, _data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Struct @@ -19546,10 +19416,6 @@ pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::St pub fn vortex_array::arrays::Struct::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Struct::array_eq(array: &vortex_array::arrays::struct_::StructData, other: &vortex_array::arrays::struct_::StructData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Struct::array_hash(array: &vortex_array::arrays::struct_::StructData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -19558,7 +19424,7 @@ pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19580,11 +19446,7 @@ pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::VarBin @@ -19596,10 +19458,6 @@ pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::Va pub fn vortex_array::arrays::VarBin::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::VarBin::array_eq(array: &vortex_array::arrays::varbin::VarBinData, other: &vortex_array::arrays::varbin::VarBinData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::VarBin::array_hash(array: &vortex_array::arrays::varbin::VarBinData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -19608,7 +19466,7 @@ pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19630,11 +19488,7 @@ pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::validate(&self, _data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::VarBinView @@ -19646,10 +19500,6 @@ pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::VarBinView::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::VarBinView::array_eq(array: &vortex_array::arrays::varbinview::VarBinViewData, other: &vortex_array::arrays::varbinview::VarBinViewData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::VarBinView::array_hash(array: &vortex_array::arrays::varbinview::VarBinViewData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -19658,7 +19508,7 @@ pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19680,11 +19530,7 @@ pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Variant @@ -19696,10 +19542,6 @@ pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::V pub fn vortex_array::arrays::Variant::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Variant::array_eq(array: &vortex_array::arrays::variant::VariantData, other: &vortex_array::arrays::variant::VariantData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Variant::array_hash(array: &vortex_array::arrays::variant::VariantData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -19708,7 +19550,7 @@ pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19730,11 +19572,7 @@ pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::dict::Dict @@ -19746,10 +19584,6 @@ pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::dict::Dict::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::dict::Dict::array_eq(array: &vortex_array::arrays::dict::DictData, other: &vortex_array::arrays::dict::DictData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::dict::Dict::array_hash(array: &vortex_array::arrays::dict::DictData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -19758,7 +19592,7 @@ pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19780,11 +19614,7 @@ pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayVie pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::null::Null @@ -19796,10 +19626,6 @@ pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::null::Null::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::null::Null::array_eq(_array: &vortex_array::arrays::null::NullData, _other: &vortex_array::arrays::null::NullData, _precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::null::Null::array_hash(_array: &vortex_array::arrays::null::NullData, _state: &mut H, _precision: vortex_array::Precision) - pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -19808,7 +19634,7 @@ pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19830,15 +19656,11 @@ pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::patched::Patched -pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray +pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedData pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched @@ -19846,10 +19668,6 @@ pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array:: pub fn vortex_array::arrays::patched::Patched::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::patched::Patched::array_eq(array: &vortex_array::arrays::patched::PatchedArray, other: &vortex_array::arrays::patched::PatchedArray, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::patched::Patched::array_hash(array: &vortex_array::arrays::patched::PatchedArray, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -19858,7 +19676,7 @@ pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayV pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19880,11 +19698,7 @@ pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::Ar pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -19896,10 +19710,6 @@ pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vorte pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_eq(array: &vortex_array::arrays::scalar_fn::ScalarFnData, other: &vortex_array::arrays::scalar_fn::ScalarFnData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_hash(array: &vortex_array::arrays::scalar_fn::ScalarFnData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -19908,7 +19718,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_arra pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19930,11 +19740,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::slice::Slice @@ -19946,10 +19752,6 @@ pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arra pub fn vortex_array::arrays::slice::Slice::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::slice::Slice::array_eq(array: &vortex_array::arrays::slice::SliceData, other: &vortex_array::arrays::slice::SliceData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::slice::Slice::array_hash(array: &vortex_array::arrays::slice::SliceData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -19958,7 +19760,7 @@ pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView< pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19980,11 +19782,7 @@ pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::Array pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> pub trait vortex_array::vtable::DynVTable: 'static + core::marker::Send + core::marker::Sync + core::fmt::Debug @@ -20112,7 +19910,7 @@ pub fn vortex_array::NotSupported::scalar_at(array: vortex_array::ArrayView<'_, pub trait vortex_array::vtable::VTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::vtable::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug +pub type vortex_array::vtable::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::ArrayHash + vortex_array::ArrayEq pub type vortex_array::vtable::VTable::OperationsVTable: vortex_array::OperationsVTable @@ -20120,10 +19918,6 @@ pub type vortex_array::vtable::VTable::ValidityVTable: vortex_array::ValidityVTa pub fn vortex_array::vtable::VTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::vtable::VTable::array_eq(array: &Self::ArrayData, other: &Self::ArrayData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::vtable::VTable::array_hash(array: &Self::ArrayData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::vtable::VTable::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::vtable::VTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -20132,7 +19926,7 @@ pub fn vortex_array::vtable::VTable::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::vtable::VTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::vtable::VTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::vtable::VTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::vtable::VTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20154,11 +19948,7 @@ pub fn vortex_array::vtable::VTable::serialize(array: vortex_array::ArrayView<'_ pub fn vortex_array::vtable::VTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::vtable::VTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] - -pub fn vortex_array::vtable::VTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::vtable::VTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::vtable::VTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Bool @@ -20170,10 +19960,6 @@ pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool pub fn vortex_array::arrays::Bool::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Bool::array_eq(array: &vortex_array::arrays::bool::BoolData, other: &vortex_array::arrays::bool::BoolData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Bool::array_hash(array: &vortex_array::arrays::bool::BoolData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -20182,7 +19968,7 @@ pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20204,11 +19990,7 @@ pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Chunked @@ -20220,10 +20002,6 @@ pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::C pub fn vortex_array::arrays::Chunked::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Chunked::array_eq(array: &vortex_array::arrays::chunked::ChunkedData, other: &vortex_array::arrays::chunked::ChunkedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Chunked::array_hash(array: &vortex_array::arrays::chunked::ChunkedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -20232,7 +20010,7 @@ pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20254,11 +20032,7 @@ pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Constant @@ -20270,10 +20044,6 @@ pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays:: pub fn vortex_array::arrays::Constant::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Constant::array_eq(array: &vortex_array::arrays::constant::ConstantData, other: &vortex_array::arrays::constant::ConstantData, _precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Constant::array_hash(array: &vortex_array::arrays::constant::ConstantData, state: &mut H, _precision: vortex_array::Precision) - pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -20282,7 +20052,7 @@ pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20304,11 +20074,7 @@ pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Decimal @@ -20320,10 +20086,6 @@ pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::D pub fn vortex_array::arrays::Decimal::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Decimal::array_eq(array: &vortex_array::arrays::decimal::DecimalData, other: &vortex_array::arrays::decimal::DecimalData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Decimal::array_hash(array: &vortex_array::arrays::decimal::DecimalData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -20332,7 +20094,7 @@ pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20354,11 +20116,7 @@ pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Extension @@ -20370,10 +20128,6 @@ pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::Validit pub fn vortex_array::arrays::Extension::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Extension::array_eq(array: &vortex_array::arrays::extension::ExtensionData, other: &vortex_array::arrays::extension::ExtensionData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Extension::array_hash(array: &vortex_array::arrays::extension::ExtensionData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -20382,7 +20136,7 @@ pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20404,11 +20158,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Filter @@ -20420,10 +20170,6 @@ pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Fi pub fn vortex_array::arrays::Filter::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Filter::array_eq(array: &vortex_array::arrays::filter::FilterData, other: &vortex_array::arrays::filter::FilterData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Filter::array_hash(array: &vortex_array::arrays::filter::FilterData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -20432,7 +20178,7 @@ pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20454,11 +20200,7 @@ pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::FixedSizeList @@ -20470,10 +20212,6 @@ pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arr pub fn vortex_array::arrays::FixedSizeList::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::FixedSizeList::array_eq(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData, other: &vortex_array::arrays::fixed_size_list::FixedSizeListData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::FixedSizeList::array_hash(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -20482,7 +20220,7 @@ pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20504,11 +20242,7 @@ pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::Arra pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::List @@ -20520,10 +20254,6 @@ pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List pub fn vortex_array::arrays::List::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::List::array_eq(array: &vortex_array::arrays::list::ListData, other: &vortex_array::arrays::list::ListData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::List::array_hash(array: &vortex_array::arrays::list::ListData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -20532,7 +20262,7 @@ pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20554,11 +20284,7 @@ pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::validate(&self, _data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::ListView @@ -20570,10 +20296,6 @@ pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays:: pub fn vortex_array::arrays::ListView::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::ListView::array_eq(array: &vortex_array::arrays::listview::ListViewData, other: &vortex_array::arrays::listview::ListViewData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::ListView::array_hash(array: &vortex_array::arrays::listview::ListViewData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -20582,7 +20304,7 @@ pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20604,11 +20326,7 @@ pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView< pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::validate(&self, _data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Masked @@ -20620,10 +20338,6 @@ pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Ma pub fn vortex_array::arrays::Masked::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Masked::array_eq(array: &vortex_array::arrays::masked::MaskedData, other: &vortex_array::arrays::masked::MaskedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Masked::array_hash(array: &vortex_array::arrays::masked::MaskedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -20632,7 +20346,7 @@ pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20654,11 +20368,7 @@ pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::validate(&self, _data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Primitive @@ -20670,10 +20380,6 @@ pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays: pub fn vortex_array::arrays::Primitive::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Primitive::array_eq(array: &vortex_array::arrays::primitive::PrimitiveData, other: &vortex_array::arrays::primitive::PrimitiveData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Primitive::array_hash(array: &vortex_array::arrays::primitive::PrimitiveData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -20682,7 +20388,7 @@ pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20704,11 +20410,7 @@ pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Shared @@ -20720,10 +20422,6 @@ pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Sh pub fn vortex_array::arrays::Shared::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Shared::array_eq(array: &vortex_array::arrays::shared::SharedData, other: &vortex_array::arrays::shared::SharedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Shared::array_hash(array: &vortex_array::arrays::shared::SharedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -20732,7 +20430,7 @@ pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20754,11 +20452,7 @@ pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::validate(&self, _data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Struct @@ -20770,10 +20464,6 @@ pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::St pub fn vortex_array::arrays::Struct::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Struct::array_eq(array: &vortex_array::arrays::struct_::StructData, other: &vortex_array::arrays::struct_::StructData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Struct::array_hash(array: &vortex_array::arrays::struct_::StructData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -20782,7 +20472,7 @@ pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20804,11 +20494,7 @@ pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::VarBin @@ -20820,10 +20506,6 @@ pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::Va pub fn vortex_array::arrays::VarBin::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::VarBin::array_eq(array: &vortex_array::arrays::varbin::VarBinData, other: &vortex_array::arrays::varbin::VarBinData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::VarBin::array_hash(array: &vortex_array::arrays::varbin::VarBinData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -20832,7 +20514,7 @@ pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20854,11 +20536,7 @@ pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::validate(&self, _data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::VarBinView @@ -20870,10 +20548,6 @@ pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::VarBinView::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::VarBinView::array_eq(array: &vortex_array::arrays::varbinview::VarBinViewData, other: &vortex_array::arrays::varbinview::VarBinViewData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::VarBinView::array_hash(array: &vortex_array::arrays::varbinview::VarBinViewData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -20882,7 +20556,7 @@ pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20904,11 +20578,7 @@ pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Variant @@ -20920,10 +20590,6 @@ pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::V pub fn vortex_array::arrays::Variant::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Variant::array_eq(array: &vortex_array::arrays::variant::VariantData, other: &vortex_array::arrays::variant::VariantData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Variant::array_hash(array: &vortex_array::arrays::variant::VariantData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -20932,7 +20598,7 @@ pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20954,11 +20620,7 @@ pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::dict::Dict @@ -20970,10 +20632,6 @@ pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::dict::Dict::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::dict::Dict::array_eq(array: &vortex_array::arrays::dict::DictData, other: &vortex_array::arrays::dict::DictData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::dict::Dict::array_hash(array: &vortex_array::arrays::dict::DictData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -20982,7 +20640,7 @@ pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21004,11 +20662,7 @@ pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayVie pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::null::Null @@ -21020,10 +20674,6 @@ pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::null::Null::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::null::Null::array_eq(_array: &vortex_array::arrays::null::NullData, _other: &vortex_array::arrays::null::NullData, _precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::null::Null::array_hash(_array: &vortex_array::arrays::null::NullData, _state: &mut H, _precision: vortex_array::Precision) - pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -21032,7 +20682,7 @@ pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21054,15 +20704,11 @@ pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::patched::Patched -pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray +pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedData pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched @@ -21070,10 +20716,6 @@ pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array:: pub fn vortex_array::arrays::patched::Patched::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::patched::Patched::array_eq(array: &vortex_array::arrays::patched::PatchedArray, other: &vortex_array::arrays::patched::PatchedArray, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::patched::Patched::array_hash(array: &vortex_array::arrays::patched::PatchedArray, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -21082,7 +20724,7 @@ pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayV pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21104,11 +20746,7 @@ pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::Ar pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -21120,10 +20758,6 @@ pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vorte pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_eq(array: &vortex_array::arrays::scalar_fn::ScalarFnData, other: &vortex_array::arrays::scalar_fn::ScalarFnData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_hash(array: &vortex_array::arrays::scalar_fn::ScalarFnData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -21132,7 +20766,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_arra pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21154,11 +20788,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::slice::Slice @@ -21170,10 +20800,6 @@ pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arra pub fn vortex_array::arrays::slice::Slice::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::slice::Slice::array_eq(array: &vortex_array::arrays::slice::SliceData, other: &vortex_array::arrays::slice::SliceData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::slice::Slice::array_hash(array: &vortex_array::arrays::slice::SliceData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -21182,7 +20808,7 @@ pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView< pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21204,23 +20830,19 @@ pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::Array pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> pub trait vortex_array::vtable::ValidityChild -pub fn vortex_array::vtable::ValidityChild::validity_child(array: &::ArrayData) -> &vortex_array::ArrayRef +pub fn vortex_array::vtable::ValidityChild::validity_child(array: vortex_array::ArrayView<'_, V>) -> vortex_array::ArrayRef impl vortex_array::ValidityChild for vortex_array::arrays::Extension -pub fn vortex_array::arrays::Extension::validity_child(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::Extension::validity_child(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>) -> vortex_array::ArrayRef impl vortex_array::ValidityChild for vortex_array::arrays::patched::Patched -pub fn vortex_array::arrays::patched::Patched::validity_child(array: &vortex_array::arrays::patched::PatchedArray) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::Patched::validity_child(array: vortex_array::ArrayView<'_, vortex_array::arrays::patched::Patched>) -> vortex_array::ArrayRef pub trait vortex_array::vtable::ValidityChildSliceHelper @@ -21636,8 +21258,16 @@ pub unsafe fn vortex_array::Array::new_unchecked< pub unsafe fn vortex_array::Array::new_unchecked_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::Array::patch(self, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + pub fn vortex_array::Array::try_new(buffer: vortex_buffer::buffer::Buffer, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult +pub fn vortex_array::Array::try_new_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult + +impl vortex_array::Array + +pub fn vortex_array::Array::into_data_parts(self) -> vortex_array::arrays::decimal::DecimalDataParts + impl vortex_array::Array pub fn vortex_array::Array::new(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_array: vortex_array::ArrayRef) -> Self @@ -21652,6 +21282,8 @@ pub fn vortex_array::Array::try_new(array: vortex_ impl vortex_array::Array +pub fn vortex_array::Array::into_data_parts(self) -> vortex_array::arrays::fixed_size_list::FixedSizeListDataParts + pub fn vortex_array::Array::new(elements: vortex_array::ArrayRef, list_size: u32, validity: vortex_array::validity::Validity, len: usize) -> Self pub unsafe fn vortex_array::Array::new_unchecked(elements: vortex_array::ArrayRef, list_size: u32, validity: vortex_array::validity::Validity, len: usize) -> Self @@ -21660,6 +21292,8 @@ pub fn vortex_array::Array::try_new(element impl vortex_array::Array +pub fn vortex_array::Array::into_data_parts(self) -> vortex_array::arrays::list::ListDataParts + pub fn vortex_array::Array::new(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> Self pub unsafe fn vortex_array::Array::new_unchecked(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> Self @@ -21668,6 +21302,8 @@ pub fn vortex_array::Array::try_new(elements: vortex impl vortex_array::Array +pub fn vortex_array::Array::into_data_parts(self) -> vortex_array::arrays::listview::ListViewDataParts + pub fn vortex_array::Array::new(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, sizes: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> Self pub unsafe fn vortex_array::Array::new_unchecked(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, sizes: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> Self @@ -21694,6 +21330,10 @@ pub fn vortex_array::Array::from_byte_buffer(bu pub fn vortex_array::Array::from_values_byte_buffer(valid_elems_buffer: vortex_buffer::ByteBuffer, ptype: vortex_array::dtype::PType, validity: vortex_array::validity::Validity, n_rows: usize) -> Self +pub fn vortex_array::Array::into_data_parts(self) -> vortex_array::arrays::primitive::PrimitiveDataParts + +pub fn vortex_array::Array::map_each_with_validity(self, f: F) -> vortex_error::VortexResult where T: vortex_array::dtype::NativePType, R: vortex_array::dtype::NativePType, F: core::ops::function::FnMut((T, bool)) -> R + pub fn vortex_array::Array::new(buffer: impl core::convert::Into>, validity: vortex_array::validity::Validity) -> Self pub unsafe fn vortex_array::Array::new_unchecked(buffer: vortex_buffer::buffer::Buffer, validity: vortex_array::validity::Validity) -> Self @@ -21718,12 +21358,6 @@ pub fn vortex_array::Array::try_into_buffer_mut impl vortex_array::Array -pub fn vortex_array::Array::narrow(&self) -> vortex_error::VortexResult - -pub fn vortex_array::Array::reinterpret_cast(&self, ptype: vortex_array::dtype::PType) -> Self - -impl vortex_array::Array - pub fn vortex_array::Array::patch(self, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::Array @@ -21738,6 +21372,8 @@ impl vortex_array::Array pub fn vortex_array::Array::from_fields>(items: &[(N, vortex_array::ArrayRef)]) -> vortex_error::VortexResult +pub fn vortex_array::Array::into_data_parts(self) -> vortex_array::arrays::struct_::StructDataParts + pub fn vortex_array::Array::new(names: vortex_array::dtype::FieldNames, fields: impl core::convert::Into>, length: usize, validity: vortex_array::validity::Validity) -> Self pub fn vortex_array::Array::new_fieldless_with_len(len: usize) -> Self @@ -21746,6 +21382,8 @@ pub unsafe fn vortex_array::Array::new_unchecked(f pub fn vortex_array::Array::project(&self, projection: &[vortex_array::dtype::FieldName]) -> vortex_error::VortexResult +pub fn vortex_array::Array::remove_column(&self, name: impl core::convert::Into) -> core::option::Option<(Self, vortex_array::ArrayRef)> + pub fn vortex_array::Array::try_from_iter, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T) -> vortex_error::VortexResult pub fn vortex_array::Array::try_from_iter_with_validity, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult @@ -21758,6 +21396,12 @@ impl vortex_array::Array pub fn vortex_array::Array::into_record_batch_with_schema(self, schema: impl core::convert::AsRef) -> vortex_error::VortexResult +impl vortex_array::Array + +pub fn vortex_array::Array::remove_column_owned(&self, name: impl core::convert::Into) -> core::option::Option<(Self, vortex_array::ArrayRef)> + +pub fn vortex_array::Array::with_column(&self, name: impl core::convert::Into, array: vortex_array::ArrayRef) -> vortex_error::VortexResult + impl vortex_array::Array pub fn vortex_array::Array::from_bytes(value: alloc::vec::Vec<&[u8]>) -> Self @@ -21774,6 +21418,8 @@ pub fn vortex_array::Array::from_strs(value: alloc pub fn vortex_array::Array::from_vec>(vec: alloc::vec::Vec, dtype: vortex_array::dtype::DType) -> Self +pub fn vortex_array::Array::into_data_parts(self) -> vortex_array::arrays::varbin::VarBinDataParts + impl vortex_array::Array pub fn vortex_array::Array::new(offsets: vortex_array::ArrayRef, bytes: vortex_buffer::ByteBuffer, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self @@ -21802,6 +21448,8 @@ pub fn vortex_array::Array::from_iter_nullable pub fn vortex_array::Array::from_iter_str, I: core::iter::traits::collect::IntoIterator>(iter: I) -> Self +pub fn vortex_array::Array::into_data_parts(self) -> vortex_array::arrays::varbinview::VarBinViewDataParts + pub fn vortex_array::Array::new_handle(views: vortex_array::buffer::BufferHandle, buffers: alloc::sync::Arc<[vortex_array::buffer::BufferHandle]>, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self pub unsafe fn vortex_array::Array::new_handle_unchecked(views: vortex_array::buffer::BufferHandle, buffers: alloc::sync::Arc<[vortex_array::buffer::BufferHandle]>, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self @@ -21830,10 +21478,6 @@ pub fn vortex_array::Array::new(len: usize) -> impl vortex_array::Array -pub fn vortex_array::Array::children(&self) -> alloc::vec::Vec - -pub fn vortex_array::Array::scalar_fn(&self) -> &vortex_array::scalar_fn::ScalarFnRef - pub fn vortex_array::Array::try_new(scalar_fn: vortex_array::scalar_fn::ScalarFnRef, children: alloc::vec::Vec, len: usize) -> vortex_error::VortexResult impl vortex_array::Array @@ -21892,18 +21536,20 @@ pub fn vortex_array::Array::encoding_id(&self) -> vortex_array::ArrayId pub fn vortex_array::Array::into_data(self) -> ::ArrayData -pub fn vortex_array::Array::into_parts(self) -> vortex_array::ArrayParts - pub fn vortex_array::Array::is_empty(&self) -> bool pub fn vortex_array::Array::len(&self) -> usize +pub fn vortex_array::Array::slots(&self) -> &[core::option::Option] + pub fn vortex_array::Array::statistics(&self) -> vortex_array::stats::StatsSetRef<'_> pub fn vortex_array::Array::try_from_array_ref(array: vortex_array::ArrayRef) -> core::result::Result pub fn vortex_array::Array::try_from_parts(new: vortex_array::ArrayParts) -> vortex_error::VortexResult +pub fn vortex_array::Array::try_into_parts(self) -> core::result::Result, Self> + pub fn vortex_array::Array::with_stats_set(self, stats: vortex_array::stats::StatsSet) -> Self impl core::convert::From> for vortex_array::Array @@ -22006,6 +21652,10 @@ impl vortex_array::IntoArray for vortex_array::Array pub fn vortex_array::Array::into_array(self) -> vortex_array::ArrayRef +impl vortex_array::TypedArrayRef for vortex_array::Array + +pub fn vortex_array::Array::to_owned(&self) -> vortex_array::Array + pub struct vortex_array::ArrayParts pub vortex_array::ArrayParts::data: ::ArrayData @@ -22014,12 +21664,16 @@ pub vortex_array::ArrayParts::dtype: vortex_array::dtype::DType pub vortex_array::ArrayParts::len: usize +pub vortex_array::ArrayParts::slots: alloc::vec::Vec> + pub vortex_array::ArrayParts::vtable: V impl vortex_array::ArrayParts pub fn vortex_array::ArrayParts::new(vtable: V, dtype: vortex_array::dtype::DType, len: usize, data: ::ArrayData) -> Self +pub fn vortex_array::ArrayParts::with_slots(self, slots: alloc::vec::Vec>) -> Self + pub struct vortex_array::ArrayRef(_) impl vortex_array::ArrayRef @@ -22050,6 +21704,8 @@ pub fn vortex_array::ArrayRef::children_names(&self) -> alloc::vec::Vec vortex_array::DepthFirstArrayIterator +pub fn vortex_array::ArrayRef::downcast(self) -> vortex_array::Array + pub fn vortex_array::ArrayRef::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::ArrayRef::encoding_id(&self) -> vortex_array::ArrayId @@ -22100,7 +21756,7 @@ pub fn vortex_array::ArrayRef::slice(&self, range: core::ops::range::Range alloc::string::String -pub fn vortex_array::ArrayRef::slots(&self) -> alloc::vec::Vec> +pub fn vortex_array::ArrayRef::slots(&self) -> &[core::option::Option] pub fn vortex_array::ArrayRef::statistics(&self) -> vortex_array::stats::StatsSetRef<'_> @@ -22108,7 +21764,7 @@ pub fn vortex_array::ArrayRef::take(&self, indices: vortex_array::ArrayRef) -> v pub fn vortex_array::ArrayRef::to_canonical(&self) -> vortex_error::VortexResult -pub fn vortex_array::ArrayRef::try_into(self) -> core::result::Result, vortex_array::ArrayRef> +pub fn vortex_array::ArrayRef::try_downcast(self) -> core::result::Result, vortex_array::ArrayRef> pub fn vortex_array::ArrayRef::valid_count(&self) -> vortex_error::VortexResult @@ -22200,10 +21856,6 @@ impl core::convert::From for vorte pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::datetime::TemporalData) -> Self -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> Self - impl core::convert::TryFrom<&vortex_array::ArrayRef> for arrow_array::record_batch::RecordBatch pub type arrow_array::record_batch::RecordBatch::Error = vortex_error::VortexError @@ -22506,6 +22158,8 @@ pub fn vortex_array::ArrayView<'a, V>::is_empty(&self) -> bool pub fn vortex_array::ArrayView<'a, V>::len(&self) -> usize +pub fn vortex_array::ArrayView<'a, V>::slots(&self) -> &'a [core::option::Option] + pub fn vortex_array::ArrayView<'a, V>::statistics(&self) -> vortex_array::stats::StatsSetRef<'_> pub fn vortex_array::ArrayView<'a, V>::validity(&self) -> vortex_error::VortexResult @@ -22530,6 +22184,10 @@ pub type vortex_array::ArrayView<'_, V>::Target = ::A pub fn vortex_array::ArrayView<'_, V>::deref(&self) -> &::ArrayData +impl vortex_array::TypedArrayRef for vortex_array::ArrayView<'_, V> + +pub fn vortex_array::ArrayView<'_, V>::to_owned(&self) -> vortex_array::Array + pub struct vortex_array::CanonicalValidity(pub vortex_array::Canonical) impl vortex_array::Executable for vortex_array::CanonicalValidity @@ -22650,65 +22308,149 @@ pub type vortex_array::ProstMetadata::Target = M pub fn vortex_array::ProstMetadata::deref(&self) -> &Self::Target -impl vortex_array::DeserializeMetadata for vortex_array::ProstMetadata where M: core::fmt::Debug + prost::message::Message + core::default::Default +impl vortex_array::DeserializeMetadata for vortex_array::ProstMetadata where M: core::fmt::Debug + prost::message::Message + core::default::Default + +pub type vortex_array::ProstMetadata::Output = M + +pub fn vortex_array::ProstMetadata::deserialize(metadata: &[u8]) -> vortex_error::VortexResult + +impl vortex_array::SerializeMetadata for vortex_array::ProstMetadata where M: prost::message::Message + +pub fn vortex_array::ProstMetadata::serialize(self) -> alloc::vec::Vec + +pub struct vortex_array::RawMetadata(pub alloc::vec::Vec) + +impl core::fmt::Debug for vortex_array::RawMetadata + +pub fn vortex_array::RawMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +impl vortex_array::DeserializeMetadata for vortex_array::RawMetadata + +pub type vortex_array::RawMetadata::Output = alloc::vec::Vec + +pub fn vortex_array::RawMetadata::deserialize(metadata: &[u8]) -> vortex_error::VortexResult + +impl vortex_array::SerializeMetadata for vortex_array::RawMetadata + +pub fn vortex_array::RawMetadata::serialize(self) -> alloc::vec::Vec + +pub struct vortex_array::RecursiveCanonical(pub vortex_array::Canonical) + +impl vortex_array::Executable for vortex_array::RecursiveCanonical + +pub fn vortex_array::RecursiveCanonical::execute(array: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + +pub struct vortex_array::ValidityVTableFromChild + +impl vortex_array::ValidityVTable for vortex_array::ValidityVTableFromChild where V: vortex_array::ValidityChild + vortex_array::VTable + +pub fn vortex_array::ValidityVTableFromChild::validity(array: vortex_array::ArrayView<'_, V>) -> vortex_error::VortexResult + +pub struct vortex_array::ValidityVTableFromChildSliceHelper + +impl vortex_array::ValidityVTable for vortex_array::ValidityVTableFromChildSliceHelper where ::ArrayData: vortex_array::ValidityChildSliceHelper + +pub fn vortex_array::ValidityVTableFromChildSliceHelper::validity(array: vortex_array::ArrayView<'_, V>) -> vortex_error::VortexResult + +pub struct vortex_array::ValidityVTableFromValiditySliceHelper + +impl vortex_array::ValidityVTable for vortex_array::ValidityVTableFromValiditySliceHelper where ::ArrayData: vortex_array::ValiditySliceHelper + +pub fn vortex_array::ValidityVTableFromValiditySliceHelper::validity(array: vortex_array::ArrayView<'_, V>) -> vortex_error::VortexResult + +pub static vortex_array::LEGACY_SESSION: std::sync::lazy_lock::LazyLock + +pub trait vortex_array::ArrayEq + +pub fn vortex_array::ArrayEq::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayEq for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayEq for vortex_array::arrays::bool::BoolData + +pub fn vortex_array::arrays::bool::BoolData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayEq for vortex_array::arrays::chunked::ChunkedData + +pub fn vortex_array::arrays::chunked::ChunkedData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayEq for vortex_array::arrays::constant::ConstantData + +pub fn vortex_array::arrays::constant::ConstantData::array_eq(&self, other: &Self, _precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayEq for vortex_array::arrays::decimal::DecimalData + +pub fn vortex_array::arrays::decimal::DecimalData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayEq for vortex_array::arrays::dict::DictData + +pub fn vortex_array::arrays::dict::DictData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayEq for vortex_array::arrays::extension::ExtensionData + +pub fn vortex_array::arrays::extension::ExtensionData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool + +impl vortex_array::ArrayEq for vortex_array::arrays::filter::FilterData -pub type vortex_array::ProstMetadata::Output = M +pub fn vortex_array::arrays::filter::FilterData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool -pub fn vortex_array::ProstMetadata::deserialize(metadata: &[u8]) -> vortex_error::VortexResult +impl vortex_array::ArrayEq for vortex_array::arrays::fixed_size_list::FixedSizeListData -impl vortex_array::SerializeMetadata for vortex_array::ProstMetadata where M: prost::message::Message +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool -pub fn vortex_array::ProstMetadata::serialize(self) -> alloc::vec::Vec +impl vortex_array::ArrayEq for vortex_array::arrays::list::ListData -pub struct vortex_array::RawMetadata(pub alloc::vec::Vec) +pub fn vortex_array::arrays::list::ListData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -impl core::fmt::Debug for vortex_array::RawMetadata +impl vortex_array::ArrayEq for vortex_array::arrays::listview::ListViewData -pub fn vortex_array::RawMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::arrays::listview::ListViewData::array_eq(&self, other: &Self, _precision: vortex_array::Precision) -> bool -impl vortex_array::DeserializeMetadata for vortex_array::RawMetadata +impl vortex_array::ArrayEq for vortex_array::arrays::masked::MaskedData -pub type vortex_array::RawMetadata::Output = alloc::vec::Vec +pub fn vortex_array::arrays::masked::MaskedData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -pub fn vortex_array::RawMetadata::deserialize(metadata: &[u8]) -> vortex_error::VortexResult +impl vortex_array::ArrayEq for vortex_array::arrays::null::NullData -impl vortex_array::SerializeMetadata for vortex_array::RawMetadata +pub fn vortex_array::arrays::null::NullData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -pub fn vortex_array::RawMetadata::serialize(self) -> alloc::vec::Vec +impl vortex_array::ArrayEq for vortex_array::arrays::patched::PatchedData -pub struct vortex_array::RecursiveCanonical(pub vortex_array::Canonical) +pub fn vortex_array::arrays::patched::PatchedData::array_eq(&self, other: &Self, _precision: vortex_array::Precision) -> bool -impl vortex_array::Executable for vortex_array::RecursiveCanonical +impl vortex_array::ArrayEq for vortex_array::arrays::primitive::PrimitiveData -pub fn vortex_array::RecursiveCanonical::execute(array: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::primitive::PrimitiveData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool -pub struct vortex_array::ValidityVTableFromChild +impl vortex_array::ArrayEq for vortex_array::arrays::scalar_fn::ScalarFnData -impl vortex_array::ValidityVTable for vortex_array::ValidityVTableFromChild where V: vortex_array::ValidityChild + vortex_array::VTable +pub fn vortex_array::arrays::scalar_fn::ScalarFnData::array_eq(&self, other: &Self, _precision: vortex_array::Precision) -> bool -pub fn vortex_array::ValidityVTableFromChild::validity(array: vortex_array::ArrayView<'_, V>) -> vortex_error::VortexResult +impl vortex_array::ArrayEq for vortex_array::arrays::shared::SharedData -pub struct vortex_array::ValidityVTableFromChildSliceHelper +pub fn vortex_array::arrays::shared::SharedData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -impl vortex_array::ValidityVTable for vortex_array::ValidityVTableFromChildSliceHelper where ::ArrayData: vortex_array::ValidityChildSliceHelper +impl vortex_array::ArrayEq for vortex_array::arrays::slice::SliceData -pub fn vortex_array::ValidityVTableFromChildSliceHelper::validity(array: vortex_array::ArrayView<'_, V>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::SliceData::array_eq(&self, other: &Self, _precision: vortex_array::Precision) -> bool -pub struct vortex_array::ValidityVTableFromValiditySliceHelper +impl vortex_array::ArrayEq for vortex_array::arrays::struct_::StructData -impl vortex_array::ValidityVTable for vortex_array::ValidityVTableFromValiditySliceHelper where ::ArrayData: vortex_array::ValiditySliceHelper +pub fn vortex_array::arrays::struct_::StructData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -pub fn vortex_array::ValidityVTableFromValiditySliceHelper::validity(array: vortex_array::ArrayView<'_, V>) -> vortex_error::VortexResult +impl vortex_array::ArrayEq for vortex_array::arrays::varbin::VarBinData -pub static vortex_array::LEGACY_SESSION: std::sync::lazy_lock::LazyLock +pub fn vortex_array::arrays::varbin::VarBinData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool -pub trait vortex_array::ArrayEq +impl vortex_array::ArrayEq for vortex_array::arrays::varbinview::VarBinViewData -pub fn vortex_array::ArrayEq::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool +pub fn vortex_array::arrays::varbinview::VarBinViewData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool -impl vortex_array::ArrayEq for vortex_array::ArrayRef +impl vortex_array::ArrayEq for vortex_array::arrays::variant::VariantData -pub fn vortex_array::ArrayRef::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool +pub fn vortex_array::arrays::variant::VariantData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool impl vortex_array::ArrayEq for vortex_array::buffer::BufferHandle @@ -22746,6 +22488,90 @@ impl vortex_array::ArrayHash for vortex_array::ArrayRef pub fn vortex_array::ArrayRef::array_hash(&self, state: &mut H, precision: vortex_array::Precision) +impl vortex_array::ArrayHash for vortex_array::arrays::bool::BoolData + +pub fn vortex_array::arrays::bool::BoolData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::chunked::ChunkedData + +pub fn vortex_array::arrays::chunked::ChunkedData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::constant::ConstantData + +pub fn vortex_array::arrays::constant::ConstantData::array_hash(&self, state: &mut H, _precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::decimal::DecimalData + +pub fn vortex_array::arrays::decimal::DecimalData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::dict::DictData + +pub fn vortex_array::arrays::dict::DictData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::extension::ExtensionData + +pub fn vortex_array::arrays::extension::ExtensionData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::filter::FilterData + +pub fn vortex_array::arrays::filter::FilterData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::fixed_size_list::FixedSizeListData + +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::list::ListData + +pub fn vortex_array::arrays::list::ListData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::listview::ListViewData + +pub fn vortex_array::arrays::listview::ListViewData::array_hash(&self, state: &mut H, _precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::masked::MaskedData + +pub fn vortex_array::arrays::masked::MaskedData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::null::NullData + +pub fn vortex_array::arrays::null::NullData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::patched::PatchedData + +pub fn vortex_array::arrays::patched::PatchedData::array_hash(&self, state: &mut H, _precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::primitive::PrimitiveData + +pub fn vortex_array::arrays::primitive::PrimitiveData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::scalar_fn::ScalarFnData + +pub fn vortex_array::arrays::scalar_fn::ScalarFnData::array_hash(&self, state: &mut H, _precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::shared::SharedData + +pub fn vortex_array::arrays::shared::SharedData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::slice::SliceData + +pub fn vortex_array::arrays::slice::SliceData::array_hash(&self, state: &mut H, _precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::struct_::StructData + +pub fn vortex_array::arrays::struct_::StructData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::varbin::VarBinData + +pub fn vortex_array::arrays::varbin::VarBinData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::varbinview::VarBinViewData + +pub fn vortex_array::arrays::varbinview::VarBinViewData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) + +impl vortex_array::ArrayHash for vortex_array::arrays::variant::VariantData + +pub fn vortex_array::arrays::variant::VariantData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) + impl vortex_array::ArrayHash for vortex_array::buffer::BufferHandle pub fn vortex_array::buffer::BufferHandle::array_hash(&self, state: &mut H, precision: vortex_array::Precision) @@ -22776,7 +22602,7 @@ pub fn core::option::Option::array_hash(&self, state: pub trait vortex_array::ArrayVTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug +pub type vortex_array::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::ArrayHash + vortex_array::ArrayEq pub type vortex_array::ArrayVTable::OperationsVTable: vortex_array::OperationsVTable @@ -22784,10 +22610,6 @@ pub type vortex_array::ArrayVTable::ValidityVTable: vortex_array::ValidityVTable pub fn vortex_array::ArrayVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::ArrayVTable::array_eq(array: &Self::ArrayData, other: &Self::ArrayData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::ArrayVTable::array_hash(array: &Self::ArrayData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::ArrayVTable::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::ArrayVTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -22796,7 +22618,7 @@ pub fn vortex_array::ArrayVTable::child(array: vortex_array::ArrayView<'_, Self> pub fn vortex_array::ArrayVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::ArrayVTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::ArrayVTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::ArrayVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -22818,11 +22640,7 @@ pub fn vortex_array::ArrayVTable::serialize(array: vortex_array::ArrayView<'_, S pub fn vortex_array::ArrayVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::ArrayVTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] - -pub fn vortex_array::ArrayVTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::ArrayVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::ArrayVTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Bool @@ -22834,10 +22652,6 @@ pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool pub fn vortex_array::arrays::Bool::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Bool::array_eq(array: &vortex_array::arrays::bool::BoolData, other: &vortex_array::arrays::bool::BoolData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Bool::array_hash(array: &vortex_array::arrays::bool::BoolData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -22846,7 +22660,7 @@ pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -22868,11 +22682,7 @@ pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Chunked @@ -22884,10 +22694,6 @@ pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::C pub fn vortex_array::arrays::Chunked::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Chunked::array_eq(array: &vortex_array::arrays::chunked::ChunkedData, other: &vortex_array::arrays::chunked::ChunkedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Chunked::array_hash(array: &vortex_array::arrays::chunked::ChunkedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -22896,7 +22702,7 @@ pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -22918,11 +22724,7 @@ pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Constant @@ -22934,10 +22736,6 @@ pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays:: pub fn vortex_array::arrays::Constant::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Constant::array_eq(array: &vortex_array::arrays::constant::ConstantData, other: &vortex_array::arrays::constant::ConstantData, _precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Constant::array_hash(array: &vortex_array::arrays::constant::ConstantData, state: &mut H, _precision: vortex_array::Precision) - pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -22946,7 +22744,7 @@ pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -22968,11 +22766,7 @@ pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Decimal @@ -22984,10 +22778,6 @@ pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::D pub fn vortex_array::arrays::Decimal::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Decimal::array_eq(array: &vortex_array::arrays::decimal::DecimalData, other: &vortex_array::arrays::decimal::DecimalData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Decimal::array_hash(array: &vortex_array::arrays::decimal::DecimalData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -22996,7 +22786,7 @@ pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23018,11 +22808,7 @@ pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Extension @@ -23034,10 +22820,6 @@ pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::Validit pub fn vortex_array::arrays::Extension::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Extension::array_eq(array: &vortex_array::arrays::extension::ExtensionData, other: &vortex_array::arrays::extension::ExtensionData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Extension::array_hash(array: &vortex_array::arrays::extension::ExtensionData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -23046,7 +22828,7 @@ pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23068,11 +22850,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Filter @@ -23084,10 +22862,6 @@ pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Fi pub fn vortex_array::arrays::Filter::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Filter::array_eq(array: &vortex_array::arrays::filter::FilterData, other: &vortex_array::arrays::filter::FilterData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Filter::array_hash(array: &vortex_array::arrays::filter::FilterData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -23096,7 +22870,7 @@ pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23118,11 +22892,7 @@ pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::FixedSizeList @@ -23134,10 +22904,6 @@ pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arr pub fn vortex_array::arrays::FixedSizeList::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::FixedSizeList::array_eq(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData, other: &vortex_array::arrays::fixed_size_list::FixedSizeListData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::FixedSizeList::array_hash(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -23146,7 +22912,7 @@ pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23168,11 +22934,7 @@ pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::Arra pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::List @@ -23184,10 +22946,6 @@ pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List pub fn vortex_array::arrays::List::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::List::array_eq(array: &vortex_array::arrays::list::ListData, other: &vortex_array::arrays::list::ListData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::List::array_hash(array: &vortex_array::arrays::list::ListData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -23196,7 +22954,7 @@ pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23218,11 +22976,7 @@ pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::validate(&self, _data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::ListView @@ -23234,10 +22988,6 @@ pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays:: pub fn vortex_array::arrays::ListView::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::ListView::array_eq(array: &vortex_array::arrays::listview::ListViewData, other: &vortex_array::arrays::listview::ListViewData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::ListView::array_hash(array: &vortex_array::arrays::listview::ListViewData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -23246,7 +22996,7 @@ pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23268,11 +23018,7 @@ pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView< pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::validate(&self, _data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Masked @@ -23284,10 +23030,6 @@ pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Ma pub fn vortex_array::arrays::Masked::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Masked::array_eq(array: &vortex_array::arrays::masked::MaskedData, other: &vortex_array::arrays::masked::MaskedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Masked::array_hash(array: &vortex_array::arrays::masked::MaskedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -23296,7 +23038,7 @@ pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23318,11 +23060,7 @@ pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::validate(&self, _data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Primitive @@ -23334,10 +23072,6 @@ pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays: pub fn vortex_array::arrays::Primitive::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Primitive::array_eq(array: &vortex_array::arrays::primitive::PrimitiveData, other: &vortex_array::arrays::primitive::PrimitiveData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Primitive::array_hash(array: &vortex_array::arrays::primitive::PrimitiveData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -23346,7 +23080,7 @@ pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23368,11 +23102,7 @@ pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Shared @@ -23384,10 +23114,6 @@ pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Sh pub fn vortex_array::arrays::Shared::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Shared::array_eq(array: &vortex_array::arrays::shared::SharedData, other: &vortex_array::arrays::shared::SharedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Shared::array_hash(array: &vortex_array::arrays::shared::SharedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -23396,7 +23122,7 @@ pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23418,11 +23144,7 @@ pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::validate(&self, _data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Struct @@ -23434,10 +23156,6 @@ pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::St pub fn vortex_array::arrays::Struct::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Struct::array_eq(array: &vortex_array::arrays::struct_::StructData, other: &vortex_array::arrays::struct_::StructData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Struct::array_hash(array: &vortex_array::arrays::struct_::StructData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -23446,7 +23164,7 @@ pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23468,11 +23186,7 @@ pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::VarBin @@ -23484,10 +23198,6 @@ pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::Va pub fn vortex_array::arrays::VarBin::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::VarBin::array_eq(array: &vortex_array::arrays::varbin::VarBinData, other: &vortex_array::arrays::varbin::VarBinData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::VarBin::array_hash(array: &vortex_array::arrays::varbin::VarBinData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -23496,7 +23206,7 @@ pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23518,11 +23228,7 @@ pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::validate(&self, _data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::VarBinView @@ -23534,10 +23240,6 @@ pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::VarBinView::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::VarBinView::array_eq(array: &vortex_array::arrays::varbinview::VarBinViewData, other: &vortex_array::arrays::varbinview::VarBinViewData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::VarBinView::array_hash(array: &vortex_array::arrays::varbinview::VarBinViewData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -23546,7 +23248,7 @@ pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23568,11 +23270,7 @@ pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Variant @@ -23584,10 +23282,6 @@ pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::V pub fn vortex_array::arrays::Variant::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Variant::array_eq(array: &vortex_array::arrays::variant::VariantData, other: &vortex_array::arrays::variant::VariantData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Variant::array_hash(array: &vortex_array::arrays::variant::VariantData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -23596,7 +23290,7 @@ pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23618,11 +23312,7 @@ pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::dict::Dict @@ -23634,10 +23324,6 @@ pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::dict::Dict::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::dict::Dict::array_eq(array: &vortex_array::arrays::dict::DictData, other: &vortex_array::arrays::dict::DictData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::dict::Dict::array_hash(array: &vortex_array::arrays::dict::DictData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -23646,7 +23332,7 @@ pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23668,11 +23354,7 @@ pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayVie pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::null::Null @@ -23684,10 +23366,6 @@ pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::null::Null::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::null::Null::array_eq(_array: &vortex_array::arrays::null::NullData, _other: &vortex_array::arrays::null::NullData, _precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::null::Null::array_hash(_array: &vortex_array::arrays::null::NullData, _state: &mut H, _precision: vortex_array::Precision) - pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -23696,7 +23374,7 @@ pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23718,15 +23396,11 @@ pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::patched::Patched -pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray +pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedData pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched @@ -23734,10 +23408,6 @@ pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array:: pub fn vortex_array::arrays::patched::Patched::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::patched::Patched::array_eq(array: &vortex_array::arrays::patched::PatchedArray, other: &vortex_array::arrays::patched::PatchedArray, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::patched::Patched::array_hash(array: &vortex_array::arrays::patched::PatchedArray, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -23746,7 +23416,7 @@ pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayV pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23768,11 +23438,7 @@ pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::Ar pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -23784,10 +23450,6 @@ pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vorte pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_eq(array: &vortex_array::arrays::scalar_fn::ScalarFnData, other: &vortex_array::arrays::scalar_fn::ScalarFnData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_hash(array: &vortex_array::arrays::scalar_fn::ScalarFnData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -23796,7 +23458,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_arra pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23818,11 +23480,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::slice::Slice @@ -23834,10 +23492,6 @@ pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arra pub fn vortex_array::arrays::slice::Slice::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::slice::Slice::array_eq(array: &vortex_array::arrays::slice::SliceData, other: &vortex_array::arrays::slice::SliceData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::slice::Slice::array_hash(array: &vortex_array::arrays::slice::SliceData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -23846,7 +23500,7 @@ pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView< pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23868,11 +23522,7 @@ pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::Array pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> pub trait vortex_array::DeserializeMetadata where Self: core::marker::Sized @@ -24050,10 +23700,6 @@ impl vortex_array::IntoArray for vortex_array::arrays::datetime::TemporalData pub fn vortex_array::arrays::datetime::TemporalData::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::IntoArray for vortex_array::arrays::patched::PatchedArray - -pub fn vortex_array::arrays::patched::PatchedArray::into_array(self) -> vortex_array::ArrayRef - impl vortex_array::IntoArray for vortex_buffer::bit::buf::BitBuffer pub fn vortex_buffer::bit::buf::BitBuffer::into_array(self) -> vortex_array::ArrayRef @@ -24238,9 +23884,21 @@ pub fn vortex_array::ArrayRef::to_struct(&self) -> vortex_array::arrays::StructA pub fn vortex_array::ArrayRef::to_varbinview(&self) -> vortex_array::arrays::VarBinViewArray +pub trait vortex_array::TypedArrayRef: core::convert::AsRef + core::ops::deref::Deref::ArrayData> + +pub fn vortex_array::TypedArrayRef::to_owned(&self) -> vortex_array::Array + +impl vortex_array::TypedArrayRef for vortex_array::Array + +pub fn vortex_array::Array::to_owned(&self) -> vortex_array::Array + +impl vortex_array::TypedArrayRef for vortex_array::ArrayView<'_, V> + +pub fn vortex_array::ArrayView<'_, V>::to_owned(&self) -> vortex_array::Array + pub trait vortex_array::VTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug +pub type vortex_array::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::ArrayHash + vortex_array::ArrayEq pub type vortex_array::VTable::OperationsVTable: vortex_array::OperationsVTable @@ -24248,10 +23906,6 @@ pub type vortex_array::VTable::ValidityVTable: vortex_array::ValidityVTable, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::VTable::array_eq(array: &Self::ArrayData, other: &Self::ArrayData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::VTable::array_hash(array: &Self::ArrayData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::VTable::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::VTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -24260,7 +23914,7 @@ pub fn vortex_array::VTable::child(array: vortex_array::ArrayView<'_, Self>, idx pub fn vortex_array::VTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::VTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::VTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::VTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24282,11 +23936,7 @@ pub fn vortex_array::VTable::serialize(array: vortex_array::ArrayView<'_, Self>) pub fn vortex_array::VTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::VTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] - -pub fn vortex_array::VTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::VTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::VTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Bool @@ -24298,10 +23948,6 @@ pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool pub fn vortex_array::arrays::Bool::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Bool::array_eq(array: &vortex_array::arrays::bool::BoolData, other: &vortex_array::arrays::bool::BoolData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Bool::array_hash(array: &vortex_array::arrays::bool::BoolData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -24310,7 +23956,7 @@ pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24332,11 +23978,7 @@ pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Chunked @@ -24348,10 +23990,6 @@ pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::C pub fn vortex_array::arrays::Chunked::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Chunked::array_eq(array: &vortex_array::arrays::chunked::ChunkedData, other: &vortex_array::arrays::chunked::ChunkedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Chunked::array_hash(array: &vortex_array::arrays::chunked::ChunkedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -24360,7 +23998,7 @@ pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24382,11 +24020,7 @@ pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Constant @@ -24398,10 +24032,6 @@ pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays:: pub fn vortex_array::arrays::Constant::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Constant::array_eq(array: &vortex_array::arrays::constant::ConstantData, other: &vortex_array::arrays::constant::ConstantData, _precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Constant::array_hash(array: &vortex_array::arrays::constant::ConstantData, state: &mut H, _precision: vortex_array::Precision) - pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -24410,7 +24040,7 @@ pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24432,11 +24062,7 @@ pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Decimal @@ -24448,10 +24074,6 @@ pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::D pub fn vortex_array::arrays::Decimal::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Decimal::array_eq(array: &vortex_array::arrays::decimal::DecimalData, other: &vortex_array::arrays::decimal::DecimalData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Decimal::array_hash(array: &vortex_array::arrays::decimal::DecimalData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -24460,7 +24082,7 @@ pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24482,11 +24104,7 @@ pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Extension @@ -24498,10 +24116,6 @@ pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::Validit pub fn vortex_array::arrays::Extension::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Extension::array_eq(array: &vortex_array::arrays::extension::ExtensionData, other: &vortex_array::arrays::extension::ExtensionData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Extension::array_hash(array: &vortex_array::arrays::extension::ExtensionData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -24510,7 +24124,7 @@ pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24532,11 +24146,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Filter @@ -24548,10 +24158,6 @@ pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Fi pub fn vortex_array::arrays::Filter::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Filter::array_eq(array: &vortex_array::arrays::filter::FilterData, other: &vortex_array::arrays::filter::FilterData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Filter::array_hash(array: &vortex_array::arrays::filter::FilterData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -24560,7 +24166,7 @@ pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24582,11 +24188,7 @@ pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::FixedSizeList @@ -24598,10 +24200,6 @@ pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arr pub fn vortex_array::arrays::FixedSizeList::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::FixedSizeList::array_eq(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData, other: &vortex_array::arrays::fixed_size_list::FixedSizeListData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::FixedSizeList::array_hash(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -24610,7 +24208,7 @@ pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24632,11 +24230,7 @@ pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::Arra pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::List @@ -24648,10 +24242,6 @@ pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List pub fn vortex_array::arrays::List::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::List::array_eq(array: &vortex_array::arrays::list::ListData, other: &vortex_array::arrays::list::ListData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::List::array_hash(array: &vortex_array::arrays::list::ListData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -24660,7 +24250,7 @@ pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24682,11 +24272,7 @@ pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::validate(&self, _data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::ListView @@ -24698,10 +24284,6 @@ pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays:: pub fn vortex_array::arrays::ListView::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::ListView::array_eq(array: &vortex_array::arrays::listview::ListViewData, other: &vortex_array::arrays::listview::ListViewData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::ListView::array_hash(array: &vortex_array::arrays::listview::ListViewData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -24710,7 +24292,7 @@ pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24732,11 +24314,7 @@ pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView< pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::validate(&self, _data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Masked @@ -24748,10 +24326,6 @@ pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Ma pub fn vortex_array::arrays::Masked::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Masked::array_eq(array: &vortex_array::arrays::masked::MaskedData, other: &vortex_array::arrays::masked::MaskedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Masked::array_hash(array: &vortex_array::arrays::masked::MaskedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -24760,7 +24334,7 @@ pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24782,11 +24356,7 @@ pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::validate(&self, _data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Primitive @@ -24798,10 +24368,6 @@ pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays: pub fn vortex_array::arrays::Primitive::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Primitive::array_eq(array: &vortex_array::arrays::primitive::PrimitiveData, other: &vortex_array::arrays::primitive::PrimitiveData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Primitive::array_hash(array: &vortex_array::arrays::primitive::PrimitiveData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -24810,7 +24376,7 @@ pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24832,11 +24398,7 @@ pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Shared @@ -24848,10 +24410,6 @@ pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Sh pub fn vortex_array::arrays::Shared::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Shared::array_eq(array: &vortex_array::arrays::shared::SharedData, other: &vortex_array::arrays::shared::SharedData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Shared::array_hash(array: &vortex_array::arrays::shared::SharedData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -24860,7 +24418,7 @@ pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24882,11 +24440,7 @@ pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::validate(&self, _data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Struct @@ -24898,10 +24452,6 @@ pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::St pub fn vortex_array::arrays::Struct::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Struct::array_eq(array: &vortex_array::arrays::struct_::StructData, other: &vortex_array::arrays::struct_::StructData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Struct::array_hash(array: &vortex_array::arrays::struct_::StructData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -24910,7 +24460,7 @@ pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24932,11 +24482,7 @@ pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::VarBin @@ -24948,10 +24494,6 @@ pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::Va pub fn vortex_array::arrays::VarBin::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::VarBin::array_eq(array: &vortex_array::arrays::varbin::VarBinData, other: &vortex_array::arrays::varbin::VarBinData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::VarBin::array_hash(array: &vortex_array::arrays::varbin::VarBinData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -24960,7 +24502,7 @@ pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24982,11 +24524,7 @@ pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::validate(&self, _data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::VarBinView @@ -24998,10 +24536,6 @@ pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::VarBinView::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::VarBinView::array_eq(array: &vortex_array::arrays::varbinview::VarBinViewData, other: &vortex_array::arrays::varbinview::VarBinViewData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::VarBinView::array_hash(array: &vortex_array::arrays::varbinview::VarBinViewData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -25010,7 +24544,7 @@ pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25032,11 +24566,7 @@ pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Variant @@ -25048,10 +24578,6 @@ pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::V pub fn vortex_array::arrays::Variant::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::Variant::array_eq(array: &vortex_array::arrays::variant::VariantData, other: &vortex_array::arrays::variant::VariantData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::Variant::array_hash(array: &vortex_array::arrays::variant::VariantData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -25060,7 +24586,7 @@ pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25082,11 +24608,7 @@ pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::dict::Dict @@ -25098,10 +24620,6 @@ pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::dict::Dict::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::dict::Dict::array_eq(array: &vortex_array::arrays::dict::DictData, other: &vortex_array::arrays::dict::DictData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::dict::Dict::array_hash(array: &vortex_array::arrays::dict::DictData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -25110,7 +24628,7 @@ pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25132,11 +24650,7 @@ pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayVie pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::null::Null @@ -25148,10 +24662,6 @@ pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays pub fn vortex_array::arrays::null::Null::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::null::Null::array_eq(_array: &vortex_array::arrays::null::NullData, _other: &vortex_array::arrays::null::NullData, _precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::null::Null::array_hash(_array: &vortex_array::arrays::null::NullData, _state: &mut H, _precision: vortex_array::Precision) - pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -25160,7 +24670,7 @@ pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25182,15 +24692,11 @@ pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize, _slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::patched::Patched -pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray +pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedData pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched @@ -25198,10 +24704,6 @@ pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array:: pub fn vortex_array::arrays::patched::Patched::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::patched::Patched::array_eq(array: &vortex_array::arrays::patched::PatchedArray, other: &vortex_array::arrays::patched::PatchedArray, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::patched::Patched::array_hash(array: &vortex_array::arrays::patched::PatchedArray, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option @@ -25210,7 +24712,7 @@ pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayV pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25232,11 +24734,7 @@ pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::Ar pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -25248,10 +24746,6 @@ pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vorte pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_eq(array: &vortex_array::arrays::scalar_fn::ScalarFnData, other: &vortex_array::arrays::scalar_fn::ScalarFnData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::array_hash(array: &vortex_array::arrays::scalar_fn::ScalarFnData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -25260,7 +24754,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_arra pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25282,11 +24776,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::slice::Slice @@ -25298,10 +24788,6 @@ pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arra pub fn vortex_array::arrays::slice::Slice::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::slice::Slice::array_eq(array: &vortex_array::arrays::slice::SliceData, other: &vortex_array::arrays::slice::SliceData, precision: vortex_array::Precision) -> bool - -pub fn vortex_array::arrays::slice::Slice::array_hash(array: &vortex_array::arrays::slice::SliceData, state: &mut H, precision: vortex_array::Precision) - pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option @@ -25310,7 +24796,7 @@ pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView< pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25332,23 +24818,19 @@ pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::Array pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> pub trait vortex_array::ValidityChild -pub fn vortex_array::ValidityChild::validity_child(array: &::ArrayData) -> &vortex_array::ArrayRef +pub fn vortex_array::ValidityChild::validity_child(array: vortex_array::ArrayView<'_, V>) -> vortex_array::ArrayRef impl vortex_array::ValidityChild for vortex_array::arrays::Extension -pub fn vortex_array::arrays::Extension::validity_child(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::Extension::validity_child(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>) -> vortex_array::ArrayRef impl vortex_array::ValidityChild for vortex_array::arrays::patched::Patched -pub fn vortex_array::arrays::patched::Patched::validity_child(array: &vortex_array::arrays::patched::PatchedArray) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::Patched::validity_child(array: vortex_array::ArrayView<'_, vortex_array::arrays::patched::Patched>) -> vortex_array::ArrayRef pub trait vortex_array::ValidityChildSliceHelper diff --git a/vortex-array/src/aggregate_fn/accumulator_grouped.rs b/vortex-array/src/aggregate_fn/accumulator_grouped.rs index c7d8558b4cd..24f8f0157ec 100644 --- a/vortex-array/src/aggregate_fn/accumulator_grouped.rs +++ b/vortex-array/src/aggregate_fn/accumulator_grouped.rs @@ -26,6 +26,8 @@ use crate::aggregate_fn::session::AggregateFnSessionExt; use crate::arrays::ChunkedArray; use crate::arrays::FixedSizeListArray; use crate::arrays::ListViewArray; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; +use crate::arrays::listview::ListViewArrayExt; use crate::builders::builder_with_capacity; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/bool.rs b/vortex-array/src/aggregate_fn/fns/is_constant/bool.rs index 9992cb6bbe8..c63025ccd7c 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/bool.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; pub(super) fn check_bool_constant(array: &BoolArray) -> bool { let true_count = array.to_bit_buffer().true_count(); diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/extension.rs b/vortex-array/src/aggregate_fn/fns/is_constant/extension.rs index 3826e1e0f0e..fa9393ede64 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/extension.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/extension.rs @@ -6,6 +6,7 @@ use vortex_error::VortexResult; use super::is_constant; use crate::ExecutionCtx; use crate::arrays::ExtensionArray; +use crate::arrays::extension::ExtensionArrayExt; /// Check if an extension array is constant by delegating to its storage array. pub(super) fn check_extension_constant( diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/fixed_size_list.rs b/vortex-array/src/aggregate_fn/fns/is_constant/fixed_size_list.rs index 74467d35672..19ca54def21 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/fixed_size_list.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/fixed_size_list.rs @@ -6,6 +6,7 @@ use vortex_error::VortexResult; use super::arrays_value_equal; use crate::ExecutionCtx; use crate::arrays::FixedSizeListArray; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; /// Check if a fixed-size list array is constant by comparing each list's elements. /// diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/list.rs b/vortex-array/src/aggregate_fn/fns/is_constant/list.rs index b6455ca7194..533ffa6edf8 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/list.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/list.rs @@ -6,6 +6,7 @@ use vortex_error::VortexResult; use super::arrays_value_equal; use crate::ExecutionCtx; use crate::arrays::ListViewArray; +use crate::arrays::listview::ListViewArrayExt; /// Check if a list view array is constant by comparing each list's elements. /// diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/struct_.rs b/vortex-array/src/aggregate_fn/fns/is_constant/struct_.rs index 590ae60d1fe..9376115521f 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/struct_.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/struct_.rs @@ -6,6 +6,7 @@ use vortex_error::VortexResult; use super::is_constant; use crate::ExecutionCtx; use crate::arrays::StructArray; +use crate::arrays::struct_::StructArrayExt; /// Check if a struct array is constant by checking each field independently. pub(super) fn check_struct_constant(s: &StructArray, ctx: &mut ExecutionCtx) -> VortexResult { diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs index ab515e713d9..c1efdc91563 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs @@ -6,6 +6,7 @@ use vortex_mask::Mask; use super::IsSortedIteratorExt; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; pub(super) fn check_bool_sorted(array: &BoolArray, strict: bool) -> VortexResult { match array.validity_mask()? { diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/extension.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/extension.rs index 2839cb5756f..dc69b7c8bf9 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/extension.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/extension.rs @@ -7,6 +7,7 @@ use crate::ExecutionCtx; use crate::aggregate_fn::fns::is_sorted::is_sorted; use crate::aggregate_fn::fns::is_sorted::is_strict_sorted; use crate::arrays::ExtensionArray; +use crate::arrays::extension::ExtensionArrayExt; pub(super) fn check_extension_sorted( array: &ExtensionArray, diff --git a/vortex-array/src/aggregate_fn/fns/min_max/bool.rs b/vortex-array/src/aggregate_fn/fns/min_max/bool.rs index 6a118788d4e..557598489e6 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/bool.rs @@ -9,6 +9,7 @@ use vortex_mask::Mask; use super::MinMaxPartial; use super::MinMaxResult; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; use crate::dtype::Nullability::NonNullable; use crate::scalar::Scalar; diff --git a/vortex-array/src/aggregate_fn/fns/min_max/extension.rs b/vortex-array/src/aggregate_fn/fns/min_max/extension.rs index 022de2d9869..60962dd008e 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/extension.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/extension.rs @@ -8,6 +8,7 @@ use super::MinMaxResult; use super::min_max; use crate::ExecutionCtx; use crate::arrays::ExtensionArray; +use crate::arrays::extension::ExtensionArrayExt; use crate::dtype::Nullability; use crate::scalar::Scalar; diff --git a/vortex-array/src/aggregate_fn/fns/sum/bool.rs b/vortex-array/src/aggregate_fn/fns/sum/bool.rs index a2dad06fff6..f80023e0b57 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/bool.rs @@ -10,6 +10,7 @@ use vortex_mask::AllOr; use super::SumState; use super::checked_add_u64; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; pub(super) fn accumulate_bool(inner: &mut SumState, b: &BoolArray) -> VortexResult { let SumState::Unsigned(acc) = inner else { diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs index bdb67697c3a..a173969cd2f 100644 --- a/vortex-array/src/array/erased.rs +++ b/vortex-array/src/array/erased.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::any::type_name; use std::fmt::Debug; use std::fmt::Formatter; use std::hash::Hasher; @@ -12,6 +13,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; +use vortex_error::vortex_panic; use vortex_mask::Mask; use crate::AnyCanonical; @@ -40,6 +42,7 @@ use crate::arrays::Primitive; use crate::arrays::SliceArray; use crate::arrays::VarBin; use crate::arrays::VarBinView; +use crate::arrays::bool::BoolArrayExt; use crate::buffer::BufferHandle; use crate::builders::ArrayBuilder; use crate::dtype::DType; @@ -93,6 +96,12 @@ impl ArrayRef { &self.0 } + /// Consumes the array reference, returning the owned backing allocation. + #[inline(always)] + pub(crate) fn into_inner(self) -> Arc { + self.0 + } + /// Returns true if the two ArrayRefs point to the same allocation. pub fn ptr_eq(this: &ArrayRef, other: &ArrayRef) -> bool { Arc::ptr_eq(&this.0, &other.0) @@ -113,7 +122,7 @@ impl ArrayHash for ArrayRef { impl ArrayEq for ArrayRef { fn array_eq(&self, other: &Self, precision: crate::Precision) -> bool { - self.0.dyn_array_eq(other.0.as_any(), precision) + self.0.dyn_array_eq(other, precision) } } @@ -343,10 +352,20 @@ impl ArrayRef { } /// Returns the array downcast to the given `Array` as an owned typed handle. - pub fn try_into(self) -> Result, ArrayRef> { + pub fn try_downcast(self) -> Result, ArrayRef> { Array::::try_from_array_ref(self) } + /// Returns the array downcast to the given `Array` as an owned typed handle. + /// + /// # Panics + /// + /// Panics if the array is not of the given type. + pub fn downcast(self) -> Array { + Self::try_downcast(self) + .unwrap_or_else(|_| vortex_panic!("Failed to downcast to {}", type_name::())) + } + /// Returns a reference to the typed `ArrayInner` if this array matches the given vtable type. pub fn as_typed(&self) -> Option> { let inner = self.0.as_any().downcast_ref::>()?; @@ -385,16 +404,37 @@ impl ArrayRef { /// Returns a new array with the slot at `slot_idx` replaced by `replacement`. /// + /// This is only valid for physical rewrites: the replacement must have the same logical + /// `DType` and `len` as the existing slot. + /// /// Takes ownership to allow in-place mutation when the refcount is 1. pub fn with_slot(self, slot_idx: usize, replacement: ArrayRef) -> VortexResult { - let nslots = self.slots().len(); + let slots = self.slots().to_vec(); + let nslots = slots.len(); vortex_ensure!( slot_idx < nslots, "slot index {} out of bounds for array with {} slots", slot_idx, nslots ); - let mut slots = self.slots().to_vec(); + let existing = slots[slot_idx] + .as_ref() + .vortex_expect("with_slot cannot replace an absent slot"); + vortex_ensure!( + existing.dtype() == replacement.dtype(), + "slot {} dtype changed from {} to {} during physical rewrite", + slot_idx, + existing.dtype(), + replacement.dtype() + ); + vortex_ensure!( + existing.len() == replacement.len(), + "slot {} len changed from {} to {} during physical rewrite", + slot_idx, + existing.len(), + replacement.len() + ); + let mut slots = slots; slots[slot_idx] = Some(replacement); let vtable = self.vtable().clone_boxed(); vtable.with_slots(self, slots) @@ -453,8 +493,8 @@ impl ArrayRef { } /// Returns the slots of the array. - pub fn slots(&self) -> Vec> { - self.0.slots(self) + pub fn slots(&self) -> &[Option] { + self.0.slots() } /// Returns the name of the slot at the given index. @@ -516,7 +556,7 @@ impl Matcher for V { } fn try_match<'a>(array: &'a ArrayRef) -> Option> { - let data = &array.0.as_any().downcast_ref::>()?.data; - Some(unsafe { ArrayView::new_unchecked(array, data) }) + let inner = array.0.as_any().downcast_ref::>()?; + Some(unsafe { ArrayView::new_unchecked(array, &inner.data) }) } } diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index 0dbf065e6a1..c5bae039a18 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -36,6 +36,9 @@ pub use vtable::*; mod view; pub use view::*; +use crate::hash::ArrayEq; +use crate::hash::ArrayHash; + /// The public API trait for all Vortex arrays. /// /// This trait is sealed and cannot be implemented outside of `vortex-array`. @@ -45,12 +48,18 @@ pub(crate) trait DynArray: 'static + private::Sealed + Send + Sync + Debug { /// Returns the array as a reference to a generic [`Any`] trait object. fn as_any(&self) -> &dyn Any; + /// Converts an owned array allocation into an owned [`Any`] allocation for downcasting. + fn into_any_arc(self: std::sync::Arc) -> std::sync::Arc; + /// Returns the length of the array. fn len(&self) -> usize; /// Returns the logical Vortex [`DType`] of the array. fn dtype(&self) -> &DType; + /// Returns the slots of the array. + fn slots(&self) -> &[Option]; + /// Returns the vtable of the array. fn vtable(&self) -> &dyn DynVTable; @@ -112,9 +121,6 @@ pub(crate) trait DynArray: 'static + private::Sealed + Send + Sync + Debug { /// Returns the number of buffers of the array. fn nbuffers(&self, this: &ArrayRef) -> usize; - /// Returns the slots of the array. - fn slots(&self, this: &ArrayRef) -> Vec>; - /// Returns the name of the slot at the given index. fn slot_name(&self, this: &ArrayRef, idx: usize) -> String; @@ -129,7 +135,7 @@ pub(crate) trait DynArray: 'static + private::Sealed + Send + Sync + Debug { fn dyn_array_hash(&self, state: &mut dyn Hasher, precision: crate::Precision); /// Compares two arrays of the same concrete type for equality. - fn dyn_array_eq(&self, other: &dyn Any, precision: crate::Precision) -> bool; + fn dyn_array_eq(&self, other: &ArrayRef, precision: crate::Precision) -> bool; } /// Trait for converting a type into a Vortex [`ArrayRef`]. @@ -158,6 +164,10 @@ impl DynArray for ArrayInner { self } + fn into_any_arc(self: std::sync::Arc) -> std::sync::Arc { + self + } + fn len(&self) -> usize { self.len } @@ -166,6 +176,10 @@ impl DynArray for ArrayInner { &self.dtype } + fn slots(&self) -> &[Option] { + &self.slots + } + fn vtable(&self) -> &dyn DynVTable { &self.vtable } @@ -261,16 +275,6 @@ impl DynArray for ArrayInner { .collect() } - fn slots(&self, this: &ArrayRef) -> Vec> { - let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; - V::slots(view).to_vec() - } - - fn slot_name(&self, this: &ArrayRef, idx: usize) -> String { - let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; - V::slot_name(view, idx) - } - fn buffers(&self, this: &ArrayRef) -> Vec { let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; (0..V::nbuffers(view)) @@ -302,6 +306,11 @@ impl DynArray for ArrayInner { V::nbuffers(view) } + fn slot_name(&self, this: &ArrayRef, idx: usize) -> String { + let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; + V::slot_name(view, idx) + } + fn metadata(&self, this: &ArrayRef) -> VortexResult>> { let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; V::serialize(view) @@ -317,16 +326,30 @@ impl DynArray for ArrayInner { self.len.hash(&mut wrapper); self.dtype.hash(&mut wrapper); self.vtable.id().hash(&mut wrapper); - V::array_hash(&self.data, &mut wrapper, precision); + self.slots.len().hash(&mut wrapper); + for slot in &self.slots { + slot.array_hash(&mut wrapper, precision); + } + self.data.array_hash(&mut wrapper, precision); } - fn dyn_array_eq(&self, other: &dyn Any, precision: crate::Precision) -> bool { - other.downcast_ref::().is_some_and(|other| { - self.len == other.len - && self.dtype == other.dtype - && self.vtable.id() == other.vtable.id() - && V::array_eq(&self.data, &other.data, precision) - }) + fn dyn_array_eq(&self, other: &ArrayRef, precision: crate::Precision) -> bool { + other + .inner() + .as_any() + .downcast_ref::() + .is_some_and(|other_inner| { + self.len == other.len() + && self.dtype == *other.dtype() + && self.vtable.id() == other.encoding_id() + && self.slots.len() == other_inner.slots.len() + && self + .slots + .iter() + .zip(other_inner.slots.iter()) + .all(|(slot, other_slot)| slot.array_eq(other_slot, precision)) + && self.data.array_eq(&other_inner.data, precision) + }) } } diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index 93e764e7d3a..0d8239af589 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -31,6 +31,7 @@ pub struct ArrayParts { pub dtype: DType, pub len: usize, pub data: V::ArrayData, + pub slots: Vec>, } impl ArrayParts { @@ -40,9 +41,31 @@ impl ArrayParts { dtype, len, data, + slots: Vec::new(), } } + + pub fn with_slots(mut self, slots: Vec>) -> Self { + self.slots = slots; + self + } +} + +/// Shared bound for helpers that should work over both owned [`Array`] and borrowed +/// [`ArrayView`]. +/// +/// Extension traits use this to share typed array logic while still exposing the backing +/// [`ArrayRef`] and the encoding-specific [`VTable::ArrayData`]. +pub trait TypedArrayRef: AsRef + Deref { + /// Returns an owned [`Array`] from the reference. + fn to_owned(&self) -> Array { + self.as_ref().clone().downcast() + } } + +impl TypedArrayRef for Array {} + +impl TypedArrayRef for ArrayView<'_, V> {} // ============================================================================= // ArrayInner — the concrete type stored inside Arc // ============================================================================= @@ -58,6 +81,7 @@ pub(crate) struct ArrayInner { pub(crate) dtype: DType, pub(crate) len: usize, pub(crate) data: V::ArrayData, + pub(crate) slots: Vec>, pub(crate) stats: ArrayStats, } @@ -65,13 +89,15 @@ impl ArrayInner { /// Create a new inner array from explicit construction parameters. #[doc(hidden)] pub fn try_new(new: ArrayParts) -> VortexResult { - new.vtable.validate(&new.data, &new.dtype, new.len)?; + new.vtable + .validate(&new.data, &new.dtype, new.len, &new.slots)?; Ok(unsafe { Self::from_data_unchecked( new.vtable, new.dtype, new.len, new.data, + new.slots, ArrayStats::default(), ) }) @@ -86,6 +112,7 @@ impl ArrayInner { dtype: DType, len: usize, data: V::ArrayData, + slots: Vec>, stats: ArrayStats, ) -> Self { Self { @@ -93,6 +120,7 @@ impl ArrayInner { dtype, len, data, + slots, stats, } } @@ -118,6 +146,7 @@ impl Clone for ArrayInner { dtype: self.dtype.clone(), len: self.len, data: self.data.clone(), + slots: self.slots.clone(), stats: self.stats.clone(), } } @@ -130,6 +159,7 @@ impl Debug for ArrayInner { .field("dtype", &self.dtype) .field("len", &self.len) .field("inner", &self.data) + .field("slots", &self.slots) .finish() } } @@ -185,6 +215,7 @@ impl Array { new.dtype, new.len, new.data, + new.slots, ArrayStats::default(), ) })); @@ -248,14 +279,25 @@ impl Array { &self.downcast_inner().data } - /// Returns the full typed array construction parts. - pub fn into_parts(self) -> ArrayParts { - let inner = self.downcast_inner(); - ArrayParts { - vtable: inner.vtable.clone(), - dtype: inner.dtype.clone(), - len: inner.len, - data: inner.data.clone(), + /// Returns the full typed array construction parts if this handle owns the allocation. + pub fn try_into_parts(self) -> Result, Self> { + let Self { inner, _phantom } = self; + let any = inner.into_inner().into_any_arc(); + let inner = Arc::downcast::>(any) + .unwrap_or_else(|_| unreachable!("typed array must contain ArrayInner for its vtable")); + + match Arc::try_unwrap(inner) { + Ok(inner) => Ok(ArrayParts { + vtable: inner.vtable, + dtype: inner.dtype, + len: inner.len, + data: inner.data, + slots: inner.slots, + }), + Err(inner) => Err(Self { + inner: ArrayRef::from_inner(inner), + _phantom: PhantomData, + }), } } @@ -269,6 +311,11 @@ impl Array { self.downcast_inner().data.clone() } + /// Returns the array slots. + pub fn slots(&self) -> &[Option] { + &self.downcast_inner().slots + } + /// Returns the internal [`ArrayRef`]. #[inline(always)] pub fn as_array(&self) -> &ArrayRef { @@ -451,11 +498,24 @@ mod tests { #[test] fn typed_array_into_parts_roundtrips() { let array = PrimitiveArray::new(buffer![1i32, 2, 3], Validity::NonNullable); - let expected = array.clone(); + let expected = PrimitiveArray::new(buffer![1i32, 2, 3], Validity::NonNullable); - let parts = array.into_parts(); + let parts = array.try_into_parts().unwrap(); let rebuilt = Array::::try_from_parts(parts).unwrap(); assert_arrays_eq!(rebuilt, expected); } + + #[test] + fn typed_array_try_into_parts_requires_unique_owner() { + let array = PrimitiveArray::new(buffer![1i32, 2, 3], Validity::NonNullable); + let alias = array.clone(); + + let array = match array.try_into_parts() { + Ok(_) => panic!("aliased arrays should not move out their backing parts"), + Err(array) => array, + }; + + assert_arrays_eq!(array, alias); + } } diff --git a/vortex-array/src/array/view.rs b/vortex-array/src/array/view.rs index 7a2027591d9..c751429abf8 100644 --- a/vortex-array/src/array/view.rs +++ b/vortex-array/src/array/view.rs @@ -45,6 +45,10 @@ impl<'a, V: VTable> ArrayView<'a, V> { self.data } + pub fn slots(&self) -> &'a [Option] { + self.array.slots() + } + pub fn dtype(&self) -> &DType { self.array.dtype() } diff --git a/vortex-array/src/array/vtable/dyn_.rs b/vortex-array/src/array/vtable/dyn_.rs index 681267403e2..656f7eba6fa 100644 --- a/vortex-array/src/array/vtable/dyn_.rs +++ b/vortex-array/src/array/vtable/dyn_.rs @@ -22,6 +22,7 @@ use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; + /// Reference-counted DynVTable pub type DynVTableRef = Arc; @@ -41,7 +42,11 @@ pub trait DynVTable: 'static + Send + Sync + Debug { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult; - /// See [`VTable::with_slots`] + + /// Rebuilds the array with the provided outer slots. + /// + /// This is only for physical rewrites of existing slots. Slot count, presence, logical dtype, + /// and logical len must remain unchanged. fn with_slots(&self, array: ArrayRef, slots: Vec>) -> VortexResult; /// See [`VTable::reduce`] @@ -83,30 +88,54 @@ impl DynVTable for V { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult { - let inner = self.deserialize(dtype, len, metadata, buffers, children, session)?; - Ok( - Array::::try_from_parts(ArrayParts::new(self.clone(), dtype.clone(), len, inner))? - .into_array(), - ) + Ok(Array::::try_from_parts( + self.deserialize(dtype, len, metadata, buffers, children, session)?, + )? + .into_array()) } fn with_slots(&self, array: ArrayRef, slots: Vec>) -> VortexResult { + let old_slots = array.slots(); + vortex_ensure!( + old_slots.len() == slots.len(), + "slot count changed from {} to {} during physical rewrite", + old_slots.len(), + slots.len() + ); + for (idx, (old_slot, new_slot)) in old_slots.iter().zip(slots.iter()).enumerate() { + vortex_ensure!( + old_slot.is_some() == new_slot.is_some(), + "slot {} presence changed during physical rewrite", + idx + ); + if let (Some(old_slot), Some(new_slot)) = (old_slot.as_ref(), new_slot.as_ref()) { + vortex_ensure!( + old_slot.dtype() == new_slot.dtype(), + "slot {} dtype changed from {} to {} during physical rewrite", + idx, + old_slot.dtype(), + new_slot.dtype() + ); + vortex_ensure!( + old_slot.len() == new_slot.len(), + "slot {} len changed from {} to {} during physical rewrite", + idx, + old_slot.len(), + new_slot.len() + ); + } + } let typed = array .as_opt::() .vortex_expect("Failed to downcast array"); - let mut data = typed.data().clone(); - V::with_slots(&mut data, slots)?; + let data = typed.data().clone(); let stats = array.statistics().to_owned(); - Ok(unsafe { - Array::::from_parts_unchecked(ArrayParts::new( - self.clone(), - array.dtype().clone(), - array.len(), - data, - )) - .with_stats_set(stats) - .into_array() - }) + Ok(Array::::try_from_parts( + ArrayParts::new(self.clone(), array.dtype().clone(), array.len(), data) + .with_slots(slots), + )? + .with_stats_set(stats) + .into_array()) } fn reduce(&self, array: &ArrayRef) -> VortexResult> { @@ -155,7 +184,6 @@ impl DynVTable for V { } fn execute(&self, array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - // Capture metadata before the move for post-validation and stats inheritance. let len = array.len(); let dtype = array.dtype().clone(); let stats = array.statistics().to_owned(); @@ -179,7 +207,6 @@ impl DynVTable for V { ); } - // TODO(ngates): do we want to do this on every execution? We used to in to_canonical. result.array().statistics().set_iter(stats.into_iter()); } diff --git a/vortex-array/src/array/vtable/mod.rs b/vortex-array/src/array/vtable/mod.rs index 305cbef6ca2..dcf3bcaac26 100644 --- a/vortex-array/src/array/vtable/mod.rs +++ b/vortex-array/src/array/vtable/mod.rs @@ -9,7 +9,6 @@ mod validity; use std::fmt::Debug; use std::fmt::Formatter; -use std::hash::Hasher; pub use dyn_::*; pub use operations::*; @@ -25,7 +24,6 @@ use crate::ArrayView; use crate::Canonical; use crate::ExecutionResult; use crate::IntoArray; -use crate::Precision; use crate::arrays::ConstantArray; use crate::arrays::constant::Constant; use crate::buffer::BufferHandle; @@ -33,6 +31,8 @@ use crate::builders::ArrayBuilder; use crate::dtype::DType; use crate::dtype::Nullability; use crate::executor::ExecutionCtx; +use crate::hash::ArrayEq; +use crate::hash::ArrayHash; use crate::patches::Patches; use crate::scalar::ScalarValue; use crate::serde::ArrayChildren; @@ -52,7 +52,7 @@ use crate::validity::Validity; /// out of bounds). Post-conditions are validated after invocation of the vtable function and will /// panic if violated. pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { - type ArrayData: 'static + Send + Sync + Clone + Debug; + type ArrayData: 'static + Send + Sync + Clone + Debug + ArrayHash + ArrayEq; type OperationsVTable: OperationsVTable; type ValidityVTable: ValidityVTable; @@ -61,13 +61,13 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { fn id(&self) -> ArrayId; /// Validates that externally supplied logical metadata matches the array data. - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()>; - - /// Hashes the array contents. - fn array_hash(array: &Self::ArrayData, state: &mut H, precision: Precision); - - /// Compares two arrays of the same type for equality. - fn array_eq(array: &Self::ArrayData, other: &Self::ArrayData, precision: Precision) -> bool; + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()>; /// Returns the number of buffers in the array. fn nbuffers(array: ArrayView<'_, Self>) -> usize; @@ -85,7 +85,7 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { /// /// The default counts non-None slots. fn nchildren(array: ArrayView<'_, Self>) -> usize { - Self::slots(array).iter().filter(|s| s.is_some()).count() + array.slots().iter().filter(|s| s.is_some()).count() } /// Returns the child at the given index. @@ -95,7 +95,8 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { /// # Panics /// Panics if `idx >= nchildren(array)`. fn child(array: ArrayView<'_, Self>, idx: usize) -> ArrayRef { - Self::slots(array) + array + .slots() .iter() .filter_map(|s| s.clone()) .nth(idx) @@ -109,7 +110,8 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { /// # Panics /// Panics if `idx >= nchildren(array)`. fn child_name(array: ArrayView<'_, Self>, idx: usize) -> String { - Self::slots(array) + array + .slots() .iter() .enumerate() .filter(|(_, s)| s.is_some()) @@ -139,7 +141,7 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { buffers: &[BufferHandle], children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult; + ) -> VortexResult>; /// Writes the array into a canonical builder. fn append_to_builder( @@ -156,36 +158,12 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { Ok(()) } - /// Returns the slots of the array as a slice. - /// - /// Slots provide fixed-position storage for child arrays. Each encoding defines named - /// constants (e.g. `VALIDITY_SLOT`, `ELEMENTS_SLOT`) that index into this slice, so child - /// access is a direct index rather than a dynamic lookup. - /// - /// Slots are `Option` to allow individual children to be _taken_ (moved out) - /// without invalidating the indices of other slots. For example, removing the validity - /// child leaves a `None` at `VALIDITY_SLOT` while all other slot indices remain stable. - /// - /// The backing storage is a `Vec` (rather than a fixed-size array) so that it can be - /// moved out of an `ArrayData` into the concrete `Array` type during deserialization - /// without copying. - /// - /// TODO: once no encodings rely on side-effects in [`Self::with_slots`], replace the - /// `slots`/`with_slots` pair with a single `slots_mut` returning `&mut [Option]`. - fn slots<'a>(array: ArrayView<'a, Self>) -> &'a [Option]; - /// Returns the name of the slot at the given index. /// /// # Panics /// Panics if `idx >= slots(array).len()`. fn slot_name(array: ArrayView<'_, Self>, idx: usize) -> String; - /// Replaces the slots in `array` with the given `slots` vec. - /// - /// Some encodings use this to perform side-effects (e.g. cache invalidation) when - /// slots change. Once those are removed, this will be replaced by `slots_mut`. - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()>; - /// Execute this array by returning an [`ExecutionResult`]. /// /// Instead of recursively executing children, implementations should return diff --git a/vortex-array/src/array/vtable/validity.rs b/vortex-array/src/array/vtable/validity.rs index 02eac298f46..350ef5d3ab7 100644 --- a/vortex-array/src/array/vtable/validity.rs +++ b/vortex-array/src/array/vtable/validity.rs @@ -44,7 +44,7 @@ where pub struct ValidityVTableFromChild; pub trait ValidityChild { - fn validity_child(array: &V::ArrayData) -> &ArrayRef; + fn validity_child(array: ArrayView<'_, V>) -> ArrayRef; } impl ValidityVTable for ValidityVTableFromChild @@ -52,7 +52,7 @@ where V: ValidityChild, { fn validity(array: ArrayView<'_, V>) -> VortexResult { - V::validity_child(array.data()).validity() + V::validity_child(array).validity() } } diff --git a/vortex-array/src/arrays/arbitrary.rs b/vortex-array/src/arrays/arbitrary.rs index 4cd7fccfba8..0c190568200 100644 --- a/vortex-array/src/arrays/arbitrary.rs +++ b/vortex-array/src/arrays/arbitrary.rs @@ -21,6 +21,7 @@ use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; use crate::arrays::VarBinArray; use crate::arrays::VarBinViewArray; +use crate::arrays::primitive::PrimitiveArrayExt; use crate::builders::ArrayBuilder; use crate::builders::DecimalBuilder; use crate::builders::FixedSizeListBuilder; diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index e66deb32cc1..256c19210c9 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -13,13 +13,13 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::buffer::BufferHandle; use crate::dtype::DType; -use crate::dtype::Nullability; use crate::validity::Validity; /// The validity bitmap indicating which elements are non-null. @@ -61,82 +61,39 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; /// ``` #[derive(Clone, Debug)] pub struct BoolData { - /// Child arrays stored as slots. See [`VTable::slots`] for design rationale. - pub(super) slots: Vec>, - pub(super) nullability: Nullability, pub(super) bits: BufferHandle, pub(super) offset: usize, - pub(super) len: usize, } pub struct BoolDataParts { pub bits: BufferHandle, pub offset: usize, pub len: usize, - pub validity: Validity, } -/// Field accessors and non-consuming methods on the inner bool data. -impl BoolData { - /// Returns the length of this array. - pub fn len(&self) -> usize { - self.len - } - - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - DType::Bool(self.nullability) - } - - /// Returns `true` if this array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 +pub trait BoolArrayExt: TypedArrayRef { + fn nullability(&self) -> crate::dtype::Nullability { + match self.as_ref().dtype() { + DType::Bool(nullability) => *nullability, + _ => unreachable!("BoolArrayExt requires a bool dtype"), + } } - /// Returns the [`Validity`] of this array. - #[allow(clippy::same_name_method)] - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) + fn validity(&self) -> Validity { + child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - /// Returns the validity as a [`Mask`]. - pub fn validity_mask(&self) -> Mask { - self.validity().to_mask(self.len()) + fn bool_validity_mask(&self) -> Mask { + self.validity().to_mask(self.as_ref().len()) } - /// Returns the underlying [`BitBuffer`] of the array. - pub fn to_bit_buffer(&self) -> BitBuffer { + fn to_bit_buffer(&self) -> BitBuffer { let buffer = self.bits.as_host().clone(); - BitBuffer::new_with_offset(buffer, self.len, self.offset) + BitBuffer::new_with_offset(buffer, self.as_ref().len(), self.offset) } - /// Returns the underlying [`BitBuffer`] of the array - pub fn into_bit_buffer(self) -> BitBuffer { - let buffer = self.bits.unwrap_host(); - BitBuffer::new_with_offset(buffer, self.len, self.offset) - } - - /// Splits into owned parts - #[inline] - pub fn into_parts(self) -> BoolDataParts { - let validity = self.validity(); - BoolDataParts { - bits: self.bits, - offset: self.offset, - len: self.len, - validity, - } - } - - pub fn to_mask(&self) -> Mask { - self.maybe_to_mask() - .vortex_expect("failed to check validity") - .vortex_expect("cannot convert nullable boolean array to mask") - } - - pub fn maybe_to_mask(&self) -> VortexResult> { - let validity = self.validity(); - let all_valid = match &validity { + fn maybe_to_mask(&self) -> VortexResult> { + let all_valid = match &self.validity() { Validity::NonNullable | Validity::AllValid => true, Validity::AllInvalid => false, Validity::Array(a) => a.statistics().compute_min::().unwrap_or(false), @@ -144,17 +101,37 @@ impl BoolData { Ok(all_valid.then(|| Mask::from_buffer(self.to_bit_buffer()))) } - pub fn to_mask_fill_null_false(&self) -> Mask { - let validity_mask = self.validity_mask(); + fn to_mask(&self) -> Mask { + self.maybe_to_mask() + .vortex_expect("failed to check validity") + .vortex_expect("cannot convert nullable boolean array to mask") + } + + fn to_mask_fill_null_false(&self) -> Mask { + let validity_mask = self.bool_validity_mask(); let buffer = match validity_mask { Mask::AllTrue(_) => self.to_bit_buffer(), - Mask::AllFalse(_) => return Mask::new_false(self.len()), + Mask::AllFalse(_) => return Mask::new_false(self.as_ref().len()), Mask::Values(validity) => validity.bit_buffer() & self.to_bit_buffer(), }; Mask::from_buffer(buffer) } +} +impl> BoolArrayExt for T {} + +/// Field accessors and non-consuming methods on the inner bool data. +impl BoolData { + /// Splits into owned parts + #[inline] + pub fn into_parts(self, len: usize) -> BoolDataParts { + BoolDataParts { + bits: self.bits, + offset: self.offset, + len, + } + } - fn make_slots(validity: &Validity, len: usize) -> Vec> { + pub(crate) fn make_slots(validity: &Validity, len: usize) -> Vec> { vec![validity_to_child(validity, len)] } } @@ -188,8 +165,11 @@ impl Array { pub fn try_new(bits: BitBuffer, validity: Validity) -> VortexResult { let dtype = DType::Bool(validity.nullability()); let len = bits.len(); + let slots = BoolData::make_slots(&validity, len); let data = BoolData::try_new(bits, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data)) }) + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data).with_slots(slots)) + }) } /// Build a new bool array from a `BufferHandle`, returning an error if the offset is @@ -201,8 +181,11 @@ impl Array { validity: Validity, ) -> VortexResult { let dtype = DType::Bool(validity.nullability()); + let slots = BoolData::make_slots(&validity, len); let data = BoolData::try_new_from_handle(bits, offset, len, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data)) }) + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data).with_slots(slots)) + }) } /// Creates a new [`BoolArray`] without validation. @@ -213,9 +196,12 @@ impl Array { pub unsafe fn new_unchecked(bits: BitBuffer, validity: Validity) -> Self { let dtype = DType::Bool(validity.nullability()); let len = bits.len(); + let slots = BoolData::make_slots(&validity, len); // SAFETY: caller guarantees validity length equals bit buffer length. let data = unsafe { BoolData::new_unchecked(bits, validity) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data)) } + unsafe { + Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data).with_slots(slots)) + } } /// Validates the components that would be used to create a [`BoolArray`]. @@ -238,7 +224,10 @@ impl Array { /// Returns the underlying [`BitBuffer`] of the array, consuming self. pub fn into_bit_buffer(self) -> BitBuffer { - self.into_data().into_bit_buffer() + let len = self.len(); + let data = self.into_data(); + let buffer = data.bits.unwrap_host(); + BitBuffer::new_with_offset(buffer, len, data.offset) } } @@ -248,15 +237,11 @@ impl BoolData { let bits = bits.shrink_offset(); Self::validate(&bits, &validity)?; - let (offset, len, buffer) = bits.into_inner(); + let (offset, _len, buffer) = bits.into_inner(); - let slots = Self::make_slots(&validity, len); Ok(Self { - slots, - nullability: validity.nullability(), bits: BufferHandle::new_host(buffer), offset, - len, }) } @@ -281,29 +266,18 @@ impl BoolData { bits.len() * 8, ); - let slots = Self::make_slots(&validity, len); - Ok(Self { - slots, - nullability: validity.nullability(), - bits, - offset, - len, - }) + Ok(Self { bits, offset }) } pub(super) unsafe fn new_unchecked(bits: BitBuffer, validity: Validity) -> Self { if cfg!(debug_assertions) { Self::try_new(bits, validity).vortex_expect("Failed to create BoolData") } else { - let (offset, len, buffer) = bits.into_inner(); - let slots = Self::make_slots(&validity, len); + let (offset, _len, buffer) = bits.into_inner(); Self { - slots, - nullability: validity.nullability(), bits: BufferHandle::new_host(buffer), offset, - len, } } } @@ -378,6 +352,7 @@ mod tests { use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::PrimitiveArray; + use crate::arrays::bool::BoolArrayExt; use crate::assert_arrays_eq; use crate::patches::Patches; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/bool/compute/cast.rs b/vortex-array/src/arrays/bool/compute/cast.rs index bff33756255..3b5fbea7607 100644 --- a/vortex-array/src/arrays/bool/compute/cast.rs +++ b/vortex-array/src/arrays/bool/compute/cast.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Bool; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; diff --git a/vortex-array/src/arrays/bool/compute/fill_null.rs b/vortex-array/src/arrays/bool/compute/fill_null.rs index d1a54b045c7..f2489f9d073 100644 --- a/vortex-array/src/arrays/bool/compute/fill_null.rs +++ b/vortex-array/src/arrays/bool/compute/fill_null.rs @@ -10,6 +10,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Bool; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; use crate::scalar::Scalar; use crate::scalar_fn::fns::fill_null::FillNullKernel; use crate::validity::Validity; @@ -48,6 +49,7 @@ mod tests { use crate::IntoArray; use crate::arrays::BoolArray; + use crate::arrays::bool::BoolArrayExt; use crate::builtins::ArrayBuiltins; use crate::canonical::ToCanonical; use crate::dtype::DType; diff --git a/vortex-array/src/arrays/bool/compute/filter.rs b/vortex-array/src/arrays/bool/compute/filter.rs index d4c4d47f862..c7b45498957 100644 --- a/vortex-array/src/arrays/bool/compute/filter.rs +++ b/vortex-array/src/arrays/bool/compute/filter.rs @@ -14,6 +14,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Bool; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; use crate::arrays::filter::FilterReduce; /// If the filter density is above 80%, we use slices to filter the array instead of indices. diff --git a/vortex-array/src/arrays/bool/compute/mask.rs b/vortex-array/src/arrays/bool/compute/mask.rs index f55574cab94..6fd278f2d3d 100644 --- a/vortex-array/src/arrays/bool/compute/mask.rs +++ b/vortex-array/src/arrays/bool/compute/mask.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Bool; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; use crate::scalar_fn::fns::mask::MaskReduce; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/bool/compute/rules.rs b/vortex-array/src/arrays/bool/compute/rules.rs index d129fd13ffe..ed2e7a4dc28 100644 --- a/vortex-array/src/arrays/bool/compute/rules.rs +++ b/vortex-array/src/arrays/bool/compute/rules.rs @@ -9,6 +9,7 @@ use crate::array::ArrayView; use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::arrays::Masked; +use crate::arrays::bool::BoolArrayExt; use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ArrayParentReduceRule; diff --git a/vortex-array/src/arrays/bool/compute/slice.rs b/vortex-array/src/arrays/bool/compute/slice.rs index eea4d51b19b..0ef11bb3085 100644 --- a/vortex-array/src/arrays/bool/compute/slice.rs +++ b/vortex-array/src/arrays/bool/compute/slice.rs @@ -10,6 +10,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Bool; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; use crate::arrays::slice::SliceReduce; impl SliceReduce for Bool { diff --git a/vortex-array/src/arrays/bool/compute/take.rs b/vortex-array/src/arrays/bool/compute/take.rs index f585a71332d..19b1508fd84 100644 --- a/vortex-array/src/arrays/bool/compute/take.rs +++ b/vortex-array/src/arrays/bool/compute/take.rs @@ -15,6 +15,7 @@ use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; +use crate::arrays::bool::BoolArrayExt; use crate::arrays::dict::TakeExecute; use crate::builtins::ArrayBuiltins; use crate::executor::ExecutionCtx; @@ -86,6 +87,7 @@ mod test { use crate::ToCanonical; use crate::arrays::BoolArray; use crate::arrays::PrimitiveArray; + use crate::arrays::bool::BoolArrayExt; use crate::assert_arrays_eq; use crate::compute::conformance::take::test_take_conformance; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/bool/mod.rs b/vortex-array/src/arrays/bool/mod.rs index b0f20794ca8..a4aee2c4faa 100644 --- a/vortex-array/src/arrays/bool/mod.rs +++ b/vortex-array/src/arrays/bool/mod.rs @@ -4,6 +4,7 @@ mod array; mod patch; +pub use array::BoolArrayExt; pub use array::BoolData; pub use array::BoolDataParts; diff --git a/vortex-array/src/arrays/bool/patch.rs b/vortex-array/src/arrays/bool/patch.rs index 87306df7e1e..fef03bc30fb 100644 --- a/vortex-array/src/arrays/bool/patch.rs +++ b/vortex-array/src/arrays/bool/patch.rs @@ -8,6 +8,7 @@ use vortex_error::VortexResult; use crate::ExecutionCtx; use crate::arrays::BoolArray; use crate::arrays::PrimitiveArray; +use crate::arrays::bool::BoolArrayExt; use crate::match_each_unsigned_integer_ptype; use crate::patches::Patches; diff --git a/vortex-array/src/arrays/bool/test_harness.rs b/vortex-array/src/arrays/bool/test_harness.rs index 4adc53b89ac..52cffafdbd9 100644 --- a/vortex-array/src/arrays/bool/test_harness.rs +++ b/vortex-array/src/arrays/bool/test_harness.rs @@ -5,6 +5,7 @@ use vortex_error::VortexExpect; use vortex_error::vortex_panic; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; impl BoolArray { pub fn opt_bool_vec(&self) -> Vec> { diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index c2a5c25ca8c..b722022071c 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -2,6 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::fmt::Formatter; +use std::hash::Hash; +use std::hash::Hasher; use kernel::PARENT_KERNELS; use prost::Message; @@ -19,7 +21,6 @@ use crate::array::Array; use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::bool::BoolData; -use crate::arrays::bool::array::NUM_SLOTS; use crate::arrays::bool::array::SLOT_NAMES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -46,6 +47,19 @@ pub struct BoolMetadata { pub offset: u32, } +impl ArrayHash for BoolData { + fn array_hash(&self, state: &mut H, precision: Precision) { + self.bits.array_hash(state, precision); + self.offset.hash(state); + } +} + +impl ArrayEq for BoolData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + self.offset == other.offset && self.bits.array_eq(&other.bits, precision) + } +} + impl VTable for Bool { type ArrayData = BoolData; @@ -56,18 +70,6 @@ impl VTable for Bool { Self::ID } - fn array_hash(array: &BoolData, state: &mut H, precision: Precision) { - array.to_bit_buffer().array_hash(state, precision); - array.validity().array_hash(state, precision); - } - - fn array_eq(array: &BoolData, other: &BoolData, precision: Precision) -> bool { - array - .to_bit_buffer() - .array_eq(&other.to_bit_buffer(), precision) - && array.validity().array_eq(&other.validity(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 1 } @@ -100,21 +102,33 @@ impl VTable for Bool { write!(f, "BoolMetadata {{ offset: {} }}", array.offset) } - fn validate(&self, data: &BoolData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &BoolData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let DType::Bool(nullability) = dtype else { + vortex_bail!("Expected bool dtype, got {dtype:?}"); + }; vortex_ensure!( - data.len() == len, - "BoolArray length {} does not match outer length {}", - data.len(), - len + data.bits.len() * 8 >= data.offset + len, + "BoolArray buffer with offset {} cannot back outer length {} (buffer bits = {})", + data.offset, + len, + data.bits.len() * 8 ); - let actual_dtype = data.dtype(); - vortex_ensure!( - &actual_dtype == dtype, - "BoolArray dtype {} does not match outer dtype {}", - actual_dtype, - dtype - ); + let validity = crate::array::child_to_validity(&slots[0], *nullability); + if let Some(validity_len) = validity.maybe_len() { + vortex_ensure!( + validity_len == len, + "BoolArray validity len {} does not match outer length {}", + validity_len, + len + ); + } Ok(()) } @@ -128,7 +142,7 @@ impl VTable for Bool { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = BoolMetadata::decode(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); @@ -144,29 +158,15 @@ impl VTable for Bool { }; let buffer = buffers[0].clone(); - - BoolData::try_new_from_handle(buffer, metadata.offset as usize, len, validity) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let slots = BoolData::make_slots(&validity, len); + let data = BoolData::try_new_from_handle(buffer, metadata.offset as usize, len, validity)?; + Ok(crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "BoolArray expects {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done(array)) } diff --git a/vortex-array/src/arrays/bool/vtable/operations.rs b/vortex-array/src/arrays/bool/vtable/operations.rs index dbcb025fe08..ece00b038ea 100644 --- a/vortex-array/src/arrays/bool/vtable/operations.rs +++ b/vortex-array/src/arrays/bool/vtable/operations.rs @@ -7,6 +7,7 @@ use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::Bool; +use crate::arrays::bool::BoolArrayExt; use crate::scalar::Scalar; impl OperationsVTable for Bool { @@ -29,6 +30,7 @@ mod tests { use crate::IntoArray; use crate::ToCanonical; use crate::arrays::BoolArray; + use crate::arrays::bool::BoolArrayExt; use crate::assert_arrays_eq; #[test] diff --git a/vortex-array/src/arrays/bool/vtable/validity.rs b/vortex-array/src/arrays/bool/vtable/validity.rs index d0f3c3eeec5..6d5a9a506bc 100644 --- a/vortex-array/src/arrays/bool/vtable/validity.rs +++ b/vortex-array/src/arrays/bool/vtable/validity.rs @@ -5,11 +5,12 @@ use vortex_error::VortexResult; use crate::array::ArrayView; use crate::array::ValidityVTable; +use crate::arrays::bool::BoolArrayExt; use crate::arrays::bool::vtable::Bool; use crate::validity::Validity; impl ValidityVTable for Bool { fn validity(array: ArrayView<'_, Bool>) -> VortexResult { - Ok(array.data().validity()) + Ok(BoolArrayExt::validity(&array)) } } diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index 33b471e1852..7b4d362e6d8 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -8,7 +8,6 @@ use std::fmt::Debug; use futures::stream; -use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -18,8 +17,9 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::arrays::Chunked; -use crate::arrays::primitive::PrimitiveData; +use crate::arrays::PrimitiveArray; use crate::dtype::DType; use crate::iter::ArrayIterator; use crate::iter::ArrayIteratorAdapter; @@ -34,185 +34,141 @@ pub(super) const CHUNKS_OFFSET: usize = 1; #[derive(Clone, Debug)] pub struct ChunkedData { - pub(super) empty_dtype: Option, - pub(super) chunk_offsets: PrimitiveData, - pub(super) chunks: Vec, - pub(super) slots: Vec>, + pub(super) chunk_offsets: Vec, } -impl ChunkedData { - /// Builds the slots vector from chunk_offsets and chunks. - pub(super) fn make_slots( - chunk_offsets: &PrimitiveData, - chunks: &[ArrayRef], - ) -> Vec> { - let mut slots = Vec::with_capacity(1 + chunks.len()); - slots.push(Some( - unsafe { - Array::from_parts_unchecked(ArrayParts::new( - crate::arrays::Primitive, - chunk_offsets.dtype(), - chunk_offsets.len(), - chunk_offsets.clone(), - )) - } - .into_array(), - )); - slots.extend(chunks.iter().map(|c| Some(c.clone()))); - slots - } - - /// Constructs a new `ChunkedArray`. - /// - /// See `ChunkedArray::new_unchecked` for more information. - /// - /// # Errors - /// - /// Returns an error if the provided components do not satisfy the invariants documented in - /// `ChunkedArray::new_unchecked`. - pub fn try_new(chunks: Vec, dtype: DType) -> VortexResult { - Self::validate(&chunks, &dtype)?; - - // SAFETY: validation done above. - unsafe { Ok(Self::new_unchecked(chunks, dtype)) } - } - - /// Creates a new `ChunkedArray` without validation from these components: - /// - /// * `chunks` is a vector of arrays to be concatenated logically. - /// * `dtype` is the common data type of all chunks. - /// - /// # Safety - /// - /// All chunks must have exactly the same [`DType`] as the provided `dtype`. - pub unsafe fn new_unchecked(chunks: Vec, dtype: DType) -> Self { - #[cfg(debug_assertions)] - Self::validate(&chunks, &dtype) - .vortex_expect("[Debug Assertion]: Invalid `ChunkedArray` parameters"); - - let nchunks = chunks.len(); - - let mut chunk_offsets_buf = BufferMut::::with_capacity(nchunks + 1); - // SAFETY: nchunks + 1 - unsafe { chunk_offsets_buf.push_unchecked(0) } - let mut curr_offset = 0; - for c in &chunks { - curr_offset += c.len() as u64; - // SAFETY: nchunks + 1 - unsafe { chunk_offsets_buf.push_unchecked(curr_offset) } - } - - let chunk_offsets = PrimitiveData::new(chunk_offsets_buf.freeze(), Validity::NonNullable); - - let slots = Self::make_slots(&chunk_offsets, &chunks); - Self { - empty_dtype: chunks.is_empty().then_some(dtype), - chunk_offsets, - chunks, - slots, - } +pub trait ChunkedArrayExt: TypedArrayRef { + fn chunk_offsets_array(&self) -> &ArrayRef { + self.as_ref().slots()[CHUNK_OFFSETS_SLOT] + .as_ref() + .vortex_expect("validated chunk offsets slot") } - /// Validates the components that would be used to create a `ChunkedArray`. - /// - /// This function checks all the invariants required by `ChunkedArray::new_unchecked`. - pub fn validate(chunks: &[ArrayRef], dtype: &DType) -> VortexResult<()> { - for chunk in chunks { - if chunk.dtype() != dtype { - vortex_bail!(MismatchedTypes: dtype, chunk.dtype()); - } - } - - Ok(()) + fn nchunks(&self) -> usize { + self.as_ref().slots().len().saturating_sub(CHUNKS_OFFSET) } - /// Returns the length of this array. - pub fn len(&self) -> usize { - self.chunk_offsets() - .last() - .copied() - .and_then(|len| usize::try_from(len).ok()) - .vortex_expect("chunk offset must fit in usize") + fn chunk(&self, idx: usize) -> &ArrayRef { + self.as_ref().slots()[CHUNKS_OFFSET + idx] + .as_ref() + .vortex_expect("validated chunk slot") } - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - self.chunks - .first() - .map(|chunk| chunk.dtype()) - .or(self.empty_dtype.as_ref()) - .vortex_expect("ChunkedArray dtype must come from chunks or the empty dtype") - } - - /// Returns `true` if this array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 + fn iter_chunks<'a>(&'a self) -> Box + 'a> { + Box::new( + self.as_ref().slots()[CHUNKS_OFFSET..] + .iter() + .map(|slot| slot.as_ref().vortex_expect("validated chunk slot")), + ) } - #[inline] - pub fn chunk(&self, idx: usize) -> &ArrayRef { - assert!(idx < self.nchunks(), "chunk index {idx} out of bounds"); - &self.chunks[idx] + fn chunks(&self) -> Vec { + self.iter_chunks().cloned().collect() } - pub fn nchunks(&self) -> usize { - self.chunks.len() + fn non_empty_chunks<'a>(&'a self) -> Box + 'a> { + Box::new(self.iter_chunks().filter(|chunk| !chunk.is_empty())) } - /// Returns the chunk offsets as a `PrimitiveData`. - pub(crate) fn chunk_offsets_data(&self) -> &PrimitiveData { + fn chunk_offsets(&self) -> &[usize] { &self.chunk_offsets } - #[inline] - pub fn chunk_offsets(&self) -> Buffer { - self.chunk_offsets_data().to_buffer() - } - - pub(crate) fn find_chunk_idx(&self, index: usize) -> VortexResult<(usize, usize)> { - assert!(index <= self.len(), "Index out of bounds of the array"); - let index = index as u64; - - // Since there might be duplicate values in offsets because of empty chunks we want to search from right - // and take the last chunk (we subtract 1 since there's a leading 0) - let index_chunk = self - .chunk_offsets() + fn find_chunk_idx(&self, index: usize) -> VortexResult<(usize, usize)> { + assert!( + index <= self.as_ref().len(), + "Index out of bounds of the array" + ); + let chunk_offsets = self.chunk_offsets(); + let index_chunk = chunk_offsets .search_sorted(&index, SearchSortedSide::Right)? .to_ends_index(self.nchunks() + 1) .saturating_sub(1); - let chunk_start = self.chunk_offsets()[index_chunk]; - - let index_in_chunk = - usize::try_from(index - chunk_start).vortex_expect("Index is too large for usize"); + let chunk_start = chunk_offsets[index_chunk]; + let index_in_chunk = index - chunk_start; Ok((index_chunk, index_in_chunk)) } - /// Returns an iterator over chunk references without allocation. - pub fn iter_chunks(&self) -> impl Iterator + '_ { - self.chunks.iter() + fn array_iterator(&self) -> impl ArrayIterator + '_ { + ArrayIteratorAdapter::new( + self.as_ref().dtype().clone(), + self.iter_chunks().map(|chunk| Ok(chunk.clone())), + ) } - /// Returns the chunks as a vector of owned references. - pub fn chunks(&self) -> Vec { - self.chunks.clone() + fn array_stream(&self) -> impl ArrayStream + '_ { + ArrayStreamAdapter::new( + self.as_ref().dtype().clone(), + stream::iter(self.iter_chunks().map(|chunk| Ok(chunk.clone()))), + ) } +} +impl> ChunkedArrayExt for T {} - pub fn non_empty_chunks(&self) -> impl Iterator + '_ { - self.chunks.iter().filter(|c| !c.is_empty()) +impl ChunkedData { + pub(super) fn compute_chunk_offsets(chunks: &[ArrayRef]) -> Vec { + let mut chunk_offsets = Vec::with_capacity(chunks.len() + 1); + chunk_offsets.push(0); + let mut curr_offset = 0; + for chunk in chunks { + curr_offset += chunk.len(); + chunk_offsets.push(curr_offset); + } + chunk_offsets } - pub fn array_iterator(&self) -> impl ArrayIterator + '_ { - ArrayIteratorAdapter::new( - self.dtype().clone(), - self.chunks.iter().map(|c| Ok(c.clone())), - ) + pub(super) fn make_slots( + chunk_offsets: &[usize], + chunks: &[ArrayRef], + ) -> Vec> { + let mut chunk_offsets_buf = BufferMut::::with_capacity(chunk_offsets.len()); + for &offset in chunk_offsets { + let offset = u64::try_from(offset) + .vortex_expect("chunk offset must fit in u64 for serialization"); + unsafe { chunk_offsets_buf.push_unchecked(offset) } + } + + let mut slots = Vec::with_capacity(1 + chunks.len()); + slots.push(Some( + PrimitiveArray::new(chunk_offsets_buf.freeze(), Validity::NonNullable).into_array(), + )); + slots.extend(chunks.iter().map(|c| Some(c.clone()))); + slots } - pub fn array_stream(&self) -> impl ArrayStream + '_ { - ArrayStreamAdapter::new( - self.dtype().clone(), - stream::iter(self.chunks.iter().map(|c| Ok(c.clone()))), - ) + /// Validates the components that would be used to create a `ChunkedArray`. + /// + /// This function checks all the invariants required by `ChunkedArray::new_unchecked`. + pub fn validate(chunks: &[ArrayRef], dtype: &DType) -> VortexResult<()> { + for chunk in chunks { + if chunk.dtype() != dtype { + vortex_bail!(MismatchedTypes: dtype, chunk.dtype()); + } + } + + Ok(()) + } +} + +impl Array { + /// Constructs a new `ChunkedArray`. + pub fn try_new(chunks: Vec, dtype: DType) -> VortexResult { + ChunkedData::validate(&chunks, &dtype)?; + let len = chunks.iter().map(|chunk| chunk.len()).sum(); + let chunk_offsets = ChunkedData::compute_chunk_offsets(&chunks); + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new( + Chunked, + dtype, + len, + ChunkedData { + chunk_offsets: chunk_offsets.clone(), + }, + ) + .with_slots(ChunkedData::make_slots(&chunk_offsets, &chunks)), + ) + }) } pub fn rechunk(&self, target_bytesize: u64, target_rowsize: usize) -> VortexResult { @@ -229,8 +185,6 @@ impl ChunkedData { && !chunks_to_combine.is_empty() { new_chunks.push( - // SAFETY: chunks_to_combine contains valid chunks of the same dtype as self. - // All chunks are guaranteed to be valid arrays matching self.dtype(). unsafe { Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) } @@ -255,8 +209,6 @@ impl ChunkedData { if !chunks_to_combine.is_empty() { new_chunks.push( - // SAFETY: chunks_to_combine contains valid chunks of the same dtype as self. - // All chunks are guaranteed to be valid arrays matching self.dtype(). unsafe { Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) } .into_array() .to_canonical()? @@ -264,43 +216,30 @@ impl ChunkedData { ); } - // SAFETY: new_chunks contains valid arrays of the same dtype as self. - // All chunks were either taken from self or created from self's chunks. unsafe { Ok(Self::new_unchecked(new_chunks, self.dtype().clone())) } } -} - -impl Array { - /// Constructs a new `ChunkedArray`. - pub fn try_new(chunks: Vec, dtype: DType) -> VortexResult { - let data = ChunkedData::try_new(chunks, dtype)?; - let dtype = data.dtype().clone(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Chunked, dtype, len, data)) }) - } - - pub fn rechunk(&self, target_bytesize: u64, target_rowsize: usize) -> VortexResult { - let data = self.data().rechunk(target_bytesize, target_rowsize)?; - Ok(unsafe { - Array::from_parts_unchecked(ArrayParts::new( - Chunked, - self.dtype().clone(), - data.len(), - data, - )) - }) - } /// Creates a new `ChunkedArray` without validation. /// /// # Safety /// - /// See [`ChunkedData::new_unchecked`]. + /// All chunks must have exactly the same [`DType`] as the provided `dtype`. pub unsafe fn new_unchecked(chunks: Vec, dtype: DType) -> Self { - let data = unsafe { ChunkedData::new_unchecked(chunks, dtype) }; - let dtype = data.dtype().clone(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Chunked, dtype, len, data)) } + let len = chunks.iter().map(|chunk| chunk.len()).sum(); + let chunk_offsets = ChunkedData::compute_chunk_offsets(&chunks); + unsafe { + Array::from_parts_unchecked( + ArrayParts::new( + Chunked, + dtype, + len, + ChunkedData { + chunk_offsets: chunk_offsets.clone(), + }, + ) + .with_slots(ChunkedData::make_slots(&chunk_offsets, &chunks)), + ) + } } } @@ -324,6 +263,7 @@ mod test { use crate::IntoArray; use crate::arrays::ChunkedArray; use crate::arrays::PrimitiveArray; + use crate::arrays::chunked::ChunkedArrayExt; use crate::assert_arrays_eq; use crate::dtype::DType; use crate::dtype::Nullability; diff --git a/vortex-array/src/arrays/chunked/compute/aggregate.rs b/vortex-array/src/arrays/chunked/compute/aggregate.rs index 6fa690542ea..474c6943ea6 100644 --- a/vortex-array/src/arrays/chunked/compute/aggregate.rs +++ b/vortex-array/src/arrays/chunked/compute/aggregate.rs @@ -8,6 +8,7 @@ use crate::ExecutionCtx; use crate::aggregate_fn::AggregateFnRef; use crate::aggregate_fn::kernels::DynAggregateKernel; use crate::arrays::Chunked; +use crate::arrays::chunked::ChunkedArrayExt; use crate::scalar::Scalar; #[derive(Debug)] diff --git a/vortex-array/src/arrays/chunked/compute/cast.rs b/vortex-array/src/arrays/chunked/compute/cast.rs index 63a432fd34e..9eed54d4300 100644 --- a/vortex-array/src/arrays/chunked/compute/cast.rs +++ b/vortex-array/src/arrays/chunked/compute/cast.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; +use crate::arrays::chunked::ChunkedArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; diff --git a/vortex-array/src/arrays/chunked/compute/fill_null.rs b/vortex-array/src/arrays/chunked/compute/fill_null.rs index 977d89b1dba..b207a152ee0 100644 --- a/vortex-array/src/arrays/chunked/compute/fill_null.rs +++ b/vortex-array/src/arrays/chunked/compute/fill_null.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; +use crate::arrays::chunked::ChunkedArrayExt; use crate::builtins::ArrayBuiltins; use crate::scalar::Scalar; use crate::scalar_fn::fns::fill_null::FillNullReduce; diff --git a/vortex-array/src/arrays/chunked/compute/filter.rs b/vortex-array/src/arrays/chunked/compute/filter.rs index aaeec137f60..0e32543761d 100644 --- a/vortex-array/src/arrays/chunked/compute/filter.rs +++ b/vortex-array/src/arrays/chunked/compute/filter.rs @@ -14,6 +14,7 @@ use crate::array::ArrayView; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; use crate::arrays::PrimitiveArray; +use crate::arrays::chunked::ChunkedArrayExt; use crate::arrays::filter::FilterKernel; use crate::search_sorted::SearchSorted; use crate::search_sorted::SearchSortedSide; @@ -95,10 +96,10 @@ pub(crate) fn chunk_filters( let mut chunk_filters = vec![ChunkFilter::None; array.nchunks()]; for (slice_start, slice_end) in slices { - let (start_chunk, start_idx) = find_chunk_idx(slice_start, &chunk_offsets)?; + let (start_chunk, start_idx) = find_chunk_idx(slice_start, chunk_offsets)?; // NOTE: we adjust slice end back by one, in case it ends on a chunk boundary, we do not // want to index into the unused chunk. - let (end_chunk, end_idx) = find_chunk_idx(slice_end - 1, &chunk_offsets)?; + let (end_chunk, end_idx) = find_chunk_idx(slice_end - 1, chunk_offsets)?; // Adjust back to an exclusive range let end_idx = end_idx + 1; @@ -117,8 +118,7 @@ pub(crate) fn chunk_filters( // start chunk: append a slice from (start_idx, start_chunk_end), i.e. whole chunk. // end chunk: append a slice from (0, end_idx). // chunks between start and end: append ChunkFilter::All. - let start_chunk_len: usize = - (chunk_offsets[start_chunk + 1] - chunk_offsets[start_chunk]).try_into()?; + let start_chunk_len = chunk_offsets[start_chunk + 1] - chunk_offsets[start_chunk]; let start_slice = (start_idx, start_chunk_len); match &mut chunk_filters[start_chunk] { f @ (ChunkFilter::All | ChunkFilter::None) => { @@ -156,7 +156,7 @@ fn filter_indices( let chunk_offsets = array.chunk_offsets(); for set_index in indices { - let (chunk_id, index) = find_chunk_idx(set_index, &chunk_offsets)?; + let (chunk_id, index) = find_chunk_idx(set_index, chunk_offsets)?; if chunk_id != current_chunk_id { // Push the chunk we've accumulated. if !chunk_indices.is_empty() { @@ -186,14 +186,12 @@ fn filter_indices( /// Mirrors the find_chunk_idx method on ChunkedArray, but avoids all of the overhead /// from scalars, dtypes, and metadata cloning. -pub(crate) fn find_chunk_idx(idx: usize, chunk_ends: &[u64]) -> VortexResult<(usize, usize)> { +pub(crate) fn find_chunk_idx(idx: usize, chunk_ends: &[usize]) -> VortexResult<(usize, usize)> { let chunk_id = chunk_ends - .search_sorted(&(idx as u64), SearchSortedSide::Right)? + .search_sorted(&idx, SearchSortedSide::Right)? .to_ends_index(chunk_ends.len()) .saturating_sub(1); - let chunk_begin: usize = chunk_ends[chunk_id] - .try_into() - .vortex_expect("chunk end must fit in usize"); + let chunk_begin = chunk_ends[chunk_id]; let chunk_offset = idx - chunk_begin; Ok((chunk_id, chunk_offset)) diff --git a/vortex-array/src/arrays/chunked/compute/mask.rs b/vortex-array/src/arrays/chunked/compute/mask.rs index 31ddfcc496f..bec7b5e83d0 100644 --- a/vortex-array/src/arrays/chunked/compute/mask.rs +++ b/vortex-array/src/arrays/chunked/compute/mask.rs @@ -9,7 +9,8 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; -use crate::arrays::scalar_fn::ScalarFnArrayExt; +use crate::arrays::chunked::ChunkedArrayExt; +use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::scalar_fn::EmptyOptions; use crate::scalar_fn::fns::mask::Mask as MaskExpr; use crate::scalar_fn::fns::mask::MaskKernel; @@ -25,8 +26,8 @@ impl MaskKernel for Chunked { .iter_chunks() .enumerate() .map(|(i, chunk)| { - let start: usize = chunk_offsets[i].try_into()?; - let end: usize = chunk_offsets[i + 1].try_into()?; + let start = chunk_offsets[i]; + let end = chunk_offsets[i + 1]; let chunk_mask = mask.slice(start..end)?; MaskExpr.try_new_array(chunk.len(), EmptyOptions, [chunk.clone(), chunk_mask]) }) diff --git a/vortex-array/src/arrays/chunked/compute/rules.rs b/vortex-array/src/arrays/chunked/compute/rules.rs index 8bbd61889ef..8ef63a188c3 100644 --- a/vortex-array/src/arrays/chunked/compute/rules.rs +++ b/vortex-array/src/arrays/chunked/compute/rules.rs @@ -13,7 +13,9 @@ use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::ScalarFnArray; use crate::arrays::ScalarFnVTable; +use crate::arrays::chunked::ChunkedArrayExt; use crate::arrays::scalar_fn::AnyScalarFn; +use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::optimizer::ArrayOptimizer; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; diff --git a/vortex-array/src/arrays/chunked/compute/slice.rs b/vortex-array/src/arrays/chunked/compute/slice.rs index d8d2917801c..14919301bd1 100644 --- a/vortex-array/src/arrays/chunked/compute/slice.rs +++ b/vortex-array/src/arrays/chunked/compute/slice.rs @@ -12,6 +12,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; +use crate::arrays::chunked::ChunkedArrayExt; use crate::arrays::slice::SliceKernel; impl SliceKernel for Chunked { diff --git a/vortex-array/src/arrays/chunked/compute/take.rs b/vortex-array/src/arrays/chunked/compute/take.rs index 9e936eaf2d3..1cd48c57319 100644 --- a/vortex-array/src/arrays/chunked/compute/take.rs +++ b/vortex-array/src/arrays/chunked/compute/take.rs @@ -12,6 +12,7 @@ use crate::array::ArrayView; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; use crate::arrays::PrimitiveArray; +use crate::arrays::chunked::ChunkedArrayExt; use crate::arrays::dict::TakeExecute; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; @@ -59,9 +60,10 @@ fn take_chunked( for chunk_idx in 0..nchunks { let chunk_start = chunk_offsets[chunk_idx]; let chunk_end = chunk_offsets[chunk_idx + 1]; - let chunk_len = usize::try_from(chunk_end - chunk_start)?; + let chunk_len = chunk_end - chunk_start; + let chunk_end_u64 = u64::try_from(chunk_end)?; - let range_end = cursor + pairs[cursor..].partition_point(|&(v, _)| v < chunk_end); + let range_end = cursor + pairs[cursor..].partition_point(|&(v, _)| v < chunk_end_u64); let chunk_pairs = &pairs[cursor..range_end]; if !chunk_pairs.is_empty() { @@ -70,7 +72,7 @@ fn take_chunked( if cursor + i > 0 && val != pairs[cursor + i - 1].0 { dedup_idx += 1; } - let local = usize::try_from(val - chunk_start)?; + let local = usize::try_from(val)? - chunk_start; if local_indices.last() != Some(&local) { local_indices.push(local); } @@ -121,6 +123,7 @@ mod test { use crate::arrays::ChunkedArray; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; + use crate::arrays::chunked::ChunkedArrayExt; use crate::assert_arrays_eq; use crate::compute::conformance::take::test_take_conformance; use crate::dtype::FieldNames; diff --git a/vortex-array/src/arrays/chunked/compute/zip.rs b/vortex-array/src/arrays/chunked/compute/zip.rs index 31642475033..0979cb15031 100644 --- a/vortex-array/src/arrays/chunked/compute/zip.rs +++ b/vortex-array/src/arrays/chunked/compute/zip.rs @@ -9,6 +9,8 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; +use crate::arrays::chunked::ChunkedArrayExt; +use crate::arrays::chunked::paired_chunks::PairedChunksExt; use crate::builtins::ArrayBuiltins; use crate::scalar_fn::fns::zip::ZipKernel; @@ -54,6 +56,7 @@ mod tests { use crate::VortexSessionExecute; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; + use crate::arrays::chunked::ChunkedArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::Nullability; diff --git a/vortex-array/src/arrays/chunked/mod.rs b/vortex-array/src/arrays/chunked/mod.rs index e48c3488ecc..766d9f6575c 100644 --- a/vortex-array/src/arrays/chunked/mod.rs +++ b/vortex-array/src/arrays/chunked/mod.rs @@ -2,11 +2,12 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::ChunkedArrayExt; pub use array::ChunkedData; pub use vtable::ChunkedArray; pub(crate) mod compute; -mod paired_chunks; +pub(crate) mod paired_chunks; mod vtable; pub use vtable::Chunked; diff --git a/vortex-array/src/arrays/chunked/paired_chunks.rs b/vortex-array/src/arrays/chunked/paired_chunks.rs index 5adfd6b4059..f786d62b9fb 100644 --- a/vortex-array/src/arrays/chunked/paired_chunks.rs +++ b/vortex-array/src/arrays/chunked/paired_chunks.rs @@ -6,7 +6,7 @@ use std::ops::Range; use vortex_error::VortexResult; use crate::ArrayRef; -use crate::arrays::chunked::ChunkedData; +use crate::arrays::chunked::ChunkedArrayExt; pub(crate) struct AlignedPair { pub left: ArrayRef, @@ -67,22 +67,24 @@ pub(crate) struct PairedChunks { total_len: usize, } -impl ChunkedData { - pub(crate) fn paired_chunks(&self, other: &ChunkedData) -> PairedChunks { +pub(crate) trait PairedChunksExt: ChunkedArrayExt { + fn paired_chunks(&self, other: &T) -> PairedChunks { assert_eq!( - self.len(), - other.len(), + self.as_ref().len(), + other.as_ref().len(), "paired_chunks requires arrays of equal length" ); PairedChunks { left: ChunkCursor::new(self.chunks()), right: ChunkCursor::new(other.chunks()), pos: 0, - total_len: self.len(), + total_len: self.as_ref().len(), } } } +impl PairedChunksExt for T {} + impl Iterator for PairedChunks { type Item = VortexResult; @@ -120,6 +122,7 @@ mod tests { use crate::IntoArray; use crate::arrays::ChunkedArray; + use crate::arrays::chunked::paired_chunks::PairedChunksExt; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; @@ -253,6 +256,6 @@ mod tests { let right = ChunkedArray::try_new(vec![buffer![10i32, 20, 30].into_array()], i32_dtype()).unwrap(); - drop(left.paired_chunks(&right).collect::>()); + left.paired_chunks(&right).for_each(drop); } } diff --git a/vortex-array/src/arrays/chunked/tests.rs b/vortex-array/src/arrays/chunked/tests.rs index f1266a5373d..a20e775cf5c 100644 --- a/vortex-array/src/arrays/chunked/tests.rs +++ b/vortex-array/src/arrays/chunked/tests.rs @@ -8,14 +8,14 @@ use vortex_buffer::buffer; use crate::IntoArray; use crate::accessor::ArrayAccessor; -use crate::array::ArrayParts; -use crate::array::VTable; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; use crate::arrays::ListArray; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; use crate::arrays::VarBinViewArray; +use crate::arrays::chunked::ChunkedArrayExt; +use crate::arrays::struct_::StructArrayExt; use crate::assert_arrays_eq; use crate::canonical::ToCanonical; use crate::dtype::DType; @@ -36,6 +36,36 @@ fn chunked_array() -> ChunkedArray { .unwrap() } +#[test] +fn with_slot_rewrites_chunk_and_offsets() { + let array = chunked_array().into_array(); + + let replacement = buffer![10u64, 11, 12].into_array(); + let array = array.with_slot(1, replacement).unwrap(); + let array = array.as_::(); + + assert_eq!(array.nchunks(), 3); + assert_eq!(array.chunk_offsets(), [0, 3, 6, 9]); + assert_arrays_eq!( + array.chunk(0).clone(), + PrimitiveArray::from_iter([10u64, 11, 12]) + ); + assert_arrays_eq!( + array.array().clone(), + PrimitiveArray::from_iter([10u64, 11, 12, 4, 5, 6, 7, 8, 9]) + ); +} + +#[test] +fn with_slot_rejects_len_mismatch() { + let err = chunked_array() + .into_array() + .with_slot(1, buffer![10u64, 11].into_array()) + .unwrap_err(); + + assert!(err.to_string().contains("physical rewrite")); +} + #[test] fn slice_middle() { assert_arrays_eq!( @@ -197,37 +227,3 @@ pub fn pack_nested_lists() { assert_eq!(l1.scalar_at(0).unwrap(), canon_values.scalar_at(0).unwrap()); assert_eq!(l2.scalar_at(0).unwrap(), canon_values.scalar_at(1).unwrap()); } - -#[test] -fn with_slots_updates_nchunks_len_and_offsets() { - let orig = chunked_array(); - let slots = vec![ - Some(buffer![0u64, 4, 9].into_array()), - Some(buffer![10u64, 11, 12, 13].into_array()), - Some(buffer![14u64, 15, 16, 17, 18].into_array()), - ]; - let expected_nchunks = slots.len() - 1; - let expected_len = orig.len(); - let dtype = orig.dtype().clone(); - - let mut data = orig.into_data(); - ::with_slots(&mut data, slots).unwrap(); - let array = - ChunkedArray::try_from_parts(ArrayParts::new(Chunked, dtype, expected_len, data)).unwrap(); - - assert_eq!(array.nchunks(), expected_nchunks); - assert_eq!(array.len(), expected_len); - assert_eq!(array.chunk_offsets(), buffer![0u64, 4, 9]); - assert_arrays_eq!( - array.chunk(0).clone(), - PrimitiveArray::from_iter([10u64, 11, 12, 13]) - ); - assert_arrays_eq!( - array.chunk(1).clone(), - PrimitiveArray::from_iter([14u64, 15, 16, 17, 18]) - ); - assert_arrays_eq!( - array, - PrimitiveArray::from_iter([10u64, 11, 12, 13, 14, 15, 16, 17, 18]) - ); -} diff --git a/vortex-array/src/arrays/chunked/vtable/canonical.rs b/vortex-array/src/arrays/chunked/vtable/canonical.rs index 2c6ba131e84..73a9b1c856a 100644 --- a/vortex-array/src/arrays/chunked/vtable/canonical.rs +++ b/vortex-array/src/arrays/chunked/vtable/canonical.rs @@ -15,7 +15,10 @@ use crate::arrays::ChunkedArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; +use crate::arrays::chunked::ChunkedArrayExt; +use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::ListViewRebuildMode; +use crate::arrays::struct_::StructArrayExt; use crate::builders::builder_with_capacity; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; @@ -199,6 +202,7 @@ mod tests { use crate::arrays::ListArray; use crate::arrays::StructArray; use crate::arrays::VarBinViewArray; + use crate::arrays::struct_::StructArrayExt; use crate::dtype::DType::List; use crate::dtype::DType::Primitive; use crate::dtype::Nullability::NonNullable; diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index 6349bfa97ec..b152311e6bb 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::hash::Hasher; + use itertools::Itertools; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -9,6 +12,8 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::ArrayEq; +use crate::ArrayHash; use crate::ArrayRef; use crate::Canonical; use crate::ExecutionCtx; @@ -18,25 +23,22 @@ use crate::Precision; use crate::ToCanonical; use crate::array::Array; use crate::array::ArrayId; +use crate::array::ArrayParts; use crate::array::ArrayView; use crate::array::VTable; -use crate::arrays::PrimitiveArray; +use crate::arrays::chunked::ChunkedArrayExt; use crate::arrays::chunked::ChunkedData; use crate::arrays::chunked::array::CHUNK_OFFSETS_SLOT; use crate::arrays::chunked::array::CHUNKS_OFFSET; use crate::arrays::chunked::compute::kernel::PARENT_KERNELS; use crate::arrays::chunked::compute::rules::PARENT_RULES; use crate::arrays::chunked::vtable::canonical::_canonicalize; -use crate::arrays::primitive::PrimitiveData; use crate::buffer::BufferHandle; use crate::builders::ArrayBuilder; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; -use crate::hash::ArrayEq; -use crate::hash::ArrayHash; use crate::serde::ArrayChildren; -use crate::validity::Validity; use crate::vtable; mod canonical; mod operations; @@ -50,6 +52,21 @@ impl Chunked { pub const ID: ArrayId = ArrayId::new_ref("vortex.chunked"); } +impl ArrayHash for ChunkedData { + fn array_hash(&self, _state: &mut H, _precision: Precision) { + // Chunk offsets are cached derived data. Slot 0 already stores the logical offsets array, + // and ArrayInner hashing includes every slot before ArrayData. + } +} + +impl ArrayEq for ChunkedData { + fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { + // Chunk offsets are cached derived data. Slot 0 already stores the logical offsets array, + // and ArrayInner equality compares every slot before ArrayData. + true + } +} + impl VTable for Chunked { type ArrayData = ChunkedData; @@ -60,38 +77,6 @@ impl VTable for Chunked { Self::ID } - fn array_hash(array: &ChunkedData, state: &mut H, precision: Precision) { - PrimitiveArray::new( - array.chunk_offsets.to_buffer::(), - Validity::NonNullable, - ) - .into_array() - .array_hash(state, precision); - for chunk in &array.chunks { - chunk.array_hash(state, precision); - } - } - - fn array_eq(array: &ChunkedData, other: &ChunkedData, precision: Precision) -> bool { - PrimitiveArray::new( - array.chunk_offsets.to_buffer::(), - Validity::NonNullable, - ) - .into_array() - .array_eq( - &PrimitiveArray::new( - other.chunk_offsets.to_buffer::(), - Validity::NonNullable, - ) - .into_array(), - precision, - ) && array.chunks.len() == other.chunks.len() - && array - .iter_chunks() - .zip(other.iter_chunks()) - .all(|(a, b)| a.array_eq(b, precision)) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -108,32 +93,83 @@ impl VTable for Chunked { Ok(Some(vec![])) } - fn validate(&self, data: &ChunkedData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &ChunkedData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { vortex_ensure!( - data.len() == len, - "ChunkedArray length {} does not match outer length {}", - data.len(), - len + !slots.is_empty(), + "ChunkedArray must have at least a chunk offsets slot" + ); + let chunk_offsets = slots[CHUNK_OFFSETS_SLOT] + .as_ref() + .vortex_expect("validated chunk offsets slot"); + vortex_ensure!( + chunk_offsets.dtype() == &DType::Primitive(PType::U64, Nullability::NonNullable), + "ChunkedArray chunk offsets must be non-nullable u64, found {}", + chunk_offsets.dtype() + ); + vortex_ensure!( + chunk_offsets.len() == data.chunk_offsets.len(), + "ChunkedArray chunk offsets slot length {} does not match cached offsets length {}", + chunk_offsets.len(), + data.chunk_offsets.len() ); vortex_ensure!( - data.dtype() == dtype, - "ChunkedArray dtype {} does not match outer dtype {}", - data.dtype(), - dtype + data.chunk_offsets.len() == slots.len() - CHUNKS_OFFSET + 1, + "ChunkedArray chunk offsets length {} does not match {} chunks", + data.chunk_offsets.len(), + slots.len() - CHUNKS_OFFSET ); + vortex_ensure!( + data.chunk_offsets + .last() + .copied() + .vortex_expect("chunked arrays always have a leading 0 offset") + == len, + "ChunkedArray length {} does not match outer length {}", + data.chunk_offsets.last().copied().unwrap_or_default(), + len + ); + for (idx, (start, end)) in data + .chunk_offsets + .iter() + .copied() + .tuple_windows() + .enumerate() + { + let chunk = slots[CHUNKS_OFFSET + idx] + .as_ref() + .vortex_expect("validated chunk slot"); + vortex_ensure!( + chunk.dtype() == dtype, + "ChunkedArray chunk dtype {} does not match outer dtype {}", + chunk.dtype(), + dtype + ); + vortex_ensure!( + chunk.len() == end - start, + "ChunkedArray chunk {} len {} does not match offsets span {}", + idx, + chunk.len(), + end - start + ); + } Ok(()) } fn deserialize( &self, dtype: &DType, - _len: usize, + len: usize, metadata: &[u8], - _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { if !metadata.is_empty() { vortex_bail!( "ChunkedArray expects empty metadata, got {} bytes", @@ -145,45 +181,41 @@ impl VTable for Chunked { } let nchunks = children.len() - 1; - - // The first child contains the row offsets of the chunks - let chunk_offsets_array = children - .get( - 0, - &DType::Primitive(PType::U64, Nullability::NonNullable), - // 1 extra offset for the end of the last chunk - nchunks + 1, - )? - .to_primitive(); - - let chunk_offsets_buf = chunk_offsets_array.to_buffer::(); - - // The remaining children contain the actual data of the chunks - let chunks: Vec = chunk_offsets_buf + let chunk_offsets = children.get( + CHUNK_OFFSETS_SLOT, + &DType::Primitive(PType::U64, Nullability::NonNullable), + nchunks + 1, + )?; + let chunk_offsets_buf = chunk_offsets.to_primitive().to_buffer::(); + let chunk_offsets_usize = chunk_offsets_buf .iter() + .copied() + .map(|offset| { + usize::try_from(offset) + .map_err(|_| vortex_err!("chunk offset {offset} exceeds usize range")) + }) + .collect::>>()?; + let mut slots = Vec::with_capacity(children.len()); + slots.push(Some(chunk_offsets)); + for (idx, (start, end)) in chunk_offsets_usize + .iter() + .copied() .tuple_windows() .enumerate() - .map(|(idx, (start, end))| { - let chunk_len = usize::try_from(end - start) - .map_err(|_| vortex_err!("chunk_len {} exceeds usize range", end - start))?; - children.get(idx + 1, dtype, chunk_len) - }) - .try_collect()?; - - let chunk_offsets = PrimitiveData::new(chunk_offsets_buf, Validity::NonNullable); - - let slots = ChunkedData::make_slots(&chunk_offsets, &chunks); - // Construct directly using the struct fields to avoid recomputing chunk_offsets - Ok(ChunkedData { - empty_dtype: chunks.is_empty().then_some(dtype.clone()), - chunk_offsets, - chunks, - slots, - }) - } + { + let chunk_len = end - start; + slots.push(Some(children.get(idx + CHUNKS_OFFSET, dtype, chunk_len)?)); + } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + Ok(ArrayParts::new( + self.clone(), + dtype.clone(), + len, + ChunkedData { + chunk_offsets: chunk_offsets_usize, + }, + ) + .with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { @@ -193,38 +225,6 @@ impl VTable for Chunked { } } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - // Slots: chunk_offsets, then chunks... - vortex_ensure!(!slots.is_empty(), "Chunked array needs at least one slot"); - - let nchunks = slots.len() - CHUNKS_OFFSET; - let chunk_offsets_ref = slots[CHUNK_OFFSETS_SLOT] - .as_ref() - .ok_or_else(|| vortex_err!("chunk_offsets slot must not be None"))?; - let chunk_offsets_buf = chunk_offsets_ref.to_primitive().to_buffer::(); - - vortex_ensure!( - chunk_offsets_buf.len() == nchunks + 1, - "Expected {} chunk offsets, found {}", - nchunks + 1, - chunk_offsets_buf.len() - ); - - let chunks: Vec = slots[CHUNKS_OFFSET..] - .iter() - .map(|s| { - s.clone() - .ok_or_else(|| vortex_err!("chunk slot must not be None")) - }) - .try_collect()?; - array.chunk_offsets = PrimitiveData::new(chunk_offsets_buf, Validity::NonNullable); - array.empty_dtype = chunks.is_empty().then_some(array.dtype().clone()); - array.chunks = chunks; - array.slots = slots; - - Ok(()) - } - fn append_to_builder( array: ArrayView<'_, Self>, builder: &mut dyn ArrayBuilder, @@ -241,7 +241,7 @@ impl VTable for Chunked { } fn reduce(array: ArrayView<'_, Self>) -> VortexResult> { - Ok(match array.chunks.len() { + Ok(match array.nchunks() { 0 => Some(Canonical::empty(array.dtype()).into_array()), 1 => Some(array.chunk(0).clone()), _ => None, diff --git a/vortex-array/src/arrays/chunked/vtable/operations.rs b/vortex-array/src/arrays/chunked/vtable/operations.rs index f27b4e6a9a6..7c7095a05c7 100644 --- a/vortex-array/src/arrays/chunked/vtable/operations.rs +++ b/vortex-array/src/arrays/chunked/vtable/operations.rs @@ -7,6 +7,7 @@ use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::Chunked; +use crate::arrays::chunked::ChunkedArrayExt; use crate::scalar::Scalar; impl OperationsVTable for Chunked { diff --git a/vortex-array/src/arrays/chunked/vtable/validity.rs b/vortex-array/src/arrays/chunked/vtable/validity.rs index a250c98be89..5f5c11c7832 100644 --- a/vortex-array/src/arrays/chunked/vtable/validity.rs +++ b/vortex-array/src/arrays/chunked/vtable/validity.rs @@ -9,6 +9,7 @@ use crate::array::ArrayView; use crate::array::ValidityVTable; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; +use crate::arrays::chunked::ChunkedArrayExt; use crate::dtype::DType; use crate::dtype::Nullability; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/constant/array.rs b/vortex-array/src/arrays/constant/array.rs index 3841a9d2b02..1acc4e898bc 100644 --- a/vortex-array/src/arrays/constant/array.rs +++ b/vortex-array/src/arrays/constant/array.rs @@ -1,18 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use crate::ArrayRef; use crate::array::Array; use crate::array::ArrayParts; use crate::arrays::Constant; use crate::scalar::Scalar; -pub(super) const NUM_SLOTS: usize = 0; - #[derive(Clone, Debug)] pub struct ConstantData { pub(super) scalar: Scalar, - pub(super) slots: Vec>, } impl ConstantData { @@ -21,10 +17,7 @@ impl ConstantData { S: Into, { let scalar = scalar.into(); - Self { - scalar, - slots: vec![], - } + Self { scalar } } /// Returns the [`Scalar`] value of this constant array. diff --git a/vortex-array/src/arrays/constant/vtable/canonical.rs b/vortex-array/src/arrays/constant/vtable/canonical.rs index 5ca796da366..137f48fd8eb 100644 --- a/vortex-array/src/arrays/constant/vtable/canonical.rs +++ b/vortex-array/src/arrays/constant/vtable/canonical.rs @@ -326,7 +326,10 @@ mod tests { use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; use crate::arrays::VarBinArray; + use crate::arrays::fixed_size_list::FixedSizeListArrayExt; + use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::ListViewRebuildMode; + use crate::arrays::struct_::StructArrayExt; use crate::assert_arrays_eq; use crate::canonical::ToCanonical; use crate::dtype::DType; diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index 55869cbd220..a41972a2557 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -3,6 +3,7 @@ use std::fmt::Debug; use std::hash::Hash; +use std::hash::Hasher; use vortex_buffer::ByteBufferMut; use vortex_error::VortexExpect; @@ -11,6 +12,8 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::ArrayEq; +use crate::ArrayHash; use crate::ArrayRef; use crate::ExecutionCtx; use crate::ExecutionResult; @@ -21,7 +24,6 @@ use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::constant::ConstantData; -use crate::arrays::constant::array::NUM_SLOTS; use crate::arrays::constant::compute::rules::PARENT_RULES; use crate::arrays::constant::vtable::canonical::constant_canonicalize; use crate::buffer::BufferHandle; @@ -53,6 +55,18 @@ impl Constant { pub const ID: ArrayId = ArrayId::new_ref("vortex.constant"); } +impl ArrayHash for ConstantData { + fn array_hash(&self, state: &mut H, _precision: Precision) { + self.scalar.hash(state); + } +} + +impl ArrayEq for ConstantData { + fn array_eq(&self, other: &Self, _precision: Precision) -> bool { + self.scalar == other.scalar + } +} + impl VTable for Constant { type ArrayData = ConstantData; @@ -63,7 +77,13 @@ impl VTable for Constant { Self::ID } - fn validate(&self, data: &ConstantData, dtype: &DType, _len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &ConstantData, + dtype: &DType, + _len: usize, + _slots: &[Option], + ) -> VortexResult<()> { vortex_ensure!( data.scalar.dtype() == dtype, "ConstantArray scalar dtype does not match outer dtype" @@ -71,18 +91,6 @@ impl VTable for Constant { Ok(()) } - fn array_hash( - array: &ConstantData, - state: &mut H, - _precision: Precision, - ) { - array.scalar.hash(state); - } - - fn array_eq(array: &ConstantData, other: &ConstantData, _precision: Precision) -> bool { - array.scalar == other.scalar - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 1 } @@ -103,24 +111,10 @@ impl VTable for Constant { } } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { vortex_panic!("ConstantArray slot_name index {idx} out of bounds") } - fn with_slots(_array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "ConstantArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - Ok(()) - } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { // HACK: Because the scalar is stored in the buffers, we do not need to serialize the // metadata at all. @@ -130,13 +124,13 @@ impl VTable for Constant { fn deserialize( &self, dtype: &DType, - _len: usize, + len: usize, _metadata: &[u8], buffers: &[BufferHandle], _children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { vortex_ensure!( buffers.len() == 1, "Expected 1 buffer, got {}", @@ -149,7 +143,12 @@ impl VTable for Constant { let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype, session)?; let scalar = Scalar::try_new(dtype.clone(), scalar_value)?; - Ok(ConstantData::new(scalar)) + Ok(crate::array::ArrayParts::new( + self.clone(), + dtype.clone(), + len, + ConstantData::new(scalar), + )) } fn reduce_parent( diff --git a/vortex-array/src/arrays/datetime/mod.rs b/vortex-array/src/arrays/datetime/mod.rs index abb3c18e68c..25913deb8b2 100644 --- a/vortex-array/src/arrays/datetime/mod.rs +++ b/vortex-array/src/arrays/datetime/mod.rs @@ -14,6 +14,7 @@ use crate::ArrayRef; use crate::IntoArray; use crate::arrays::Extension; use crate::arrays::ExtensionArray; +use crate::arrays::extension::ExtensionArrayExt; use crate::dtype::DType; use crate::dtype::extension::ExtDTypeRef; use crate::extension::datetime::AnyTemporal; diff --git a/vortex-array/src/arrays/decimal/array.rs b/vortex-array/src/arrays/decimal/array.rs index df5be8ce202..4a1de72b520 100644 --- a/vortex-array/src/arrays/decimal/array.rs +++ b/vortex-array/src/arrays/decimal/array.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use itertools::Itertools; +use vortex_buffer::Alignment; use vortex_buffer::BitBufferMut; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; @@ -16,6 +17,7 @@ use crate::ExecutionCtx; use crate::IntoArray; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Decimal; @@ -98,9 +100,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; /// ``` #[derive(Clone, Debug)] pub struct DecimalData { - pub(super) slots: Vec>, pub(super) decimal_dtype: DecimalDType, - pub(super) nullability: Nullability, pub(super) values: BufferHandle, pub(super) values_type: DecimalType, } @@ -112,6 +112,51 @@ pub struct DecimalDataParts { pub validity: Validity, } +pub trait DecimalArrayExt: TypedArrayRef { + fn decimal_dtype(&self) -> DecimalDType { + match self.as_ref().dtype() { + DType::Decimal(decimal_dtype, _) => *decimal_dtype, + _ => unreachable!("DecimalArrayExt requires a decimal dtype"), + } + } + + fn nullability(&self) -> Nullability { + match self.as_ref().dtype() { + DType::Decimal(_, nullability) => *nullability, + _ => unreachable!("DecimalArrayExt requires a decimal dtype"), + } + } + + fn validity_child(&self) -> Option<&ArrayRef> { + self.as_ref().slots()[VALIDITY_SLOT].as_ref() + } + + fn validity(&self) -> Validity { + child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) + } + + fn values_type(&self) -> DecimalType { + self.values_type + } + + fn precision(&self) -> u8 { + self.decimal_dtype().precision() + } + + fn scale(&self) -> i8 { + self.decimal_dtype().scale() + } + + fn buffer_handle(&self) -> &BufferHandle { + &self.values + } + + fn buffer(&self) -> Buffer { + DecimalData::buffer::(self) + } +} +impl> DecimalArrayExt for T {} + impl DecimalData { /// Build the slots vector for this array. pub(super) fn make_slots(validity: &Validity, len: usize) -> Vec> { @@ -124,13 +169,8 @@ impl DecimalData { /// /// Panics if the provided components do not satisfy the invariants documented in /// [`DecimalArray::new_unchecked`]. - pub fn new( - buffer: Buffer, - decimal_dtype: DecimalDType, - validity: Validity, - ) -> Self { - Self::try_new(buffer, decimal_dtype, validity) - .vortex_expect("DecimalArray construction failed") + pub fn new(buffer: Buffer, decimal_dtype: DecimalDType) -> Self { + Self::try_new(buffer, decimal_dtype).vortex_expect("DecimalArray construction failed") } /// Creates a new [`DecimalArray`] from a [`BufferHandle`] of values that may live in @@ -144,9 +184,8 @@ impl DecimalData { values: BufferHandle, values_type: DecimalType, decimal_dtype: DecimalDType, - validity: Validity, ) -> Self { - Self::try_new_handle(values, values_type, decimal_dtype, validity) + Self::try_new_handle(values, values_type, decimal_dtype) .vortex_expect("DecimalArray construction failed") } @@ -161,12 +200,11 @@ impl DecimalData { pub fn try_new( buffer: Buffer, decimal_dtype: DecimalDType, - validity: Validity, ) -> VortexResult { let values = BufferHandle::new_host(buffer.into_byte_buffer()); let values_type = T::DECIMAL_TYPE; - Self::try_new_handle(values, values_type, decimal_dtype, validity) + Self::try_new_handle(values, values_type, decimal_dtype) } /// Constructs a new `DecimalArray` with validation from a [`BufferHandle`]. @@ -180,12 +218,11 @@ impl DecimalData { values: BufferHandle, values_type: DecimalType, decimal_dtype: DecimalDType, - validity: Validity, ) -> VortexResult { - Self::validate(&values, values_type, &validity)?; + Self::validate(&values, values_type)?; // SAFETY: validate ensures all invariants are met. - Ok(unsafe { Self::new_unchecked_handle(values, values_type, decimal_dtype, validity) }) + Ok(unsafe { Self::new_unchecked_handle(values, values_type, decimal_dtype) }) } /// Creates a new [`DecimalArray`] without validation from these components: @@ -204,7 +241,6 @@ impl DecimalData { pub unsafe fn new_unchecked( buffer: Buffer, decimal_dtype: DecimalDType, - validity: Validity, ) -> Self { // SAFETY: new_unchecked_handle inherits the safety guarantees of new_unchecked unsafe { @@ -212,7 +248,6 @@ impl DecimalData { BufferHandle::new_host(buffer.into_byte_buffer()), T::DECIMAL_TYPE, decimal_dtype, - validity, ) } } @@ -230,42 +265,34 @@ impl DecimalData { values: BufferHandle, values_type: DecimalType, decimal_dtype: DecimalDType, - validity: Validity, ) -> Self { - #[cfg(debug_assertions)] - { - Self::validate(&values, values_type, &validity) - .vortex_expect("[Debug Assertion]: Invalid `DecimalArray` parameters"); - } - - let len = values.len() / values_type.byte_width(); Self { - slots: Self::make_slots(&validity, len), + decimal_dtype, values, values_type, - decimal_dtype, - nullability: validity.nullability(), } } /// Validates the components that would be used to create a [`DecimalArray`] from a byte buffer. /// /// This function checks all the invariants required by [`DecimalArray::new_unchecked`]. - fn validate( - buffer: &BufferHandle, - values_type: DecimalType, - validity: &Validity, - ) -> VortexResult<()> { - if let Some(validity_len) = validity.maybe_len() { - let expected_len = values_type.byte_width() * validity_len; + fn validate(buffer: &BufferHandle, values_type: DecimalType) -> VortexResult<()> { + let byte_width = values_type.byte_width(); + vortex_ensure!( + buffer.len().is_multiple_of(byte_width), + InvalidArgument: "decimal buffer size {} is not divisible by element width {}", + buffer.len(), + byte_width, + ); + match_each_decimal_value_type!(values_type, |D| { vortex_ensure!( - buffer.len() == expected_len, - InvalidArgument: "expected buffer of size {} bytes, was {} bytes", - expected_len, - buffer.len(), + buffer.is_aligned_to(Alignment::of::()), + InvalidArgument: "decimal buffer alignment {:?} is invalid for values type {:?}", + buffer.alignment(), + D::DECIMAL_TYPE, ); - } - + Ok::<(), vortex_error::VortexError>(()) + })?; Ok(()) } @@ -282,7 +309,6 @@ impl DecimalData { byte_buffer: ByteBuffer, values_type: DecimalType, decimal_dtype: DecimalDType, - validity: Validity, ) -> Self { // SAFETY: inherits the same safety contract as `new_unchecked_from_byte_buffer` unsafe { @@ -290,7 +316,6 @@ impl DecimalData { BufferHandle::new_host(byte_buffer), values_type, decimal_dtype, - validity, ) } } @@ -300,31 +325,11 @@ impl DecimalData { self.values.len() / self.values_type.byte_width() } - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - DType::Decimal(self.decimal_dtype, self.nullability) - } - /// Returns `true` if this array is empty. pub fn is_empty(&self) -> bool { self.len() == 0 } - /// Reconstructs the validity from the slot state. - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) - } - - pub fn into_parts(self) -> DecimalDataParts { - let validity = self.validity(); - DecimalDataParts { - decimal_dtype: self.decimal_dtype, - values: self.values, - values_type: self.values_type, - validity, - } - } - /// Returns the underlying [`ByteBuffer`] of the array. pub fn buffer_handle(&self) -> &BufferHandle { &self.values @@ -341,100 +346,36 @@ impl DecimalData { Buffer::::from_byte_buffer(self.values.as_host().clone()) } - /// Returns the decimal type information - pub fn decimal_dtype(&self) -> DecimalDType { - self.decimal_dtype - } - /// Return the `DecimalType` used to represent the values in the array. pub fn values_type(&self) -> DecimalType { self.values_type } + /// Returns the decimal type information. + pub fn decimal_dtype(&self) -> DecimalDType { + self.decimal_dtype + } + pub fn precision(&self) -> u8 { - self.decimal_dtype().precision() + self.decimal_dtype.precision() } pub fn scale(&self) -> i8 { - self.decimal_dtype().scale() + self.decimal_dtype.scale() } +} - pub fn from_iter>( - iter: I, - decimal_dtype: DecimalDType, - ) -> Self { - let iter = iter.into_iter(); - - Self::new( - BufferMut::from_iter(iter).freeze(), +impl Array { + pub fn into_data_parts(self) -> DecimalDataParts { + let validity = DecimalArrayExt::validity(&self); + let decimal_dtype = DecimalArrayExt::decimal_dtype(&self); + let data = self.into_data(); + DecimalDataParts { decimal_dtype, - Validity::NonNullable, - ) - } - - pub fn from_option_iter>>( - iter: I, - decimal_dtype: DecimalDType, - ) -> Self { - let iter = iter.into_iter(); - let mut values = BufferMut::with_capacity(iter.size_hint().0); - let mut validity = BitBufferMut::with_capacity(values.capacity()); - - for i in iter { - match i { - None => { - validity.append(false); - values.push(T::default()); - } - Some(e) => { - validity.append(true); - values.push(e); - } - } + values: data.values, + values_type: data.values_type, + validity, } - Self::new( - values.freeze(), - decimal_dtype, - Validity::from(validity.freeze()), - ) - } - - #[expect( - clippy::cognitive_complexity, - reason = "complexity from nested match_each_* macros" - )] - pub fn patch(self, patches: &Patches, ctx: &mut ExecutionCtx) -> VortexResult { - let offset = patches.offset(); - let patch_indices = patches.indices().clone().execute::(ctx)?; - let patch_values = patches.values().clone().execute::(ctx)?; - - let patch_validity = patch_values.validity()?; - let patched_validity = self.validity().patch( - self.len(), - offset, - &patch_indices.clone().into_array(), - &patch_validity, - ctx, - )?; - assert_eq!(self.decimal_dtype(), patch_values.decimal_dtype()); - - Ok(match_each_integer_ptype!(patch_indices.ptype(), |I| { - let patch_indices = patch_indices.as_slice::(); - match_each_decimal_value_type!(patch_values.values_type(), |PatchDVT| { - let patch_values = patch_values.buffer::(); - match_each_decimal_value_type!(self.values_type(), |ValuesDVT| { - let buffer = self.buffer::().into_mut(); - patch_typed( - buffer, - self.decimal_dtype(), - patch_indices, - offset, - patch_values, - patched_validity, - ) - }) - }) - })) } } @@ -445,10 +386,8 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { - let dtype = DType::Decimal(decimal_dtype, validity.nullability()); - let len = buffer.len(); - let data = DecimalData::new(buffer, decimal_dtype, validity); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } + Self::try_new(buffer, decimal_dtype, validity) + .vortex_expect("DecimalArray construction failed") } /// Creates a new [`DecimalArray`] without validation. @@ -463,8 +402,13 @@ impl Array { ) -> Self { let dtype = DType::Decimal(decimal_dtype, validity.nullability()); let len = buffer.len(); - let data = unsafe { DecimalData::new_unchecked(buffer, decimal_dtype, validity) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } + let slots = DecimalData::make_slots(&validity, len); + let data = unsafe { DecimalData::new_unchecked(buffer, decimal_dtype) }; + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Decimal, dtype, len, data).with_slots(slots), + ) + } } /// Creates a new [`DecimalArray`] from a host-native buffer with validation. @@ -475,8 +419,9 @@ impl Array { ) -> VortexResult { let dtype = DType::Decimal(decimal_dtype, validity.nullability()); let len = buffer.len(); - let data = DecimalData::try_new(buffer, decimal_dtype, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) }) + let slots = DecimalData::make_slots(&validity, len); + let data = DecimalData::try_new(buffer, decimal_dtype)?; + Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data).with_slots(slots)) } /// Creates a new [`DecimalArray`] from an iterator of values. @@ -488,10 +433,11 @@ impl Array { iter: I, decimal_dtype: DecimalDType, ) -> Self { - let data = DecimalData::from_iter(iter, decimal_dtype); - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } + Self::new( + BufferMut::from_iter(iter).freeze(), + decimal_dtype, + Validity::NonNullable, + ) } /// Creates a new [`DecimalArray`] from an iterator of optional values. @@ -499,10 +445,28 @@ impl Array { iter: I, decimal_dtype: DecimalDType, ) -> Self { - let data = DecimalData::from_option_iter(iter, decimal_dtype); - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } + let iter = iter.into_iter(); + let mut values = BufferMut::with_capacity(iter.size_hint().0); + let mut validity = BitBufferMut::with_capacity(values.capacity()); + + for value in iter { + match value { + Some(value) => { + values.push(value); + validity.append(true); + } + None => { + values.push(T::default()); + validity.append(false); + } + } + } + + Self::new( + values.freeze(), + decimal_dtype, + Validity::from(validity.freeze()), + ) } /// Creates a new [`DecimalArray`] from a [`BufferHandle`]. @@ -512,10 +476,22 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { + Self::try_new_handle(values, values_type, decimal_dtype, validity) + .vortex_expect("DecimalArray construction failed") + } + + /// Creates a new [`DecimalArray`] from a [`BufferHandle`] with validation. + pub fn try_new_handle( + values: BufferHandle, + values_type: DecimalType, + decimal_dtype: DecimalDType, + validity: Validity, + ) -> VortexResult { let dtype = DType::Decimal(decimal_dtype, validity.nullability()); let len = values.len() / values_type.byte_width(); - let data = DecimalData::new_handle(values, values_type, decimal_dtype, validity); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } + let slots = DecimalData::make_slots(&validity, len); + let data = DecimalData::try_new_handle(values, values_type, decimal_dtype)?; + Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data).with_slots(slots)) } /// Creates a new [`DecimalArray`] without validation from a [`BufferHandle`]. @@ -531,10 +507,59 @@ impl Array { ) -> Self { let dtype = DType::Decimal(decimal_dtype, validity.nullability()); let len = values.len() / values_type.byte_width(); - let data = unsafe { - DecimalData::new_unchecked_handle(values, values_type, decimal_dtype, validity) - }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } + let slots = DecimalData::make_slots(&validity, len); + let data = unsafe { DecimalData::new_unchecked_handle(values, values_type, decimal_dtype) }; + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Decimal, dtype, len, data).with_slots(slots), + ) + } + } + + #[allow( + clippy::cognitive_complexity, + reason = "patching depends on both patch and value physical types" + )] + pub fn patch(self, patches: &Patches, ctx: &mut ExecutionCtx) -> VortexResult { + let offset = patches.offset(); + let dtype = self.dtype().clone(); + let len = self.len(); + let patch_indices = patches.indices().clone().execute::(ctx)?; + let patch_values = patches.values().clone().execute::(ctx)?; + + let patch_validity = patch_values.validity()?; + let patched_validity = self.validity()?.patch( + self.len(), + offset, + &patch_indices.clone().into_array(), + &patch_validity, + ctx, + )?; + assert_eq!(self.decimal_dtype(), patch_values.decimal_dtype()); + + let data = self.into_data(); + let data = match_each_integer_ptype!(patch_indices.ptype(), |I| { + let patch_indices = patch_indices.as_slice::(); + match_each_decimal_value_type!(patch_values.values_type(), |PatchDVT| { + let patch_values = patch_values.buffer::(); + match_each_decimal_value_type!(data.values_type(), |ValuesDVT| { + let buffer = data.buffer::().into_mut(); + patch_typed( + buffer, + data.decimal_dtype(), + patch_indices, + offset, + patch_values, + ) + }) + }) + }); + let slots = DecimalData::make_slots(&patched_validity, len); + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Decimal, dtype, len, data).with_slots(slots), + ) + }) } } @@ -544,7 +569,6 @@ fn patch_typed( patch_indices: &[I], patch_indices_offset: usize, patch_values: Buffer, - patched_validity: Validity, ) -> DecimalData where I: IntegerPType, @@ -565,5 +589,5 @@ where ); } - DecimalData::new(buffer.freeze(), decimal_dtype, patched_validity) + DecimalData::new(buffer.freeze(), decimal_dtype) } diff --git a/vortex-array/src/arrays/decimal/mod.rs b/vortex-array/src/arrays/decimal/mod.rs index a0b90613f52..489a67a0531 100644 --- a/vortex-array/src/arrays/decimal/mod.rs +++ b/vortex-array/src/arrays/decimal/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::DecimalArrayExt; pub use array::DecimalData; pub use array::DecimalDataParts; pub use vtable::DecimalArray; diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index dc7a46c779d..789c312179a 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::hash::Hasher; + use kernel::PARENT_KERNELS; use prost::Message; use vortex_buffer::Alignment; @@ -33,7 +35,6 @@ use std::hash::Hash; use crate::Precision; use crate::array::ArrayId; -use crate::arrays::decimal::array::NUM_SLOTS; use crate::arrays::decimal::array::SLOT_NAMES; use crate::arrays::decimal::compute::rules::RULES; use crate::hash::ArrayEq; @@ -47,6 +48,19 @@ pub struct DecimalMetadata { pub(super) values_type: i32, } +impl ArrayHash for DecimalData { + fn array_hash(&self, state: &mut H, precision: Precision) { + self.values.array_hash(state, precision); + std::mem::discriminant(&self.values_type).hash(state); + } +} + +impl ArrayEq for DecimalData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + self.values.array_eq(&other.values, precision) && self.values_type == other.values_type + } +} + impl VTable for Decimal { type ArrayData = DecimalData; @@ -57,18 +71,6 @@ impl VTable for Decimal { Self::ID } - fn array_hash(array: &DecimalData, state: &mut H, precision: Precision) { - array.values.array_hash(state, precision); - std::mem::discriminant(&array.values_type).hash(state); - array.validity().array_hash(state, precision); - } - - fn array_eq(array: &DecimalData, other: &DecimalData, precision: Precision) -> bool { - array.values.array_eq(&other.values, precision) - && array.values_type == other.values_type - && array.validity().array_eq(&other.validity(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 1 } @@ -96,21 +98,33 @@ impl VTable for Decimal { )) } - fn validate(&self, data: &DecimalData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &DecimalData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let DType::Decimal(_, nullability) = dtype else { + vortex_bail!("Expected decimal dtype, got {dtype:?}"); + }; vortex_ensure!( data.len() == len, + InvalidArgument: "DecimalArray length {} does not match outer length {}", data.len(), len ); - - let actual_dtype = data.dtype(); - vortex_ensure!( - &actual_dtype == dtype, - "DecimalArray dtype {} does not match outer dtype {}", - actual_dtype, - dtype - ); + let validity = crate::array::child_to_validity(&slots[0], *nullability); + if let Some(validity_len) = validity.maybe_len() { + vortex_ensure!( + validity_len == len, + InvalidArgument: + "DecimalArray validity len {} does not match outer length {}", + validity_len, + len + ); + } Ok(()) } @@ -124,7 +138,7 @@ impl VTable for Decimal { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = DecimalMetadata::decode(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); @@ -144,36 +158,23 @@ impl VTable for Decimal { vortex_bail!("Expected Decimal dtype, got {:?}", dtype) }; - match_each_decimal_value_type!(metadata.values_type(), |D| { + let slots = DecimalData::make_slots(&validity, len); + let data = match_each_decimal_value_type!(metadata.values_type(), |D| { // Check and reinterpret-cast the buffer vortex_ensure!( values.is_aligned_to(Alignment::of::()), "DecimalArray buffer not aligned for values type {:?}", D::DECIMAL_TYPE ); - DecimalData::try_new_handle(values, metadata.values_type(), *decimal_dtype, validity) - }) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + DecimalData::try_new_handle(values, metadata.values_type(), *decimal_dtype) + })?; + Ok(crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "DecimalArray expects {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done(array)) } diff --git a/vortex-array/src/arrays/decimal/vtable/validity.rs b/vortex-array/src/arrays/decimal/vtable/validity.rs index 5479ac890f3..500fb191fad 100644 --- a/vortex-array/src/arrays/decimal/vtable/validity.rs +++ b/vortex-array/src/arrays/decimal/vtable/validity.rs @@ -5,11 +5,12 @@ use vortex_error::VortexResult; use crate::array::ArrayView; use crate::array::ValidityVTable; +use crate::arrays::decimal::DecimalArrayExt; use crate::arrays::decimal::vtable::Decimal; use crate::validity::Validity; impl ValidityVTable for Decimal { fn validity(array: ArrayView<'_, Decimal>) -> VortexResult { - Ok(array.data().validity()) + Ok(DecimalArrayExt::validity(&array)) } } diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index a791d79b798..84e4c141470 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -12,6 +12,7 @@ use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::arrays::Dict; use crate::dtype::DType; use crate::dtype::PType; @@ -42,7 +43,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["codes", "values"]; #[derive(Debug, Clone)] pub struct DictData { - pub(super) slots: Vec>, /// Indicates whether all dictionary values are definitely referenced by at least one code. /// `true` = all values are referenced (computed during encoding). /// `false` = unknown/might have unreferenced values. @@ -51,11 +51,6 @@ pub struct DictData { pub(super) all_values_referenced: bool, } -pub struct DictDataParts { - pub codes: ArrayRef, - pub values: ArrayRef, -} - impl DictData { /// Build a new `DictArray` without validating the codes or values. /// @@ -63,9 +58,8 @@ impl DictData { /// This should be called only when you can guarantee the invariants checked /// by the safe `DictArray::try_new` constructor are valid, for example when /// you are filtering or slicing an existing valid `DictArray`. - pub unsafe fn new_unchecked(codes: ArrayRef, values: ArrayRef) -> Self { + pub unsafe fn new_unchecked() -> Self { Self { - slots: vec![Some(codes), Some(values)], all_values_referenced: false, } } @@ -81,14 +75,6 @@ impl DictData { /// that all values are referenced. pub unsafe fn set_all_values_referenced(mut self, all_values_referenced: bool) -> Self { self.all_values_referenced = all_values_referenced; - - #[cfg(debug_assertions)] - { - use vortex_error::VortexExpect; - self.validate_all_values_referenced() - .vortex_expect("validation should succeed when all values are referenced") - } - self } @@ -96,8 +82,8 @@ impl DictData { /// /// This constructor will panic if `codes` or `values` do not pass validation for building /// a new `DictArray`. See `DictArray::try_new` for a description of the error conditions. - pub fn new(codes: ArrayRef, values: ArrayRef) -> Self { - Self::try_new(codes, values).vortex_expect("DictArray new") + pub fn new(codes_dtype: &DType) -> Self { + Self::try_new(codes_dtype).vortex_expect("DictArray new") } /// Build a new `DictArray` from its components, `codes` and `values`. @@ -111,75 +97,35 @@ impl DictData { /// of the `values` array. Otherwise, this constructor returns an error. /// /// It is an error to provide a nullable `codes` with non-nullable `values`. - pub(crate) fn try_new(codes: ArrayRef, values: ArrayRef) -> VortexResult { - if !codes.dtype().is_int() { - vortex_bail!(MismatchedTypes: "int", codes.dtype()); + pub(crate) fn try_new(codes_dtype: &DType) -> VortexResult { + if !codes_dtype.is_int() { + vortex_bail!(MismatchedTypes: "int", codes_dtype); } - Ok(unsafe { Self::new_unchecked(codes, values) }) - } - - /// Returns the length of this array. - pub fn len(&self) -> usize { - self.codes().len() - } - - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - self.values() - .dtype() - .union_nullability(self.codes().dtype().nullability()) - } - - /// Returns `true` if this array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn into_parts(mut self) -> DictDataParts { - DictDataParts { - codes: self.slots[CODES_SLOT] - .take() - .vortex_expect("DictArray codes slot"), - values: self.slots[VALUES_SLOT] - .take() - .vortex_expect("DictArray values slot"), - } + Ok(unsafe { Self::new_unchecked() }) } +} - #[inline] - pub fn codes(&self) -> &ArrayRef { - self.slots[CODES_SLOT] +pub trait DictArrayExt: TypedArrayRef { + fn codes(&self) -> &ArrayRef { + self.as_ref().slots()[CODES_SLOT] .as_ref() .vortex_expect("DictArray codes slot") } - #[inline] - pub fn values(&self) -> &ArrayRef { - self.slots[VALUES_SLOT] + fn values(&self) -> &ArrayRef { + self.as_ref().slots()[VALUES_SLOT] .as_ref() .vortex_expect("DictArray values slot") } - /// Returns `true` if all dictionary values are definitely referenced by at least one code. - /// - /// When `true`, operations like min/max can safely operate on all values without needing to - /// compute which values are actually referenced. When `false`, it is unknown whether all - /// values are referenced (conservative default). #[inline] - pub fn has_all_values_referenced(&self) -> bool { + fn has_all_values_referenced(&self) -> bool { self.all_values_referenced } - /// Validates that the `all_values_referenced` flag matches reality. - /// - /// Returns `Ok(())` if the flag is consistent with the actual referenced values, - /// or an error describing the mismatch. - /// - /// This is primarily useful for testing and debugging. - pub fn validate_all_values_referenced(&self) -> VortexResult<()> { - if self.all_values_referenced { - // Skip host-only validation when codes are not host-resident. + fn validate_all_values_referenced(&self) -> VortexResult<()> { + if self.has_all_values_referenced() { if !self.codes().is_host() { return Ok(()); } @@ -192,62 +138,17 @@ impl DictData { Ok(()) } -} - -impl Array { - /// Build a new `DictArray` from its components, `codes` and `values`. - pub fn new(codes: ArrayRef, values: ArrayRef) -> Self { - let data = DictData::new(codes, values); - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) } - } - - /// Build a new `DictArray` from its components, `codes` and `values`. - pub fn try_new(codes: ArrayRef, values: ArrayRef) -> VortexResult { - let data = DictData::try_new(codes, values)?; - let dtype = data.dtype(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) }) - } - - /// Build a new `DictArray` without validating the codes or values. - /// - /// # Safety - /// - /// See [`DictData::new_unchecked`]. - pub unsafe fn new_unchecked(codes: ArrayRef, values: ArrayRef) -> Self { - let data = unsafe { DictData::new_unchecked(codes, values) }; - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) } - } - /// Set whether all values in the dictionary are referenced by at least one code. - /// - /// # Safety - /// - /// See [`DictData::set_all_values_referenced`]. - pub unsafe fn set_all_values_referenced(self, all_values_referenced: bool) -> Self { - let dtype = self.dtype().clone(); - let len = self.len(); - let data = unsafe { - self.into_data() - .set_all_values_referenced(all_values_referenced) - }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) } - } -} - -impl DictData { - pub fn compute_referenced_values_mask(&self, referenced: bool) -> VortexResult { + #[allow( + clippy::cognitive_complexity, + reason = "branching depends on validity representation and code type" + )] + fn compute_referenced_values_mask(&self, referenced: bool) -> VortexResult { let codes_validity = self.codes().validity_mask()?; let codes_primitive = self.codes().to_primitive(); let values_len = self.values().len(); - // Initialize with the starting value: false for referenced, true for unreferenced let init_value = !referenced; - // Value to set when we find a referenced code: true for referenced, false for unreferenced let referenced_value = referenced; let mut values_vec = vec![init_value; values_len]; @@ -259,13 +160,13 @@ impl DictData { clippy::cast_sign_loss, reason = "codes are non-negative indices; a negative signed code would wrap to a large usize and panic on the bounds-checked array index" )] - for &code in codes_primitive.as_slice::

().iter() { - values_vec[code as usize] = referenced_value; + for &idx in codes_primitive.as_slice::

() { + values_vec[idx as usize] = referenced_value; } }); } AllOr::None => {} - AllOr::Some(buf) => { + AllOr::Some(mask) => { match_each_integer_ptype!(codes_primitive.ptype(), |P| { let codes = codes_primitive.as_slice::

(); @@ -274,14 +175,79 @@ impl DictData { clippy::cast_sign_loss, reason = "codes are non-negative indices; a negative signed code would wrap to a large usize and panic on the bounds-checked array index" )] - buf.set_indices().for_each(|idx| { + mask.set_indices().for_each(|idx| { values_vec[codes[idx] as usize] = referenced_value; - }) + }); }); } } - Ok(BitBuffer::collect_bool(values_len, |idx| values_vec[idx])) + Ok(BitBuffer::from(values_vec)) + } +} +impl> DictArrayExt for T {} + +impl Array { + /// Build a new `DictArray` from its components, `codes` and `values`. + pub fn new(codes: ArrayRef, values: ArrayRef) -> Self { + Self::try_new(codes, values).vortex_expect("DictArray new") + } + + /// Build a new `DictArray` from its components, `codes` and `values`. + pub fn try_new(codes: ArrayRef, values: ArrayRef) -> VortexResult { + let dtype = values + .dtype() + .union_nullability(codes.dtype().nullability()); + let len = codes.len(); + let data = DictData::try_new(codes.dtype())?; + Array::try_from_parts( + ArrayParts::new(Dict, dtype, len, data).with_slots(vec![Some(codes), Some(values)]), + ) + } + + /// Build a new `DictArray` without validating the codes or values. + /// + /// # Safety + /// + /// See [`DictData::new_unchecked`]. + pub unsafe fn new_unchecked(codes: ArrayRef, values: ArrayRef) -> Self { + let dtype = values + .dtype() + .union_nullability(codes.dtype().nullability()); + let len = codes.len(); + let data = unsafe { DictData::new_unchecked() }; + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Dict, dtype, len, data).with_slots(vec![Some(codes), Some(values)]), + ) + } + } + + /// Set whether all values in the dictionary are referenced by at least one code. + /// + /// # Safety + /// + /// See [`DictData::set_all_values_referenced`]. + pub unsafe fn set_all_values_referenced(self, all_values_referenced: bool) -> Self { + let dtype = self.dtype().clone(); + let len = self.len(); + let slots = self.slots().to_vec(); + let data = unsafe { + self.into_data() + .set_all_values_referenced(all_values_referenced) + }; + let array = unsafe { + Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data).with_slots(slots)) + }; + + #[cfg(debug_assertions)] + if all_values_referenced { + array + .validate_all_values_referenced() + .vortex_expect("validation should succeed when all values are referenced"); + } + + array } } diff --git a/vortex-array/src/arrays/dict/compute/cast.rs b/vortex-array/src/arrays/dict/compute/cast.rs index a7895c9844b..36af43741c6 100644 --- a/vortex-array/src/arrays/dict/compute/cast.rs +++ b/vortex-array/src/arrays/dict/compute/cast.rs @@ -8,6 +8,7 @@ use super::DictArray; use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; +use crate::arrays::dict::DictArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; @@ -54,6 +55,7 @@ mod tests { use crate::ToCanonical; use crate::arrays::Dict; use crate::arrays::PrimitiveArray; + use crate::arrays::dict::DictArrayExt; use crate::assert_arrays_eq; use crate::builders::dict::dict_encode; use crate::builtins::ArrayBuiltins; diff --git a/vortex-array/src/arrays/dict/compute/compare.rs b/vortex-array/src/arrays/dict/compute/compare.rs index 9e2c119ca7e..810cc09e828 100644 --- a/vortex-array/src/arrays/dict/compute/compare.rs +++ b/vortex-array/src/arrays/dict/compute/compare.rs @@ -11,6 +11,7 @@ use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::ConstantArray; +use crate::arrays::dict::DictArrayExt; use crate::builtins::ArrayBuiltins; use crate::scalar_fn::fns::binary::CompareKernel; use crate::scalar_fn::fns::operators::CompareOperator; diff --git a/vortex-array/src/arrays/dict/compute/fill_null.rs b/vortex-array/src/arrays/dict/compute/fill_null.rs index 36dad87b911..b0f2e61ef97 100644 --- a/vortex-array/src/arrays/dict/compute/fill_null.rs +++ b/vortex-array/src/arrays/dict/compute/fill_null.rs @@ -12,6 +12,8 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::BoolArray; use crate::arrays::ConstantArray; +use crate::arrays::bool::BoolArrayExt; +use crate::arrays::dict::DictArrayExt; use crate::builtins::ArrayBuiltins; use crate::match_each_integer_ptype; use crate::scalar::Scalar; diff --git a/vortex-array/src/arrays/dict/compute/is_constant.rs b/vortex-array/src/arrays/dict/compute/is_constant.rs index 302af0f11ba..0e7ed40bc95 100644 --- a/vortex-array/src/arrays/dict/compute/is_constant.rs +++ b/vortex-array/src/arrays/dict/compute/is_constant.rs @@ -11,6 +11,7 @@ use crate::aggregate_fn::fns::is_constant::IsConstant; use crate::aggregate_fn::fns::is_constant::is_constant; use crate::aggregate_fn::kernels::DynAggregateKernel; use crate::arrays::Dict; +use crate::arrays::dict::DictArrayExt; use crate::scalar::Scalar; /// Dict-specific is_constant kernel. diff --git a/vortex-array/src/arrays/dict/compute/is_sorted.rs b/vortex-array/src/arrays/dict/compute/is_sorted.rs index 6787e6213ed..8bfb0346182 100644 --- a/vortex-array/src/arrays/dict/compute/is_sorted.rs +++ b/vortex-array/src/arrays/dict/compute/is_sorted.rs @@ -11,6 +11,7 @@ use crate::aggregate_fn::fns::is_sorted::is_sorted; use crate::aggregate_fn::fns::is_sorted::is_strict_sorted; use crate::aggregate_fn::kernels::DynAggregateKernel; use crate::arrays::Dict; +use crate::arrays::dict::DictArrayExt; use crate::scalar::Scalar; /// Dict-specific is_sorted kernel. diff --git a/vortex-array/src/arrays/dict/compute/like.rs b/vortex-array/src/arrays/dict/compute/like.rs index 0d7113acd64..31ad470d8a4 100644 --- a/vortex-array/src/arrays/dict/compute/like.rs +++ b/vortex-array/src/arrays/dict/compute/like.rs @@ -9,7 +9,8 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::ConstantArray; -use crate::arrays::scalar_fn::ScalarFnArrayExt; +use crate::arrays::dict::DictArrayExt; +use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::optimizer::ArrayOptimizer; use crate::scalar_fn::fns::like::Like; use crate::scalar_fn::fns::like::LikeOptions; @@ -58,7 +59,7 @@ mod tests { use crate::arrays::DictArray; use crate::arrays::VarBinArray; use crate::arrays::dict::compute::like::ConstantArray; - use crate::arrays::scalar_fn::ScalarFnArrayExt; + use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::assert_arrays_eq; use crate::optimizer::ArrayOptimizer; use crate::scalar_fn::fns::like::Like; diff --git a/vortex-array/src/arrays/dict/compute/mask.rs b/vortex-array/src/arrays/dict/compute/mask.rs index 3d098f046bc..1019fdcbb64 100644 --- a/vortex-array/src/arrays/dict/compute/mask.rs +++ b/vortex-array/src/arrays/dict/compute/mask.rs @@ -8,7 +8,8 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Dict; use crate::arrays::DictArray; -use crate::arrays::scalar_fn::ScalarFnArrayExt; +use crate::arrays::dict::DictArrayExt; +use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::scalar_fn::EmptyOptions; use crate::scalar_fn::fns::mask::Mask as MaskExpr; use crate::scalar_fn::fns::mask::MaskReduce; diff --git a/vortex-array/src/arrays/dict/compute/min_max.rs b/vortex-array/src/arrays/dict/compute/min_max.rs index b54e7bfffdf..6f71a597812 100644 --- a/vortex-array/src/arrays/dict/compute/min_max.rs +++ b/vortex-array/src/arrays/dict/compute/min_max.rs @@ -12,6 +12,7 @@ use crate::aggregate_fn::fns::min_max::make_minmax_dtype; use crate::aggregate_fn::fns::min_max::min_max; use crate::aggregate_fn::kernels::DynAggregateKernel; use crate::arrays::Dict; +use crate::arrays::dict::DictArrayExt; use crate::scalar::Scalar; /// Dict-specific min/max kernel. diff --git a/vortex-array/src/arrays/dict/compute/mod.rs b/vortex-array/src/arrays/dict/compute/mod.rs index dc0bf9ac47f..1cbe9ff67ed 100644 --- a/vortex-array/src/arrays/dict/compute/mod.rs +++ b/vortex-array/src/arrays/dict/compute/mod.rs @@ -22,6 +22,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; +use crate::arrays::dict::DictArrayExt; use crate::arrays::filter::FilterReduce; impl TakeExecute for Dict { diff --git a/vortex-array/src/arrays/dict/compute/rules.rs b/vortex-array/src/arrays/dict/compute/rules.rs index 1bd638e168f..a62c5ad1034 100644 --- a/vortex-array/src/arrays/dict/compute/rules.rs +++ b/vortex-array/src/arrays/dict/compute/rules.rs @@ -14,8 +14,10 @@ use crate::arrays::Dict; use crate::arrays::DictArray; use crate::arrays::ScalarFnArray; use crate::arrays::ScalarFnVTable; +use crate::arrays::dict::DictArrayExt; use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::scalar_fn::AnyScalarFn; +use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::slice::SliceReduceAdaptor; use crate::builtins::ArrayBuiltins; use crate::optimizer::ArrayOptimizer; diff --git a/vortex-array/src/arrays/dict/compute/slice.rs b/vortex-array/src/arrays/dict/compute/slice.rs index c4e8b798d2b..4fdf3c8b8e8 100644 --- a/vortex-array/src/arrays/dict/compute/slice.rs +++ b/vortex-array/src/arrays/dict/compute/slice.rs @@ -12,6 +12,7 @@ use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Dict; use crate::arrays::DictArray; +use crate::arrays::dict::DictArrayExt; use crate::arrays::slice::SliceReduce; use crate::scalar::Scalar; diff --git a/vortex-array/src/arrays/dict/execute.rs b/vortex-array/src/arrays/dict/execute.rs index da2f549c62c..d95cd239a44 100644 --- a/vortex-array/src/arrays/dict/execute.rs +++ b/vortex-array/src/arrays/dict/execute.rs @@ -29,6 +29,7 @@ use crate::arrays::VarBinViewArray; use crate::arrays::VariantArray; use crate::arrays::dict::TakeExecute; use crate::arrays::dict::TakeReduce; +use crate::arrays::variant::VariantArrayExt; /// Take from a canonical array using indices (codes), returning a new canonical array. /// diff --git a/vortex-array/src/arrays/dict/take.rs b/vortex-array/src/arrays/dict/take.rs index b3e18f6a9a1..bed0018d548 100644 --- a/vortex-array/src/arrays/dict/take.rs +++ b/vortex-array/src/arrays/dict/take.rs @@ -11,6 +11,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::ConstantArray; +use crate::arrays::dict::DictArrayExt; use crate::expr::stats::Precision; use crate::expr::stats::Stat; use crate::expr::stats::StatsProvider; diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index 25be2053332..6f17c5c157a 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::hash::Hasher; + use kernel::PARENT_KERNELS; use prost::Message; use vortex_error::VortexExpect; @@ -12,12 +14,14 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use super::DictData; -use super::DictDataParts; use super::DictMetadata; -use super::array::NUM_SLOTS; +use super::array::CODES_SLOT; use super::array::SLOT_NAMES; +use super::array::VALUES_SLOT; use super::take_canonical; use crate::AnyCanonical; +use crate::ArrayEq; +use crate::ArrayHash; use crate::ArrayRef; use crate::Canonical; use crate::Precision; @@ -27,6 +31,7 @@ use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::ConstantArray; use crate::arrays::Primitive; +use crate::arrays::dict::DictArrayExt; use crate::arrays::dict::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -34,8 +39,6 @@ use crate::dtype::Nullability; use crate::dtype::PType; use crate::executor::ExecutionCtx; use crate::executor::ExecutionResult; -use crate::hash::ArrayEq; -use crate::hash::ArrayHash; use crate::require_child; use crate::scalar::Scalar; use crate::serde::ArrayChildren; @@ -53,6 +56,16 @@ impl Dict { pub const ID: ArrayId = ArrayId::new_ref("vortex.dict"); } +impl ArrayHash for DictData { + fn array_hash(&self, _state: &mut H, _precision: Precision) {} +} + +impl ArrayEq for DictData { + fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { + true + } +} + impl VTable for Dict { type ArrayData = DictData; @@ -63,25 +76,31 @@ impl VTable for Dict { Self::ID } - fn validate(&self, data: &DictData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!(data.codes().len() == len, "DictArray codes length mismatch"); + fn validate( + &self, + data: &DictData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + _ = data; + let codes = slots[CODES_SLOT] + .as_ref() + .vortex_expect("DictArray codes slot"); + let values = slots[VALUES_SLOT] + .as_ref() + .vortex_expect("DictArray values slot"); + vortex_ensure!(codes.len() == len, "DictArray codes length mismatch"); vortex_ensure!( - data.dtype() == *dtype, + values + .dtype() + .union_nullability(codes.dtype().nullability()) + == *dtype, "DictArray dtype does not match codes/values dtype" ); Ok(()) } - fn array_hash(array: &DictData, state: &mut H, precision: Precision) { - array.codes().array_hash(state, precision); - array.values().array_hash(state, precision); - } - - fn array_eq(array: &DictData, other: &DictData, precision: Precision) -> bool { - array.codes().array_eq(other.codes(), precision) - && array.values().array_eq(other.values(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -105,7 +124,7 @@ impl VTable for Dict { ) })?, is_nullable_codes: Some(array.codes().dtype().is_nullable()), - all_values_referenced: Some(array.all_values_referenced), + all_values_referenced: Some(array.has_all_values_referenced()), } .encode_to_vec(), )) @@ -120,7 +139,7 @@ impl VTable for Dict { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = DictMetadata::decode(metadata)?; if children.len() != 2 { vortex_bail!( @@ -139,31 +158,18 @@ impl VTable for Dict { let values = children.get(1, dtype, metadata.values_len as usize)?; let all_values_referenced = metadata.all_values_referenced.unwrap_or(false); - // SAFETY: We've validated the metadata and children. - Ok(unsafe { - DictData::new_unchecked(codes, values).set_all_values_referenced(all_values_referenced) - }) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + Ok( + crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, unsafe { + DictData::new_unchecked().set_all_values_referenced(all_values_referenced) + }) + .with_slots(vec![Some(codes), Some(values)]), + ) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "DictArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { if array.is_empty() { let result_dtype = array @@ -186,12 +192,13 @@ impl VTable for Dict { let array = require_child!(array, array.values(), 1 => AnyCanonical); - let DictDataParts { codes, values, .. } = array.into_data().into_parts(); - - let codes = codes - .try_into::() + let codes = array + .codes() + .clone() + .try_downcast::() .ok() .vortex_expect("must be primitive"); + let values = array.values().clone(); debug_assert!(values.is_canonical()); // TODO: add canonical owned cast. let values = values.to_canonical()?; diff --git a/vortex-array/src/arrays/dict/vtable/operations.rs b/vortex-array/src/arrays/dict/vtable/operations.rs index af8eff0cbe5..5fda203aa7a 100644 --- a/vortex-array/src/arrays/dict/vtable/operations.rs +++ b/vortex-array/src/arrays/dict/vtable/operations.rs @@ -8,6 +8,7 @@ use super::Dict; use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::OperationsVTable; +use crate::arrays::dict::DictArrayExt; use crate::scalar::Scalar; impl OperationsVTable for Dict { diff --git a/vortex-array/src/arrays/dict/vtable/validity.rs b/vortex-array/src/arrays/dict/vtable/validity.rs index fc13dcdc84d..48264fa0a41 100644 --- a/vortex-array/src/arrays/dict/vtable/validity.rs +++ b/vortex-array/src/arrays/dict/vtable/validity.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::array::ValidityVTable; use crate::arrays::DictArray; +use crate::arrays::dict::DictArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::Nullability; use crate::scalar::Scalar; diff --git a/vortex-array/src/arrays/extension/array.rs b/vortex-array/src/arrays/extension/array.rs index 757c3042484..5cd178d89e8 100644 --- a/vortex-array/src/arrays/extension/array.rs +++ b/vortex-array/src/arrays/extension/array.rs @@ -7,6 +7,7 @@ use vortex_error::VortexResult; use crate::ArrayRef; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::arrays::Extension; use crate::dtype::DType; use crate::dtype::extension::ExtDTypeRef; @@ -58,7 +59,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["storage"]; pub struct ExtensionData { /// The storage dtype. This **must** be a [`Extension::DType`] variant. pub(super) ext_dtype: ExtDTypeRef, - pub(super) slots: Vec>, } impl ExtensionData { @@ -67,8 +67,8 @@ impl ExtensionData { /// # Panics /// /// Panics if the storage array in not compatible with the extension dtype. - pub fn new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> Self { - Self::try_new(ext_dtype, storage_array).vortex_expect("Failed to create `ExtensionArray`") + pub fn new(ext_dtype: ExtDTypeRef, storage_dtype: &DType) -> Self { + Self::try_new(ext_dtype, storage_dtype).vortex_expect("Failed to create `ExtensionArray`") } /// Tries to construct a new `ExtensionArray`. @@ -76,17 +76,17 @@ impl ExtensionData { /// # Errors /// /// Returns an error if the storage array in not compatible with the extension dtype. - pub fn try_new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> VortexResult { + pub fn try_new(ext_dtype: ExtDTypeRef, storage_dtype: &DType) -> VortexResult { // TODO(connor): Replace these statements once we add `validate_storage_array`. // ext_dtype.validate_storage_array(&storage_array)?; assert_eq!( ext_dtype.storage_dtype(), - storage_array.dtype(), + storage_dtype, "ExtensionArray: storage_dtype must match storage array DType", ); // SAFETY: we validate that the inputs are valid above. - Ok(unsafe { Self::new_unchecked(ext_dtype, storage_array) }) + Ok(unsafe { Self::new_unchecked(ext_dtype, storage_dtype) }) } /// Creates a new `ExtensionArray`. @@ -96,7 +96,7 @@ impl ExtensionData { /// The caller must ensure that the storage array is compatible with the extension dtype. In /// other words, they must know that `ext_dtype.validate_storage_array(&storage_array)` has been /// called successfully on this storage array. - pub unsafe fn new_unchecked(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> Self { + pub unsafe fn new_unchecked(ext_dtype: ExtDTypeRef, storage_dtype: &DType) -> Self { // TODO(connor): Replace these statements once we add `validate_storage_array`. // #[cfg(debug_assertions)] // ext_dtype @@ -104,42 +104,27 @@ impl ExtensionData { // .vortex_expect("[Debug Assertion]: Invalid storage array for `ExtensionArray`"); debug_assert_eq!( ext_dtype.storage_dtype(), - storage_array.dtype(), + storage_dtype, "ExtensionArray: storage_dtype must match storage array DType", ); - Self { - ext_dtype, - slots: vec![Some(storage_array)], - } - } - - /// Returns the length of this array. - pub fn len(&self) -> usize { - self.storage_array().len() - } - - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - DType::Extension(self.ext_dtype.clone()) - } - - /// Returns `true` if this array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 + Self { ext_dtype } } /// The extension dtype of this array. pub fn ext_dtype(&self) -> &ExtDTypeRef { &self.ext_dtype } +} - pub fn storage_array(&self) -> &ArrayRef { - self.slots[STORAGE_SLOT] +pub trait ExtensionArrayExt: TypedArrayRef { + fn storage_array(&self) -> &ArrayRef { + self.as_ref().slots()[STORAGE_SLOT] .as_ref() .vortex_expect("ExtensionArray storage slot") } } +impl> ExtensionArrayExt for T {} impl Array { /// Constructs a new `ExtensionArray`. @@ -150,15 +135,23 @@ impl Array { pub fn new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> Self { let dtype = DType::Extension(ext_dtype.clone()); let len = storage_array.len(); - let data = ExtensionData::new(ext_dtype, storage_array); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Extension, dtype, len, data)) } + let data = ExtensionData::new(ext_dtype, storage_array.dtype()); + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Extension, dtype, len, data).with_slots(vec![Some(storage_array)]), + ) + } } /// Tries to construct a new `ExtensionArray`. pub fn try_new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> VortexResult { let dtype = DType::Extension(ext_dtype.clone()); let len = storage_array.len(); - let data = ExtensionData::try_new(ext_dtype, storage_array)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Extension, dtype, len, data)) }) + let data = ExtensionData::try_new(ext_dtype, storage_array.dtype())?; + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Extension, dtype, len, data).with_slots(vec![Some(storage_array)]), + ) + }) } } diff --git a/vortex-array/src/arrays/extension/compute/cast.rs b/vortex-array/src/arrays/extension/compute/cast.rs index 31ceb8ccf8f..f2705f5c7b4 100644 --- a/vortex-array/src/arrays/extension/compute/cast.rs +++ b/vortex-array/src/arrays/extension/compute/cast.rs @@ -6,6 +6,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Extension; use crate::arrays::ExtensionArray; +use crate::arrays::extension::ExtensionArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; diff --git a/vortex-array/src/arrays/extension/compute/compare.rs b/vortex-array/src/arrays/extension/compute/compare.rs index 60fc1430b76..ecd6bbd5614 100644 --- a/vortex-array/src/arrays/extension/compute/compare.rs +++ b/vortex-array/src/arrays/extension/compute/compare.rs @@ -9,6 +9,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::ConstantArray; use crate::arrays::Extension; +use crate::arrays::extension::ExtensionArrayExt; use crate::builtins::ArrayBuiltins; use crate::scalar_fn::fns::binary::CompareKernel; use crate::scalar_fn::fns::operators::CompareOperator; diff --git a/vortex-array/src/arrays/extension/compute/filter.rs b/vortex-array/src/arrays/extension/compute/filter.rs index 274c838859a..1a9ffe499c5 100644 --- a/vortex-array/src/arrays/extension/compute/filter.rs +++ b/vortex-array/src/arrays/extension/compute/filter.rs @@ -9,6 +9,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Extension; use crate::arrays::ExtensionArray; +use crate::arrays::extension::ExtensionArrayExt; use crate::arrays::filter::FilterReduce; impl FilterReduce for Extension { diff --git a/vortex-array/src/arrays/extension/compute/mask.rs b/vortex-array/src/arrays/extension/compute/mask.rs index 95c70ddb78f..817df8d7fc8 100644 --- a/vortex-array/src/arrays/extension/compute/mask.rs +++ b/vortex-array/src/arrays/extension/compute/mask.rs @@ -8,7 +8,8 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Extension; use crate::arrays::ExtensionArray; -use crate::arrays::scalar_fn::ScalarFnArrayExt; +use crate::arrays::extension::ExtensionArrayExt; +use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::scalar_fn::EmptyOptions; use crate::scalar_fn::fns::mask::Mask as MaskExpr; use crate::scalar_fn::fns::mask::MaskReduce; diff --git a/vortex-array/src/arrays/extension/compute/rules.rs b/vortex-array/src/arrays/extension/compute/rules.rs index 50512fa184b..af1cbc832b9 100644 --- a/vortex-array/src/arrays/extension/compute/rules.rs +++ b/vortex-array/src/arrays/extension/compute/rules.rs @@ -9,6 +9,7 @@ use crate::array::ArrayView; use crate::arrays::Extension; use crate::arrays::ExtensionArray; use crate::arrays::Filter; +use crate::arrays::extension::ExtensionArrayExt; use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ArrayParentReduceRule; @@ -61,7 +62,9 @@ mod tests { use crate::arrays::ExtensionArray; use crate::arrays::FilterArray; use crate::arrays::PrimitiveArray; + use crate::arrays::extension::ExtensionArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayExt; + use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; diff --git a/vortex-array/src/arrays/extension/compute/slice.rs b/vortex-array/src/arrays/extension/compute/slice.rs index d0ac8edddf6..3b5e4b707ff 100644 --- a/vortex-array/src/arrays/extension/compute/slice.rs +++ b/vortex-array/src/arrays/extension/compute/slice.rs @@ -10,6 +10,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Extension; use crate::arrays::ExtensionArray; +use crate::arrays::extension::ExtensionArrayExt; use crate::arrays::slice::SliceReduce; impl SliceReduce for Extension { diff --git a/vortex-array/src/arrays/extension/compute/take.rs b/vortex-array/src/arrays/extension/compute/take.rs index 49d41c8181d..8fb0da11222 100644 --- a/vortex-array/src/arrays/extension/compute/take.rs +++ b/vortex-array/src/arrays/extension/compute/take.rs @@ -10,6 +10,7 @@ use crate::array::ArrayView; use crate::arrays::Extension; use crate::arrays::ExtensionArray; use crate::arrays::dict::TakeExecute; +use crate::arrays::extension::ExtensionArrayExt; impl TakeExecute for Extension { fn take( diff --git a/vortex-array/src/arrays/extension/mod.rs b/vortex-array/src/arrays/extension/mod.rs index 3e4502cc522..802545025c1 100644 --- a/vortex-array/src/arrays/extension/mod.rs +++ b/vortex-array/src/arrays/extension/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::ExtensionArrayExt; pub use array::ExtensionData; pub use vtable::ExtensionArray; diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index 3ef3bc05617..fc48542c4c9 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -5,13 +5,18 @@ mod kernel; mod operations; mod validity; +use std::hash::Hasher; + use kernel::PARENT_KERNELS; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::ArrayEq; +use crate::ArrayHash; use crate::ArrayRef; use crate::ExecutionCtx; use crate::ExecutionResult; @@ -22,18 +27,26 @@ use crate::array::ArrayView; use crate::array::VTable; use crate::array::ValidityVTableFromChild; use crate::arrays::extension::ExtensionData; -use crate::arrays::extension::array::NUM_SLOTS; use crate::arrays::extension::array::SLOT_NAMES; +use crate::arrays::extension::array::STORAGE_SLOT; use crate::arrays::extension::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; -use crate::hash::ArrayEq; -use crate::hash::ArrayHash; use crate::serde::ArrayChildren; use crate::vtable; vtable!(Extension, Extension, ExtensionData); +impl ArrayHash for ExtensionData { + fn array_hash(&self, _state: &mut H, _precision: Precision) {} +} + +impl ArrayEq for ExtensionData { + fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { + true + } +} + impl VTable for Extension { type ArrayData = ExtensionData; @@ -44,20 +57,6 @@ impl VTable for Extension { Self::ID } - fn array_hash( - array: &ExtensionData, - state: &mut H, - precision: Precision, - ) { - array.storage_array().array_hash(state, precision); - } - - fn array_eq(array: &ExtensionData, other: &ExtensionData, precision: Precision) -> bool { - array - .storage_array() - .array_eq(other.storage_array(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -70,10 +69,6 @@ impl VTable for Extension { None } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } @@ -82,15 +77,25 @@ impl VTable for Extension { Ok(Some(vec![])) } - fn validate(&self, data: &ExtensionData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &ExtensionData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + _ = data; + let storage = slots[STORAGE_SLOT] + .as_ref() + .vortex_expect("ExtensionArray storage slot"); vortex_ensure!( - data.len() == len, + storage.len() == len, "ExtensionArray length {} does not match outer length {}", - data.len(), + storage.len(), len ); - let actual_dtype = data.dtype(); + let actual_dtype = DType::Extension(data.ext_dtype.clone()); vortex_ensure!( &actual_dtype == dtype, "ExtensionArray dtype {} does not match outer dtype {}", @@ -110,7 +115,7 @@ impl VTable for Extension { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { if !metadata.is_empty() { vortex_bail!( "ExtensionArray expects empty metadata, got {} bytes", @@ -124,18 +129,13 @@ impl VTable for Extension { vortex_bail!("Expected 1 child, got {}", children.len()); } let storage = children.get(0, ext_dtype.storage_dtype(), len)?; - Ok(ExtensionData::new(ext_dtype.clone(), storage)) - } - - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "ExtensionArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) + Ok(crate::array::ArrayParts::new( + self.clone(), + dtype.clone(), + len, + ExtensionData::new(ext_dtype.clone(), storage.dtype()), + ) + .with_slots(vec![Some(storage)])) } fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { diff --git a/vortex-array/src/arrays/extension/vtable/operations.rs b/vortex-array/src/arrays/extension/vtable/operations.rs index d53772a64e9..d86c7599aef 100644 --- a/vortex-array/src/arrays/extension/vtable/operations.rs +++ b/vortex-array/src/arrays/extension/vtable/operations.rs @@ -7,6 +7,7 @@ use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::Extension; +use crate::arrays::extension::ExtensionArrayExt; use crate::scalar::Scalar; impl OperationsVTable for Extension { diff --git a/vortex-array/src/arrays/extension/vtable/validity.rs b/vortex-array/src/arrays/extension/vtable/validity.rs index a264f9133aa..6d535687088 100644 --- a/vortex-array/src/arrays/extension/vtable/validity.rs +++ b/vortex-array/src/arrays/extension/vtable/validity.rs @@ -2,12 +2,13 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use crate::ArrayRef; +use crate::array::ArrayView; use crate::array::ValidityChild; use crate::arrays::Extension; -use crate::arrays::extension::ExtensionData; +use crate::arrays::extension::ExtensionArrayExt; impl ValidityChild for Extension { - fn validity_child(array: &ExtensionData) -> &ArrayRef { - array.storage_array() + fn validity_child(array: ArrayView<'_, Extension>) -> ArrayRef { + array.storage_array().clone() } } diff --git a/vortex-array/src/arrays/filter/array.rs b/vortex-array/src/arrays/filter/array.rs index 7a0677d5d92..efe362337f4 100644 --- a/vortex-array/src/arrays/filter/array.rs +++ b/vortex-array/src/arrays/filter/array.rs @@ -9,8 +9,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::arrays::Filter; -use crate::dtype::DType; /// The source array being filtered. pub(super) const CHILD_SLOT: usize = 0; @@ -24,35 +24,38 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; /// The resulting array contains only the elements where the mask is true. #[derive(Clone, Debug)] pub struct FilterData { - pub(super) slots: Vec>, - /// The boolean mask selecting which elements to keep. pub(super) mask: Mask, } pub struct FilterDataParts { - pub child: ArrayRef, pub mask: Mask, } +pub trait FilterArrayExt: TypedArrayRef { + fn child(&self) -> &ArrayRef { + self.as_ref().slots()[CHILD_SLOT] + .as_ref() + .vortex_expect("validated filter child slot") + } +} +impl> FilterArrayExt for T {} + impl FilterData { - pub fn new(array: ArrayRef, mask: Mask) -> Self { - Self::try_new(array, mask).vortex_expect("FilterArray construction failed") + pub fn new(mask: Mask) -> Self { + Self { mask } } - fn try_new(array: ArrayRef, mask: Mask) -> VortexResult { + fn try_new(array_len: usize, mask: Mask) -> VortexResult { vortex_ensure_eq!( - array.len(), + array_len, mask.len(), "FilterArray length mismatch: array has length {} but mask has length {}", - array.len(), + array_len, mask.len() ); - Ok(Self { - slots: vec![Some(array)], - mask, - }) + Ok(Self { mask }) } /// Returns the length of this array (number of elements after filtering). @@ -60,35 +63,18 @@ impl FilterData { self.mask.true_count() } - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - self.child().dtype() - } - /// Returns `true` if this array is empty. pub fn is_empty(&self) -> bool { self.len() == 0 } - /// The child array being filtered. - pub fn child(&self) -> &ArrayRef { - self.slots[CHILD_SLOT] - .as_ref() - .vortex_expect("FilterArray child slot") - } - /// The mask used to filter the child array. pub fn filter_mask(&self) -> &Mask { &self.mask } - pub fn into_parts(mut self) -> FilterDataParts { - FilterDataParts { - child: self.slots[CHILD_SLOT] - .take() - .vortex_expect("FilterArray child slot"), - mask: self.mask, - } + pub fn into_parts(self) -> FilterDataParts { + FilterDataParts { mask: self.mask } } } @@ -97,15 +83,23 @@ impl Array { pub fn new(array: ArrayRef, mask: Mask) -> Self { let dtype = array.dtype().clone(); let len = mask.true_count(); - let data = FilterData::new(array, mask); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Filter, dtype, len, data)) } + let data = FilterData::new(mask); + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Filter, dtype, len, data).with_slots(vec![Some(array)]), + ) + } } /// Constructs a new `FilterArray`. pub fn try_new(array: ArrayRef, mask: Mask) -> VortexResult { let dtype = array.dtype().clone(); let len = mask.true_count(); - let data = FilterData::try_new(array, mask)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Filter, dtype, len, data)) }) + let data = FilterData::try_new(array.len(), mask)?; + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Filter, dtype, len, data).with_slots(vec![Some(array)]), + ) + }) } } diff --git a/vortex-array/src/arrays/filter/execute/bool.rs b/vortex-array/src/arrays/filter/execute/bool.rs index e6151a6d770..1a35d1e74aa 100644 --- a/vortex-array/src/arrays/filter/execute/bool.rs +++ b/vortex-array/src/arrays/filter/execute/bool.rs @@ -7,6 +7,7 @@ use vortex_error::VortexExpect; use vortex_mask::MaskValues; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; use crate::arrays::filter::execute::bitbuffer; use crate::arrays::filter::execute::filter_validity; diff --git a/vortex-array/src/arrays/filter/execute/fixed_size_list.rs b/vortex-array/src/arrays/filter/execute/fixed_size_list.rs index 65482f16005..3085debf9e1 100644 --- a/vortex-array/src/arrays/filter/execute/fixed_size_list.rs +++ b/vortex-array/src/arrays/filter/execute/fixed_size_list.rs @@ -10,6 +10,7 @@ use vortex_mask::MaskValues; use crate::arrays::FixedSizeListArray; use crate::arrays::filter::execute::filter_validity; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; /// Density threshold for choosing between indices and slices representation when expanding masks. /// diff --git a/vortex-array/src/arrays/filter/execute/listview.rs b/vortex-array/src/arrays/filter/execute/listview.rs index dac1c9d4928..e6c30416e17 100644 --- a/vortex-array/src/arrays/filter/execute/listview.rs +++ b/vortex-array/src/arrays/filter/execute/listview.rs @@ -9,6 +9,7 @@ use vortex_mask::MaskValues; use crate::arrays::ListViewArray; use crate::arrays::filter::execute::filter_validity; use crate::arrays::filter::execute::values_to_mask; +use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::ListViewRebuildMode; // TODO(connor)[ListView]: Make use of this threshold after we start migrating operators. @@ -88,6 +89,7 @@ mod test { use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; use crate::arrays::filter::execute::ConstantArray; + use crate::arrays::listview::ListViewArrayExt; use crate::assert_arrays_eq; use crate::compute::conformance::filter::test_filter_conformance; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/filter/execute/mod.rs b/vortex-array/src/arrays/filter/execute/mod.rs index a27a56906b7..8a364cc71b6 100644 --- a/vortex-array/src/arrays/filter/execute/mod.rs +++ b/vortex-array/src/arrays/filter/execute/mod.rs @@ -22,6 +22,9 @@ use crate::arrays::ExtensionArray; use crate::arrays::Filter; use crate::arrays::NullArray; use crate::arrays::VariantArray; +use crate::arrays::extension::ExtensionArrayExt; +use crate::arrays::filter::FilterArrayExt; +use crate::arrays::variant::VariantArrayExt; use crate::scalar::Scalar; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/filter/execute/struct_.rs b/vortex-array/src/arrays/filter/execute/struct_.rs index f53d0cda6c0..a5b81b16751 100644 --- a/vortex-array/src/arrays/filter/execute/struct_.rs +++ b/vortex-array/src/arrays/filter/execute/struct_.rs @@ -10,6 +10,7 @@ use crate::ArrayRef; use crate::arrays::StructArray; use crate::arrays::filter::execute::filter_validity; use crate::arrays::filter::execute::values_to_mask; +use crate::arrays::struct_::StructArrayExt; pub fn filter_struct(array: &StructArray, mask: &Arc) -> StructArray { let filtered_validity = filter_validity( diff --git a/vortex-array/src/arrays/filter/mod.rs b/vortex-array/src/arrays/filter/mod.rs index 7491899230e..39859bc5b25 100644 --- a/vortex-array/src/arrays/filter/mod.rs +++ b/vortex-array/src/arrays/filter/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::FilterArrayExt; pub use array::FilterData; pub use array::FilterDataParts; pub use vtable::FilterArray; diff --git a/vortex-array/src/arrays/filter/rules.rs b/vortex-array/src/arrays/filter/rules.rs index 0095d47b422..45e938d7768 100644 --- a/vortex-array/src/arrays/filter/rules.rs +++ b/vortex-array/src/arrays/filter/rules.rs @@ -11,6 +11,7 @@ use crate::array::ArrayView; use crate::arrays::Filter; use crate::arrays::Struct; use crate::arrays::StructArray; +use crate::arrays::filter::FilterArrayExt; use crate::arrays::struct_::StructDataParts; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ArrayReduceRule; @@ -74,7 +75,7 @@ impl ArrayReduceRule for FilterStructRule { struct_fields, validity, .. - } = struct_array.into_owned().into_data().into_parts(); + } = struct_array.into_owned().into_data_parts(); let filtered_validity = validity.filter(mask)?; diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index fb7bdbf9d0b..75258f542e5 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -3,6 +3,7 @@ use std::hash::Hasher; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -21,8 +22,9 @@ use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::array::VTable; use crate::array::ValidityVTable; +use crate::arrays::filter::FilterArrayExt; +use crate::arrays::filter::array::CHILD_SLOT; use crate::arrays::filter::array::FilterData; -use crate::arrays::filter::array::NUM_SLOTS; use crate::arrays::filter::array::SLOT_NAMES; use crate::arrays::filter::execute::execute_filter; use crate::arrays::filter::execute::execute_filter_fast_paths; @@ -46,6 +48,18 @@ impl Filter { pub const ID: ArrayId = ArrayId::new_ref("vortex.filter"); } +impl ArrayHash for FilterData { + fn array_hash(&self, state: &mut H, precision: Precision) { + self.mask.array_hash(state, precision); + } +} + +impl ArrayEq for FilterData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + self.mask.array_eq(&other.mask, precision) + } +} + impl VTable for Filter { type ArrayData = FilterData; type OperationsVTable = Self; @@ -55,11 +69,24 @@ impl VTable for Filter { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { vortex_ensure!( - data.child().dtype() == dtype, + slots[CHILD_SLOT].is_some(), + "FilterArray child slot must be present" + ); + let child = slots[CHILD_SLOT] + .as_ref() + .vortex_expect("validated child slot"); + vortex_ensure!( + child.dtype() == dtype, "FilterArray dtype {} does not match outer dtype {}", - data.child().dtype(), + child.dtype(), dtype ); vortex_ensure!( @@ -69,24 +96,14 @@ impl VTable for Filter { len ); vortex_ensure!( - data.child().len() == data.mask.len(), + child.len() == data.mask.len(), "FilterArray child length {} does not match mask length {}", - data.child().len(), + child.len(), data.mask.len() ); Ok(()) } - fn array_hash(array: &FilterData, state: &mut H, precision: Precision) { - array.child().array_hash(state, precision); - array.mask.array_hash(state, precision); - } - - fn array_eq(array: &FilterData, other: &FilterData, precision: Precision) -> bool { - array.child().array_eq(other.child(), precision) - && array.mask.array_eq(&other.mask, precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -99,10 +116,6 @@ impl VTable for Filter { None } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } @@ -121,21 +134,10 @@ impl VTable for Filter { _buffers: &[BufferHandle], _children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { vortex_bail!("Filter array is not serializable") } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "FilterArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { if let Some(canonical) = execute_filter_fast_paths(array.as_view(), ctx)? { return Ok(ExecutionResult::done(canonical)); diff --git a/vortex-array/src/arrays/fixed_size_list/array.rs b/vortex-array/src/arrays/fixed_size_list/array.rs index eb1ef6d7aff..47d5bfc72e3 100644 --- a/vortex-array/src/arrays/fixed_size_list/array.rs +++ b/vortex-array/src/arrays/fixed_size_list/array.rs @@ -10,11 +10,11 @@ use vortex_error::vortex_ensure; use crate::ArrayRef; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::FixedSizeList; use crate::dtype::DType; -use crate::dtype::Nullability; use crate::validity::Validity; /// The `elements` data array, where each fixed-size list scalar is a _slice_ of the `elements` @@ -55,6 +55,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "validity"]; /// ``` /// # fn main() -> vortex_error::VortexResult<()> { /// use vortex_array::arrays::{FixedSizeListArray, PrimitiveArray}; +/// use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; /// use vortex_array::validity::Validity; /// use vortex_array::IntoArray; /// use vortex_buffer::buffer; @@ -81,17 +82,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "validity"]; /// ``` #[derive(Clone, Debug)] pub struct FixedSizeListData { - /// The nullability of the fixed-size list array. - pub(super) nullability: Nullability, - - /// Slots holding [elements]. - pub(super) slots: Vec>, - - /// The size of each fixed-size list scalar in the array. - /// - /// We store the size of each fixed-size list in the array as a field for convenience. - list_size: u32, - /// The length of the array. /// /// Note that this is different from the size of each fixed-size list scalar (`list_size`). @@ -102,15 +92,29 @@ pub struct FixedSizeListData { pub(super) degenerate_len: usize, } +pub struct FixedSizeListDataParts { + pub elements: ArrayRef, + pub validity: Validity, + pub dtype: DType, +} + impl FixedSizeListData { + pub(crate) fn make_slots( + elements: &ArrayRef, + validity: &Validity, + len: usize, + ) -> Vec> { + vec![Some(elements.clone()), validity_to_child(validity, len)] + } + /// Creates a new `FixedSizeListArray`. /// /// # Panics /// /// Panics if the provided components do not satisfy the invariants documented /// in `FixedSizeListArray::new_unchecked`. - pub fn new(elements: ArrayRef, list_size: u32, validity: Validity, len: usize) -> Self { - Self::try_new(elements, list_size, validity, len) + pub fn build(elements: ArrayRef, list_size: u32, validity: Validity, len: usize) -> Self { + Self::try_build(elements, list_size, validity, len) .vortex_expect("FixedSizeListArray construction failed") } @@ -122,7 +126,7 @@ impl FixedSizeListData { /// /// Returns an error if the provided components do not satisfy the invariants documented /// in `FixedSizeListArray::new_unchecked`. - pub(crate) fn try_new( + pub(crate) fn try_build( elements: ArrayRef, list_size: u32, validity: Validity, @@ -131,7 +135,7 @@ impl FixedSizeListData { Self::validate(&elements, len, list_size, &validity)?; // SAFETY: we validate that the inputs are valid above. - Ok(unsafe { Self::new_unchecked(elements, list_size, validity, len) }) + Ok(unsafe { Self::new_unchecked(list_size, len) }) } /// Creates a new `FixedSizeListArray` without validation from these components: @@ -150,39 +154,12 @@ impl FixedSizeListData { /// fixed-size list is equal to the length of the validity). /// - The length of the `elements` array is equal to the length of the outer array times the /// `list_size` (`elements.len() == list_size * len`). - pub unsafe fn new_unchecked( - elements: ArrayRef, - list_size: u32, - validity: Validity, - len: usize, - ) -> Self { - #[cfg(debug_assertions)] - Self::validate(&elements, len, list_size, &validity) - .vortex_expect("[Debug Assertion]: Invalid `FixedSizeListArray` parameters"); - - let nullability = validity.nullability(); - let validity_slot = validity_to_child(&validity, len); - + pub unsafe fn new_unchecked(list_size: u32, len: usize) -> Self { Self { - nullability, - slots: vec![Some(elements), validity_slot], - list_size, degenerate_len: if list_size == 0 { len } else { 0 }, } } - pub fn into_parts(mut self) -> (ArrayRef, Validity, DType) { - let dtype = self.dtype(); - let validity = self.validity(); - ( - self.slots[ELEMENTS_SLOT] - .take() - .vortex_expect("FixedSizeListArray elements slot"), - validity, - dtype, - ) - } - /// Validates the components that would be used to create a `FixedSizeListArray`. /// /// This function checks all the invariants required by `FixedSizeListArray::new_unchecked`. @@ -218,53 +195,57 @@ impl FixedSizeListData { Ok(()) } +} - /// Returns the dtype of the array. - pub fn dtype(&self) -> DType { - DType::FixedSizeList( - Arc::new(self.elements().dtype().clone()), - self.list_size, - self.nullability, - ) - } - - /// Returns the length of the array. - pub fn len(&self) -> usize { - if self.list_size == 0 { - self.degenerate_len - } else { - self.elements().len() / self.list_size as usize +pub trait FixedSizeListArrayExt: TypedArrayRef { + fn dtype_parts(&self) -> (&DType, u32, crate::dtype::Nullability) { + match self.as_ref().dtype() { + DType::FixedSizeList(element_dtype, list_size, nullability) => { + (element_dtype.as_ref(), *list_size, *nullability) + } + _ => unreachable!("FixedSizeListArrayExt requires a fixed-size list dtype"), } } - /// Returns `true` if the array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 + fn elements(&self) -> &ArrayRef { + self.as_ref().slots()[ELEMENTS_SLOT] + .as_ref() + .vortex_expect("FixedSizeListArray elements slot") } - /// Returns the validity of the array. - #[allow(clippy::same_name_method)] - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) + fn list_size(&self) -> u32 { + let (_, list_size, _) = self.dtype_parts(); + list_size } - /// Returns the validity as a [`Mask`](vortex_mask::Mask). - pub fn validity_mask(&self) -> vortex_mask::Mask { - self.validity().to_mask(self.len()) + fn fixed_size_list_validity(&self) -> Validity { + let (_, _, nullability) = self.dtype_parts(); + child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], nullability) } - /// Returns the elements array. - pub fn elements(&self) -> &ArrayRef { - self.slots[ELEMENTS_SLOT] - .as_ref() - .vortex_expect("FixedSizeListArray elements slot") + fn fixed_size_list_validity_mask(&self) -> vortex_mask::Mask { + self.fixed_size_list_validity().to_mask(self.as_ref().len()) } - /// The size of each fixed-size list scalar in the array. - pub const fn list_size(&self) -> u32 { - self.list_size + fn fixed_size_list_elements_at(&self, index: usize) -> VortexResult { + debug_assert!( + index < self.as_ref().len(), + "index {} out of bounds: the len is {}", + index, + self.as_ref().len(), + ); + debug_assert!( + self.fixed_size_list_validity() + .is_valid(index) + .unwrap_or(false) + ); + + let start = self.list_size() as usize * index; + let end = self.list_size() as usize * (index + 1); + self.elements().slice(start..end) } } +impl> FixedSizeListArrayExt for T {} impl Array { /// Creates a new `FixedSizeListArray`. @@ -274,8 +255,13 @@ impl Array { list_size, validity.nullability(), ); - let data = FixedSizeListData::new(elements, list_size, validity, len); - unsafe { Array::from_parts_unchecked(ArrayParts::new(FixedSizeList, dtype, len, data)) } + let slots = FixedSizeListData::make_slots(&elements, &validity, len); + let data = FixedSizeListData::build(elements, list_size, validity, len); + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(FixedSizeList, dtype, len, data).with_slots(slots), + ) + } } /// Constructs a new `FixedSizeListArray`. @@ -290,12 +276,13 @@ impl Array { list_size, validity.nullability(), ); - let data = FixedSizeListData::try_new(elements, list_size, validity, len)?; - Ok( - unsafe { - Array::from_parts_unchecked(ArrayParts::new(FixedSizeList, dtype, len, data)) - }, - ) + let slots = FixedSizeListData::make_slots(&elements, &validity, len); + let data = FixedSizeListData::try_build(elements, list_size, validity, len)?; + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new(FixedSizeList, dtype, len, data).with_slots(slots), + ) + }) } /// Creates a new `FixedSizeListArray` without validation. @@ -314,23 +301,25 @@ impl Array { list_size, validity.nullability(), ); - let data = unsafe { FixedSizeListData::new_unchecked(elements, list_size, validity, len) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(FixedSizeList, dtype, len, data)) } + let slots = FixedSizeListData::make_slots(&elements, &validity, len); + let data = unsafe { FixedSizeListData::new_unchecked(list_size, len) }; + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(FixedSizeList, dtype, len, data).with_slots(slots), + ) + } } -} -impl FixedSizeListData { - pub fn fixed_size_list_elements_at(&self, index: usize) -> VortexResult { - debug_assert!( - index < self.len(), - "index {} out of bounds: the len is {}", - index, - self.len(), - ); - debug_assert!(self.validity().is_valid(index).unwrap_or(false)); - - let start = self.list_size as usize * index; - let end = self.list_size as usize * (index + 1); - self.elements().slice(start..end) + pub fn into_data_parts(self) -> FixedSizeListDataParts { + let dtype = self.dtype().clone(); + let elements = self.slots()[ELEMENTS_SLOT] + .clone() + .vortex_expect("FixedSizeListArray elements slot"); + let validity = self.fixed_size_list_validity(); + FixedSizeListDataParts { + elements, + validity, + dtype, + } } } diff --git a/vortex-array/src/arrays/fixed_size_list/compute/cast.rs b/vortex-array/src/arrays/fixed_size_list/compute/cast.rs index 05a6951ad1d..791462d51c9 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/cast.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/cast.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::FixedSizeList; use crate::arrays::FixedSizeListArray; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; diff --git a/vortex-array/src/arrays/fixed_size_list/compute/mask.rs b/vortex-array/src/arrays/fixed_size_list/compute/mask.rs index 612426a4c59..3df0413e1aa 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/mask.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/mask.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::FixedSizeList; use crate::arrays::FixedSizeListArray; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::scalar_fn::fns::mask::MaskReduce; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/fixed_size_list/compute/slice.rs b/vortex-array/src/arrays/fixed_size_list/compute/slice.rs index 6a26246de76..5146fd2b600 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/slice.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/slice.rs @@ -10,6 +10,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::FixedSizeList; use crate::arrays::FixedSizeListArray; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::arrays::slice::SliceReduce; impl SliceReduce for FixedSizeList { diff --git a/vortex-array/src/arrays/fixed_size_list/compute/take.rs b/vortex-array/src/arrays/fixed_size_list/compute/take.rs index ac9090c5a4b..259eb382013 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/take.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/take.rs @@ -15,6 +15,8 @@ use crate::arrays::FixedSizeListArray; use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; use crate::arrays::dict::TakeExecute; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; +use crate::arrays::primitive::PrimitiveArrayExt; use crate::dtype::IntegerPType; use crate::executor::ExecutionCtx; use crate::match_each_integer_ptype; @@ -146,7 +148,7 @@ fn take_nullable_fsl( let indices: &[I] = indices_array.as_slice::(); let new_len = indices.len(); - let array_validity = array.validity_mask(); + let array_validity = array.fixed_size_list_validity_mask(); let indices_validity = indices_array.validity_mask(); // We must use placeholder zeros for null lists to maintain the array length without diff --git a/vortex-array/src/arrays/fixed_size_list/mod.rs b/vortex-array/src/arrays/fixed_size_list/mod.rs index 875ee8ba393..5d55d22d880 100644 --- a/vortex-array/src/arrays/fixed_size_list/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/mod.rs @@ -2,7 +2,9 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::FixedSizeListArrayExt; pub use array::FixedSizeListData; +pub use array::FixedSizeListDataParts; pub use vtable::FixedSizeListArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/fixed_size_list/tests/basic.rs b/vortex-array/src/arrays/fixed_size_list/tests/basic.rs index 6e465477233..3d4b6e20ce2 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/basic.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/basic.rs @@ -7,6 +7,7 @@ use vortex_buffer::buffer; use crate::IntoArray; use crate::arrays::FixedSizeListArray; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; diff --git a/vortex-array/src/arrays/fixed_size_list/tests/degenerate.rs b/vortex-array/src/arrays/fixed_size_list/tests/degenerate.rs index 3f2fb51dca2..a6fb279788d 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/degenerate.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/degenerate.rs @@ -8,6 +8,7 @@ use vortex_buffer::buffer; use crate::IntoArray; use crate::arrays::FixedSizeListArray; use crate::arrays::PrimitiveArray; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; diff --git a/vortex-array/src/arrays/fixed_size_list/tests/nested.rs b/vortex-array/src/arrays/fixed_size_list/tests/nested.rs index 37f57180803..41447881996 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/nested.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/nested.rs @@ -10,6 +10,7 @@ use crate::ToCanonical; use crate::arrays::FixedSizeListArray; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::builders::ArrayBuilder; use crate::builders::ListBuilder; use crate::dtype::DType; diff --git a/vortex-array/src/arrays/fixed_size_list/tests/nullability.rs b/vortex-array/src/arrays/fixed_size_list/tests/nullability.rs index 4cba013426c..13894ae3ec4 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/nullability.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/nullability.rs @@ -9,6 +9,7 @@ use crate::IntoArray; use crate::arrays::BoolArray; use crate::arrays::FixedSizeListArray; use crate::arrays::PrimitiveArray; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs index ab3e468c187..d3af276de02 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs @@ -2,13 +2,18 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::hash::Hash; +use std::hash::Hasher; +use std::sync::Arc; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::ArrayEq; +use crate::ArrayHash; use crate::ArrayRef; use crate::ExecutionCtx; use crate::ExecutionResult; @@ -23,8 +28,6 @@ use crate::arrays::fixed_size_list::array::SLOT_NAMES; use crate::arrays::fixed_size_list::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; -use crate::hash::ArrayEq; -use crate::hash::ArrayHash; use crate::serde::ArrayChildren; use crate::validity::Validity; use crate::vtable; @@ -41,6 +44,20 @@ impl FixedSizeList { pub const ID: ArrayId = ArrayId::new_ref("vortex.fixed_size_list"); } +impl ArrayHash for FixedSizeListData { + fn array_hash(&self, state: &mut H, precision: Precision) { + let _precision = precision; + self.degenerate_len.hash(state); + } +} + +impl ArrayEq for FixedSizeListData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + let _precision = precision; + self.degenerate_len == other.degenerate_len + } +} + impl VTable for FixedSizeList { type ArrayData = FixedSizeListData; @@ -51,26 +68,6 @@ impl VTable for FixedSizeList { Self::ID } - fn array_hash( - array: &FixedSizeListData, - state: &mut H, - precision: Precision, - ) { - array.elements().array_hash(state, precision); - array.list_size().hash(state); - array.validity().array_hash(state, precision); - } - - fn array_eq( - array: &FixedSizeListData, - other: &FixedSizeListData, - precision: Precision, - ) -> bool { - array.elements().array_eq(other.elements(), precision) - && array.list_size() == other.list_size() - && array.validity().array_eq(&other.validity(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -104,15 +101,37 @@ impl VTable for FixedSizeList { Ok(Some(vec![])) } - fn validate(&self, data: &FixedSizeListData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &FixedSizeListData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + vortex_ensure!( + slots.len() == NUM_SLOTS, + "FixedSizeListArray expected {NUM_SLOTS} slots, found {}", + slots.len() + ); + let DType::FixedSizeList(_, list_size, nullability) = dtype else { + vortex_bail!("Expected `DType::FixedSizeList`, got {dtype:?}"); + }; + let elements = slots[crate::arrays::fixed_size_list::array::ELEMENTS_SLOT] + .as_ref() + .vortex_expect("FixedSizeListArray elements slot"); vortex_ensure!( - data.len() == len, + if *list_size == 0 { + data.degenerate_len == len + } else { + elements.len() / *list_size as usize == len + }, "FixedSizeListArray length {} does not match outer length {}", - data.len(), + len, len ); - let actual_dtype = data.dtype(); + let actual_dtype = + DType::FixedSizeList(Arc::new(elements.dtype().clone()), *list_size, *nullability); vortex_ensure!( &actual_dtype == dtype, "FixedSizeListArray dtype {} does not match outer dtype {}", @@ -132,7 +151,7 @@ impl VTable for FixedSizeList { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { if !metadata.is_empty() { vortex_bail!( "FixedSizeListArray expects empty metadata, got {} bytes", @@ -165,28 +184,16 @@ impl VTable for FixedSizeList { let num_elements = len * (*list_size as usize); let elements = children.get(0, element_dtype.as_ref(), num_elements)?; - FixedSizeListData::try_new(elements, *list_size, validity, len) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let data = + FixedSizeListData::try_build(elements.clone(), *list_size, validity.clone(), len)?; + let slots = FixedSizeListData::make_slots(&elements, &validity, len); + Ok(crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "FixedSizeListArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done(array)) } diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/operations.rs b/vortex-array/src/arrays/fixed_size_list/vtable/operations.rs index 10cebb6a786..ebd2ed463a4 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/operations.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/operations.rs @@ -7,6 +7,7 @@ use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::FixedSizeList; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::scalar::Scalar; impl OperationsVTable for FixedSizeList { diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/validity.rs b/vortex-array/src/arrays/fixed_size_list/vtable/validity.rs index 5be93a4cc97..2740dd95d9c 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/validity.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/validity.rs @@ -5,11 +5,12 @@ use vortex_error::VortexResult; use crate::array::ArrayView; use crate::array::ValidityVTable; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::arrays::fixed_size_list::vtable::FixedSizeList; use crate::validity::Validity; impl ValidityVTable for FixedSizeList { fn validity(array: ArrayView<'_, FixedSizeList>) -> VortexResult { - Ok(array.data().validity()) + Ok(array.fixed_size_list_validity()) } } diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index b8f99c35804..12f1b76d477 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -17,6 +17,7 @@ use crate::VortexSessionExecute; use crate::aggregate_fn::fns::min_max::min_max; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::ConstantArray; @@ -25,7 +26,6 @@ use crate::arrays::Primitive; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::NativePType; -use crate::dtype::Nullability; use crate::match_each_integer_ptype; use crate::match_each_native_ptype; use crate::scalar_fn::fns::operators::Operator; @@ -68,6 +68,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "offsets", "validi /// /// ``` /// use vortex_array::arrays::{ListArray, PrimitiveArray}; +/// use vortex_array::arrays::list::ListArrayExt; /// use vortex_array::validity::Validity; /// use vortex_array::IntoArray; /// use vortex_buffer::buffer; @@ -92,11 +93,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "offsets", "validi /// let third_list = list_array.list_elements_at(2).unwrap(); /// assert!(third_list.is_empty()); // [] /// ``` -#[derive(Clone, Debug)] -pub struct ListData { - pub(super) nullability: Nullability, - pub(super) slots: Vec>, -} +#[derive(Clone, Debug, Default)] +pub struct ListData; pub struct ListDataParts { pub elements: ArrayRef, @@ -106,14 +104,27 @@ pub struct ListDataParts { } impl ListData { + pub(crate) fn make_slots( + elements: &ArrayRef, + offsets: &ArrayRef, + validity: &Validity, + len: usize, + ) -> Vec> { + vec![ + Some(elements.clone()), + Some(offsets.clone()), + validity_to_child(validity, len), + ] + } + /// Creates a new `ListArray`. /// /// # Panics /// /// Panics if the provided components do not satisfy the invariants documented /// in `ListArray::new_unchecked`. - pub fn new(elements: ArrayRef, offsets: ArrayRef, validity: Validity) -> Self { - Self::try_new(elements, offsets, validity).vortex_expect("ListArray new") + pub fn build(elements: ArrayRef, offsets: ArrayRef, validity: Validity) -> Self { + Self::try_build(elements, offsets, validity).vortex_expect("ListArray new") } /// Constructs a new `ListArray`. @@ -124,7 +135,7 @@ impl ListData { /// /// Returns an error if the provided components do not satisfy the invariants documented in /// `ListArray::new_unchecked`. - pub(crate) fn try_new( + pub(crate) fn try_build( elements: ArrayRef, offsets: ArrayRef, validity: Validity, @@ -132,7 +143,7 @@ impl ListData { Self::validate(&elements, &offsets, &validity)?; // SAFETY: validate ensures all invariants are met. - Ok(unsafe { Self::new_unchecked(elements, offsets, validity) }) + Ok(unsafe { Self::new_unchecked() }) } /// Creates a new `ListArray` without validation from these components: @@ -151,18 +162,8 @@ impl ListData { /// - All offset values must be non-negative. /// - The maximum offset must not exceed `elements.len()`. /// - If validity is an array, its length must equal `offsets.len() - 1`. - pub unsafe fn new_unchecked(elements: ArrayRef, offsets: ArrayRef, validity: Validity) -> Self { - #[cfg(debug_assertions)] - Self::validate(&elements, &offsets, &validity) - .vortex_expect("[Debug Assertion]: Invalid `ListViewArray` parameters"); - - let len = offsets.len().saturating_sub(1); - let validity_slot = validity_to_child(&validity, len); - - Self { - nullability: validity.nullability(), - slots: vec![Some(elements), Some(offsets), validity_slot], - } + pub unsafe fn new_unchecked() -> Self { + Self } /// Validates the components that would be used to create a `ListArray`. @@ -249,40 +250,45 @@ impl ListData { Ok(()) } - /// Returns the dtype of the array. - pub fn dtype(&self) -> DType { - DType::List(Arc::new(self.elements().dtype().clone()), self.nullability) + // TODO(connor)[ListView]: Create 2 functions `reset_offsets` and `recursive_reset_offsets`, + // where `reset_offsets` is infallible. + // Also, `reset_offsets` can be made more efficient by replacing `sub_scalar` with a match on + // the offset type and manual subtraction and fast path where `offsets[0] == 0`. +} + +pub trait ListArrayExt: TypedArrayRef { + fn nullability(&self) -> crate::dtype::Nullability { + match self.as_ref().dtype() { + DType::List(_, nullability) => *nullability, + _ => unreachable!("ListArrayExt requires a list dtype"), + } } - /// Returns the length of the array. - pub fn len(&self) -> usize { - self.offsets().len().saturating_sub(1) + fn elements(&self) -> &ArrayRef { + self.as_ref().slots()[ELEMENTS_SLOT] + .as_ref() + .vortex_expect("ListArray elements slot") } - /// Returns `true` if the array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 + fn offsets(&self) -> &ArrayRef { + self.as_ref().slots()[OFFSETS_SLOT] + .as_ref() + .vortex_expect("ListArray offsets slot") } - /// Returns the validity of the array. - #[allow(clippy::same_name_method)] - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) + fn list_validity(&self) -> Validity { + child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - /// Returns the validity as a [`Mask`](vortex_mask::Mask). - pub fn validity_mask(&self) -> vortex_mask::Mask { - self.validity().to_mask(self.len()) + fn list_validity_mask(&self) -> vortex_mask::Mask { + self.list_validity().to_mask(self.as_ref().len()) } - /// Returns the offset at the given index from the list array. - /// - /// Returns an error if the index is out of bounds or scalar_at fails. - pub fn offset_at(&self, index: usize) -> VortexResult { + fn offset_at(&self, index: usize) -> VortexResult { vortex_ensure!( - index <= self.len(), + index <= self.as_ref().len(), "Index {index} out of bounds 0..={}", - self.len() + self.as_ref().len() ); if let Some(p) = self.offsets().as_opt::() { @@ -298,70 +304,55 @@ impl ListData { } } - /// Returns the elements of the list scalar at the given index of the list array. - pub fn list_elements_at(&self, index: usize) -> VortexResult { + fn list_elements_at(&self, index: usize) -> VortexResult { let start = self.offset_at(index)?; let end = self.offset_at(index + 1)?; self.elements().slice(start..end) } - /// Returns elements of the list array referenced by the offsets array. - /// - /// This is useful for discarding any potentially unused parts of the underlying `elements` - /// child array. - pub fn sliced_elements(&self) -> VortexResult { + fn sliced_elements(&self) -> VortexResult { let start = self.offset_at(0)?; - let end = self.offset_at(self.len())?; + let end = self.offset_at(self.as_ref().len())?; self.elements().slice(start..end) } - /// Returns the offsets array. - pub fn offsets(&self) -> &ArrayRef { - self.slots[OFFSETS_SLOT] - .as_ref() - .vortex_expect("ListArray offsets slot") - } - - /// Returns the element dtype of the list array. - pub fn element_dtype(&self) -> &DType { + fn element_dtype(&self) -> &DType { self.elements().dtype() } - /// Returns the elements array. - pub fn elements(&self) -> &ArrayRef { - self.slots[ELEMENTS_SLOT] - .as_ref() - .vortex_expect("ListArray elements slot") - } - - pub fn into_parts(mut self) -> ListDataParts { - let dtype = self.dtype(); - let validity = self.validity(); - ListDataParts { - elements: self.slots[ELEMENTS_SLOT] - .take() - .vortex_expect("ListArray elements slot"), - offsets: self.slots[OFFSETS_SLOT] - .take() - .vortex_expect("ListArray offsets slot"), - validity, - dtype, + fn reset_offsets(&self, recurse: bool) -> VortexResult> { + let mut elements = self.sliced_elements()?; + if recurse && elements.is_canonical() { + elements = elements.to_canonical()?.compact()?.into_array(); + } else if recurse && let Some(child_list_array) = elements.as_opt::() { + elements = child_list_array + .into_owned() + .reset_offsets(recurse)? + .into_array(); } - } - // TODO(connor)[ListView]: Create 2 functions `reset_offsets` and `recursive_reset_offsets`, - // where `reset_offsets` is infallible. - // Also, `reset_offsets` can be made more efficient by replacing `sub_scalar` with a match on - // the offset type and manual subtraction and fast path where `offsets[0] == 0`. + let offsets = self.offsets(); + let first_offset = offsets.scalar_at(0)?; + let adjusted_offsets = offsets.clone().binary( + ConstantArray::new(first_offset, offsets.len()).into_array(), + Operator::Sub, + )?; + + Array::::try_new(elements, adjusted_offsets, self.list_validity()) + } } +impl> ListArrayExt for T {} impl Array { /// Creates a new `ListArray`. pub fn new(elements: ArrayRef, offsets: ArrayRef, validity: Validity) -> Self { let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); let len = offsets.len().saturating_sub(1); - let data = ListData::new(elements, offsets, validity); - unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) } + let slots = ListData::make_slots(&elements, &offsets, &validity, len); + let data = ListData::build(elements, offsets, validity); + unsafe { + Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data).with_slots(slots)) + } } /// Constructs a new `ListArray`. @@ -372,8 +363,11 @@ impl Array { ) -> VortexResult { let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); let len = offsets.len().saturating_sub(1); - let data = ListData::try_new(elements, offsets, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) }) + let slots = ListData::make_slots(&elements, &offsets, &validity, len); + let data = ListData::try_build(elements, offsets, validity)?; + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data).with_slots(slots)) + }) } /// Creates a new `ListArray` without validation. @@ -384,32 +378,27 @@ impl Array { pub unsafe fn new_unchecked(elements: ArrayRef, offsets: ArrayRef, validity: Validity) -> Self { let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); let len = offsets.len().saturating_sub(1); - let data = unsafe { ListData::new_unchecked(elements, offsets, validity) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) } + let slots = ListData::make_slots(&elements, &offsets, &validity, len); + let data = unsafe { ListData::new_unchecked() }; + unsafe { + Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data).with_slots(slots)) + } } -} -impl ListData { - pub fn reset_offsets(&self, recurse: bool) -> VortexResult { - let mut elements = self.sliced_elements()?; - if recurse && elements.is_canonical() { - elements = elements.to_canonical()?.compact()?.into_array(); - } else if recurse && let Some(child_list_array) = elements.as_opt::() { - let data = child_list_array.reset_offsets(recurse)?; - let dtype = data.dtype(); - let len = data.len(); - elements = - unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) } - .into_array(); + pub fn into_data_parts(self) -> ListDataParts { + let dtype = self.dtype().clone(); + let elements = self.slots()[ELEMENTS_SLOT] + .clone() + .vortex_expect("ListArray elements slot"); + let offsets = self.slots()[OFFSETS_SLOT] + .clone() + .vortex_expect("ListArray offsets slot"); + let validity = self.list_validity(); + ListDataParts { + elements, + offsets, + validity, + dtype, } - - let offsets = self.offsets(); - let first_offset = offsets.scalar_at(0)?; - let adjusted_offsets = offsets.clone().binary( - ConstantArray::new(first_offset, offsets.len()).into_array(), - Operator::Sub, - )?; - - Self::try_new(elements, adjusted_offsets, self.validity()) } } diff --git a/vortex-array/src/arrays/list/compute/cast.rs b/vortex-array/src/arrays/list/compute/cast.rs index c2357de99f0..bac84916390 100644 --- a/vortex-array/src/arrays/list/compute/cast.rs +++ b/vortex-array/src/arrays/list/compute/cast.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::List; use crate::arrays::ListArray; +use crate::arrays::list::ListArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; diff --git a/vortex-array/src/arrays/list/compute/filter.rs b/vortex-array/src/arrays/list/compute/filter.rs index 87f56fcb7a8..2b98e3c36d1 100644 --- a/vortex-array/src/arrays/list/compute/filter.rs +++ b/vortex-array/src/arrays/list/compute/filter.rs @@ -21,6 +21,7 @@ use crate::arrays::ConstantArray; use crate::arrays::List; use crate::arrays::ListArray; use crate::arrays::filter::FilterKernel; +use crate::arrays::list::ListArrayExt; use crate::dtype::IntegerPType; use crate::match_each_integer_ptype; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/list/compute/mask.rs b/vortex-array/src/arrays/list/compute/mask.rs index 24e6e79935b..e9e895d08f1 100644 --- a/vortex-array/src/arrays/list/compute/mask.rs +++ b/vortex-array/src/arrays/list/compute/mask.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::List; use crate::arrays::ListArray; +use crate::arrays::list::ListArrayExt; use crate::scalar_fn::fns::mask::MaskReduce; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/list/compute/slice.rs b/vortex-array/src/arrays/list/compute/slice.rs index dc747257f04..d29eaa5184c 100644 --- a/vortex-array/src/arrays/list/compute/slice.rs +++ b/vortex-array/src/arrays/list/compute/slice.rs @@ -10,6 +10,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::List; use crate::arrays::ListArray; +use crate::arrays::list::ListArrayExt; use crate::arrays::slice::SliceReduce; impl SliceReduce for List { diff --git a/vortex-array/src/arrays/list/compute/take.rs b/vortex-array/src/arrays/list/compute/take.rs index cc053c1d588..0dfe49ef35d 100644 --- a/vortex-array/src/arrays/list/compute/take.rs +++ b/vortex-array/src/arrays/list/compute/take.rs @@ -12,6 +12,8 @@ use crate::arrays::ListArray; use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; use crate::arrays::dict::TakeExecute; +use crate::arrays::list::ListArrayExt; +use crate::arrays::primitive::PrimitiveArrayExt; use crate::builders::ArrayBuilder; use crate::builders::PrimitiveBuilder; use crate::dtype::IntegerPType; @@ -57,7 +59,7 @@ fn _take( indices_array: ArrayView<'_, Primitive>, ctx: &mut ExecutionCtx, ) -> VortexResult { - let data_validity = array.validity_mask(); + let data_validity = array.list_validity_mask(); let indices_validity = indices_array.validity_mask(); if !indices_validity.all_true() || !data_validity.all_true() { @@ -122,7 +124,7 @@ fn _take_nullable(ctx)?; let offsets: &[O] = offsets_array.as_slice(); let indices: &[I] = indices_array.as_slice(); - let data_validity = array.validity_mask(); + let data_validity = array.list_validity_mask(); let indices_validity = indices_array.validity_mask(); let mut new_offsets = PrimitiveBuilder::::with_capacity( diff --git a/vortex-array/src/arrays/list/mod.rs b/vortex-array/src/arrays/list/mod.rs index 3147097ca80..c884f343a10 100644 --- a/vortex-array/src/arrays/list/mod.rs +++ b/vortex-array/src/arrays/list/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::ListArrayExt; pub use array::ListData; pub use array::ListDataParts; pub use vtable::ListArray; diff --git a/vortex-array/src/arrays/list/tests.rs b/vortex-array/src/arrays/list/tests.rs index f815acb3bb0..e2e7407d111 100644 --- a/vortex-array/src/arrays/list/tests.rs +++ b/vortex-array/src/arrays/list/tests.rs @@ -14,8 +14,6 @@ use crate::Canonical; use crate::IntoArray; use crate::LEGACY_SESSION; use crate::VortexSessionExecute; -use crate::array::Array; -use crate::array::ArrayParts; use crate::arrays::FilterArray; use crate::arrays::List; use crate::arrays::PrimitiveArray; @@ -915,20 +913,8 @@ fn test_recursive_compact_list_of_lists() { assert_eq!(recursive_flat_elements.len(), 7); // Verify data integrity is preserved - let non_recursive_array = Array::try_from_parts(ArrayParts::new( - List, - non_recursive.dtype(), - non_recursive.len(), - non_recursive.clone(), - )) - .unwrap(); - let recursive_array = Array::try_from_parts(ArrayParts::new( - List, - recursive.dtype(), - recursive.len(), - recursive.clone(), - )) - .unwrap(); + let non_recursive_array = non_recursive.into_array(); + let recursive_array = recursive.into_array(); assert_eq!( non_recursive_array.scalar_at(0).unwrap(), recursive_array.scalar_at(0).unwrap() diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index dd999ee3419..300ff5c59a2 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -1,13 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::hash::Hasher; +use std::sync::Arc; + use prost::Message; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::ArrayEq; +use crate::ArrayHash; use crate::ArrayRef; use crate::ExecutionCtx; use crate::ExecutionResult; @@ -17,6 +23,7 @@ use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; +use crate::arrays::list::ListArrayExt; use crate::arrays::list::ListData; use crate::arrays::list::array::NUM_SLOTS; use crate::arrays::list::array::SLOT_NAMES; @@ -27,8 +34,6 @@ use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; -use crate::hash::ArrayEq; -use crate::hash::ArrayHash; use crate::serde::ArrayChildren; use crate::validity::Validity; use crate::vtable; @@ -44,6 +49,16 @@ pub struct ListMetadata { offset_ptype: i32, } +impl ArrayHash for ListData { + fn array_hash(&self, _state: &mut H, _precision: Precision) {} +} + +impl ArrayEq for ListData { + fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { + true + } +} + impl VTable for List { type ArrayData = ListData; @@ -54,18 +69,6 @@ impl VTable for List { Self::ID } - fn array_hash(array: &ListData, state: &mut H, precision: Precision) { - array.elements().array_hash(state, precision); - array.offsets().array_hash(state, precision); - array.validity().array_hash(state, precision); - } - - fn array_eq(array: &ListData, other: &ListData, precision: Precision) -> bool { - array.elements().array_eq(other.elements(), precision) - && array.offsets().array_eq(other.offsets(), precision) - && array.validity().array_eq(&other.validity(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -96,15 +99,32 @@ impl VTable for List { )) } - fn validate(&self, data: &ListData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + _data: &ListData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { vortex_ensure!( - data.len() == len, + slots.len() == NUM_SLOTS, + "ListArray expected {NUM_SLOTS} slots, found {}", + slots.len() + ); + let elements = slots[crate::arrays::list::array::ELEMENTS_SLOT] + .as_ref() + .vortex_expect("ListArray elements slot"); + let offsets = slots[crate::arrays::list::array::OFFSETS_SLOT] + .as_ref() + .vortex_expect("ListArray offsets slot"); + vortex_ensure!( + offsets.len().saturating_sub(1) == len, "ListArray length {} does not match outer length {}", - data.len(), + offsets.len().saturating_sub(1), len ); - let actual_dtype = data.dtype(); + let actual_dtype = DType::List(Arc::new(elements.dtype().clone()), dtype.nullability()); vortex_ensure!( &actual_dtype == dtype, "ListArray dtype {} does not match outer dtype {}", @@ -124,7 +144,7 @@ impl VTable for List { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = ListMetadata::decode(metadata)?; let validity = if children.len() == 2 { Validity::from(dtype.nullability()) @@ -150,28 +170,15 @@ impl VTable for List { len + 1, )?; - ListData::try_new(elements, offsets, validity) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let data = ListData::try_build(elements.clone(), offsets.clone(), validity.clone())?; + let slots = ListData::make_slots(&elements, &offsets, &validity, len); + Ok(crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "ListArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done( list_view_from_list(array, ctx)?.into_array(), diff --git a/vortex-array/src/arrays/list/vtable/operations.rs b/vortex-array/src/arrays/list/vtable/operations.rs index 6530248d3ac..c9fc69c7bbe 100644 --- a/vortex-array/src/arrays/list/vtable/operations.rs +++ b/vortex-array/src/arrays/list/vtable/operations.rs @@ -9,6 +9,7 @@ use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::List; +use crate::arrays::list::ListArrayExt; use crate::scalar::Scalar; impl OperationsVTable for List { diff --git a/vortex-array/src/arrays/list/vtable/validity.rs b/vortex-array/src/arrays/list/vtable/validity.rs index 5d6b6c14d4f..679468529a4 100644 --- a/vortex-array/src/arrays/list/vtable/validity.rs +++ b/vortex-array/src/arrays/list/vtable/validity.rs @@ -5,11 +5,12 @@ use vortex_error::VortexResult; use crate::array::ArrayView; use crate::array::ValidityVTable; +use crate::arrays::list::ListArrayExt; use crate::arrays::list::vtable::List; use crate::validity::Validity; impl ValidityVTable for List { fn validity(array: ArrayView<'_, List>) -> VortexResult { - Ok(array.data().validity()) + Ok(array.list_validity()) } } diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index c25dce40f43..f36161e8550 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -14,6 +14,7 @@ use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::ListView; @@ -22,7 +23,6 @@ use crate::arrays::PrimitiveArray; use crate::arrays::bool; use crate::dtype::DType; use crate::dtype::IntegerPType; -use crate::dtype::Nullability; use crate::match_each_integer_ptype; use crate::validity::Validity; @@ -71,6 +71,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "offsets", "sizes" /// ``` /// # fn main() -> vortex_error::VortexResult<()> { /// # use vortex_array::arrays::{ListViewArray, PrimitiveArray}; +/// # use vortex_array::arrays::listview::ListViewArrayExt; /// # use vortex_array::validity::Validity; /// # use vortex_array::IntoArray; /// # use vortex_buffer::buffer; @@ -108,12 +109,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "offsets", "sizes" /// [`ListArray`]: crate::arrays::ListArray #[derive(Clone, Debug)] pub struct ListViewData { - /// The nullability of the list array. - pub(super) nullability: Nullability, - - /// Slots holding [elements, offsets, sizes]. - pub(super) slots: Vec>, - // TODO(connor)[ListView]: Add the n+1 memory allocation optimization. /// A flag denoting if the array is zero-copyable* to a [`ListArray`](crate::arrays::ListArray). /// @@ -142,15 +137,31 @@ pub struct ListViewDataParts { } impl ListViewData { + pub(crate) fn make_slots( + elements: &ArrayRef, + offsets: &ArrayRef, + sizes: &ArrayRef, + validity: &Validity, + len: usize, + ) -> Vec> { + vec![ + Some(elements.clone()), + Some(offsets.clone()), + Some(sizes.clone()), + validity_to_child(validity, len), + ] + } + /// Creates a new `ListViewArray`. /// /// # Panics /// /// Panics if the provided components do not satisfy the invariants documented /// in `ListViewArray::new_unchecked`. - pub fn new(elements: ArrayRef, offsets: ArrayRef, sizes: ArrayRef, validity: Validity) -> Self { - Self::try_new(elements, offsets, sizes, validity) - .vortex_expect("`ListViewArray` construction failed") + pub fn new() -> Self { + Self { + is_zero_copy_to_list: false, + } } /// Constructs a new `ListViewArray`. @@ -159,22 +170,8 @@ impl ListViewData { /// /// Returns an error if the provided components do not satisfy the invariants documented /// in `ListViewArray::new_unchecked`. - pub fn try_new( - elements: ArrayRef, - offsets: ArrayRef, - sizes: ArrayRef, - validity: Validity, - ) -> VortexResult { - Self::validate(&elements, &offsets, &sizes, &validity)?; - - let len = offsets.len(); - let validity_slot = validity_to_child(&validity, len); - - Ok(Self { - nullability: validity.nullability(), - slots: vec![Some(elements), Some(offsets), Some(sizes), validity_slot], - is_zero_copy_to_list: false, - }) + pub fn try_new() -> VortexResult { + Ok(Self::new()) } /// Creates a new `ListViewArray` without validation. @@ -196,25 +193,8 @@ impl ListViewData { /// - For each `i`, `offsets[i] + sizes[i]` must not overflow and must be `<= elements.len()` /// (even if the corresponding view is defined as null by the validity array). /// - If validity is an array, its length must equal `offsets.len()`. - pub unsafe fn new_unchecked( - elements: ArrayRef, - offsets: ArrayRef, - sizes: ArrayRef, - validity: Validity, - ) -> Self { - if cfg!(debug_assertions) { - Self::validate(&elements, &offsets, &sizes, &validity) - .vortex_expect("Failed to crate `ListViewArray`"); - } - - let len = offsets.len(); - let validity_slot = validity_to_child(&validity, len); - - Self { - nullability: validity.nullability(), - slots: vec![Some(elements), Some(offsets), Some(sizes), validity_slot], - is_zero_copy_to_list: false, - } + pub unsafe fn new_unchecked() -> Self { + Self::new() } /// Validates the components that would be used to create a `ListViewArray`. @@ -299,106 +279,67 @@ impl ListViewData { /// /// Note that leading and trailing unreferenced elements **ARE** allowed. pub unsafe fn with_zero_copy_to_list(mut self, is_zctl: bool) -> Self { - if cfg!(debug_assertions) && is_zctl { - validate_zctl( - self.elements(), - self.offsets().to_primitive(), - self.sizes().to_primitive(), - ) - .vortex_expect("Failed to validate zero-copy to list flag"); - } self.is_zero_copy_to_list = is_zctl; self } - /// Verifies that the `ListViewArray` is zero-copyable to a [`ListArray`]. - /// - /// This will run an expensive validation of the `ListViewArray`'s components. It will check the - /// following things: - /// - /// - Offsets must be sorted (but not strictly sorted, zero-length lists are allowed). - /// - No gaps in elements between first and last referenced elements. - /// - No overlapping list views (each element referenced at most once). - /// - /// Note that leading and trailing unreferenced elements **ARE** allowed. - /// - /// This method should really only be called if the caller knows that the `ListViewArray` will - /// be converted into a [`ListArray`] in the future, and the caller wants to set the - /// optimization flag to `true` with the unsafe [`with_zero_copy_to_list`] method. - /// - /// [`ListArray`]: crate::arrays::ListArray - /// [`with_zero_copy_to_list`]: Self::with_zero_copy_to_list - pub fn verify_is_zero_copy_to_list(&self) -> bool { - validate_zctl( - self.elements(), - self.offsets().to_primitive(), - self.sizes().to_primitive(), - ) - .is_ok() + /// Returns true if the `ListViewArray` is zero-copyable to a + /// [`ListArray`](crate::arrays::ListArray). + pub fn is_zero_copy_to_list(&self) -> bool { + self.is_zero_copy_to_list } +} - pub fn into_parts(mut self) -> ListViewDataParts { - let validity = self.validity(); - ListViewDataParts { - elements_dtype: Arc::new(self.elements().dtype().clone()), - elements: self.slots[ELEMENTS_SLOT] - .take() - .vortex_expect("ListViewArray elements slot"), - offsets: self.slots[OFFSETS_SLOT] - .take() - .vortex_expect("ListViewArray offsets slot"), - sizes: self.slots[SIZES_SLOT] - .take() - .vortex_expect("ListViewArray sizes slot"), - validity, +impl Default for ListViewData { + fn default() -> Self { + Self::new() + } +} + +pub trait ListViewArrayExt: TypedArrayRef { + fn nullability(&self) -> crate::dtype::Nullability { + match self.as_ref().dtype() { + DType::List(_, nullability) => *nullability, + _ => unreachable!("ListViewArrayExt requires a list dtype"), } } - /// Returns the dtype of the array. - pub fn dtype(&self) -> DType { - DType::List(Arc::new(self.elements().dtype().clone()), self.nullability) + fn elements(&self) -> &ArrayRef { + self.as_ref().slots()[ELEMENTS_SLOT] + .as_ref() + .vortex_expect("ListViewArray elements slot") } - /// Returns the length of the array. - pub fn len(&self) -> usize { - debug_assert_eq!(self.offsets().len(), self.sizes().len()); - self.offsets().len() + fn offsets(&self) -> &ArrayRef { + self.as_ref().slots()[OFFSETS_SLOT] + .as_ref() + .vortex_expect("ListViewArray offsets slot") } - /// Returns `true` if the array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 + fn sizes(&self) -> &ArrayRef { + self.as_ref().slots()[SIZES_SLOT] + .as_ref() + .vortex_expect("ListViewArray sizes slot") } - /// Returns the validity of the array. - #[allow(clippy::same_name_method)] - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) + fn listview_validity(&self) -> Validity { + child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - /// Returns the validity as a [`Mask`](vortex_mask::Mask). - pub fn validity_mask(&self) -> vortex_mask::Mask { - self.validity().to_mask(self.len()) + fn listview_validity_mask(&self) -> vortex_mask::Mask { + self.listview_validity().to_mask(self.as_ref().len()) } - /// Returns the offset at the given index. - /// - /// Note that it is possible the corresponding list view is null (which is only defined by the - /// validity map). Regardless, we are still guaranteed that this offset is valid by the - /// invariants of `ListViewArray`. - pub fn offset_at(&self, index: usize) -> usize { + fn offset_at(&self, index: usize) -> usize { assert!( - index < self.len(), + index < self.as_ref().len(), "Index {index} out of bounds 0..{}", - self.len() + self.as_ref().len() ); - - // Fast path for `PrimitiveArray`. self.offsets() .as_opt::() .map(|p| match_each_integer_ptype!(p.ptype(), |P| { p.as_slice::

()[index].as_() })) .unwrap_or_else(|| { - // Slow path: use `scalar_at` if we can't downcast directly to `PrimitiveArray`. self.offsets() .scalar_at(index) .vortex_expect("offsets must support scalar_at") @@ -408,25 +349,17 @@ impl ListViewData { }) } - /// Returns the size at the given index. - /// - /// Note that it is possible the corresponding list view is null (which is only defined by the - /// validity map). Regardless, we are still guaranteed that this size is valid by the invariants - /// of `ListViewArray`. - pub fn size_at(&self, index: usize) -> usize { + fn size_at(&self, index: usize) -> usize { assert!( - index < self.len(), + index < self.as_ref().len(), "Index {} out of bounds 0..{}", index, - self.len() + self.as_ref().len() ); - - // Fast path for `PrimitiveArray`. self.sizes() .as_opt::() .map(|p| match_each_integer_ptype!(p.ptype(), |P| { p.as_slice::

()[index].as_() })) .unwrap_or_else(|| { - // Slow path: use `scalar_at` if we can't downcast directly to `PrimitiveArray`. self.sizes() .scalar_at(index) .vortex_expect("sizes must support scalar_at") @@ -436,52 +369,37 @@ impl ListViewData { }) } - /// Returns the elements at the given index from the list array. - /// - /// # Errors - /// - /// Returns an error if the slice operation fails. - pub fn list_elements_at(&self, index: usize) -> VortexResult { + fn list_elements_at(&self, index: usize) -> VortexResult { let offset = self.offset_at(index); let size = self.size_at(index); self.elements().slice(offset..offset + size) } - /// Returns the offsets array. - pub fn offsets(&self) -> &ArrayRef { - self.slots[OFFSETS_SLOT] - .as_ref() - .vortex_expect("ListViewArray offsets slot") - } - - /// Returns the sizes array. - pub fn sizes(&self) -> &ArrayRef { - self.slots[SIZES_SLOT] - .as_ref() - .vortex_expect("ListViewArray sizes slot") - } - - /// Returns the elements array. - pub fn elements(&self) -> &ArrayRef { - self.slots[ELEMENTS_SLOT] - .as_ref() - .vortex_expect("ListViewArray elements slot") - } - - /// Returns true if the `ListViewArray` is zero-copyable to a - /// [`ListArray`](crate::arrays::ListArray). - pub fn is_zero_copy_to_list(&self) -> bool { - self.is_zero_copy_to_list + fn verify_is_zero_copy_to_list(&self) -> bool { + validate_zctl( + self.elements(), + self.offsets().to_primitive(), + self.sizes().to_primitive(), + ) + .is_ok() } } +impl> ListViewArrayExt for T {} impl Array { /// Creates a new `ListViewArray`. pub fn new(elements: ArrayRef, offsets: ArrayRef, sizes: ArrayRef, validity: Validity) -> Self { let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); let len = offsets.len(); - let data = ListViewData::new(elements, offsets, sizes, validity); - unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) } + let slots = ListViewData::make_slots(&elements, &offsets, &sizes, &validity, len); + ListViewData::validate(&elements, &offsets, &sizes, &validity) + .vortex_expect("`ListViewArray` construction failed"); + let data = ListViewData::new(); + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(ListView, dtype, len, data).with_slots(slots), + ) + } } /// Constructs a new `ListViewArray`. @@ -493,8 +411,14 @@ impl Array { ) -> VortexResult { let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); let len = offsets.len(); - let data = ListViewData::try_new(elements, offsets, sizes, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) }) + let slots = ListViewData::make_slots(&elements, &offsets, &sizes, &validity, len); + ListViewData::validate(&elements, &offsets, &sizes, &validity)?; + let data = ListViewData::try_new()?; + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new(ListView, dtype, len, data).with_slots(slots), + ) + }) } /// Creates a new `ListViewArray` without validation. @@ -510,8 +434,13 @@ impl Array { ) -> Self { let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); let len = offsets.len(); - let data = unsafe { ListViewData::new_unchecked(elements, offsets, sizes, validity) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) } + let slots = ListViewData::make_slots(&elements, &offsets, &sizes, &validity, len); + let data = unsafe { ListViewData::new_unchecked() }; + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(ListView, dtype, len, data).with_slots(slots), + ) + } } /// Mark whether this list view can be zero-copy converted to a list. @@ -520,10 +449,43 @@ impl Array { /// /// See [`ListViewData::with_zero_copy_to_list`]. pub unsafe fn with_zero_copy_to_list(self, is_zctl: bool) -> Self { + if cfg!(debug_assertions) && is_zctl { + validate_zctl( + self.elements(), + self.offsets().to_primitive(), + self.sizes().to_primitive(), + ) + .vortex_expect("Failed to validate zero-copy to list flag"); + } let dtype = self.dtype().clone(); let len = self.len(); + let slots = self.slots().to_vec(); let data = unsafe { self.into_data().with_zero_copy_to_list(is_zctl) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) } + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(ListView, dtype, len, data).with_slots(slots), + ) + } + } + + pub fn into_data_parts(self) -> ListViewDataParts { + let elements = self.slots()[ELEMENTS_SLOT] + .clone() + .vortex_expect("ListViewArray elements slot"); + let offsets = self.slots()[OFFSETS_SLOT] + .clone() + .vortex_expect("ListViewArray offsets slot"); + let sizes = self.slots()[SIZES_SLOT] + .clone() + .vortex_expect("ListViewArray sizes slot"); + let validity = self.listview_validity(); + ListViewDataParts { + elements_dtype: Arc::new(elements.dtype().clone()), + elements, + offsets, + sizes, + validity, + } } } diff --git a/vortex-array/src/arrays/listview/compute/cast.rs b/vortex-array/src/arrays/listview/compute/cast.rs index 225391dab29..77cd7ab7736 100644 --- a/vortex-array/src/arrays/listview/compute/cast.rs +++ b/vortex-array/src/arrays/listview/compute/cast.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::ListView; use crate::arrays::ListViewArray; +use crate::arrays::listview::ListViewArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; diff --git a/vortex-array/src/arrays/listview/compute/mask.rs b/vortex-array/src/arrays/listview/compute/mask.rs index 65dbb23b020..e14233a71d0 100644 --- a/vortex-array/src/arrays/listview/compute/mask.rs +++ b/vortex-array/src/arrays/listview/compute/mask.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::ListView; use crate::arrays::ListViewArray; +use crate::arrays::listview::ListViewArrayExt; use crate::scalar_fn::fns::mask::MaskReduce; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/listview/compute/rules.rs b/vortex-array/src/arrays/listview/compute/rules.rs index e1ddd8ac392..9b9396d7fe7 100644 --- a/vortex-array/src/arrays/listview/compute/rules.rs +++ b/vortex-array/src/arrays/listview/compute/rules.rs @@ -10,6 +10,7 @@ use crate::arrays::Filter; use crate::arrays::ListView; use crate::arrays::ListViewArray; use crate::arrays::dict::TakeReduceAdaptor; +use crate::arrays::listview::ListViewArrayExt; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; diff --git a/vortex-array/src/arrays/listview/compute/slice.rs b/vortex-array/src/arrays/listview/compute/slice.rs index 15dd83d8232..2dd29278392 100644 --- a/vortex-array/src/arrays/listview/compute/slice.rs +++ b/vortex-array/src/arrays/listview/compute/slice.rs @@ -10,6 +10,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::ListView; use crate::arrays::ListViewArray; +use crate::arrays::listview::ListViewArrayExt; use crate::arrays::slice::SliceReduce; impl SliceReduce for ListView { diff --git a/vortex-array/src/arrays/listview/compute/take.rs b/vortex-array/src/arrays/listview/compute/take.rs index 9f0d97bd98d..5cfadd40214 100644 --- a/vortex-array/src/arrays/listview/compute/take.rs +++ b/vortex-array/src/arrays/listview/compute/take.rs @@ -10,6 +10,7 @@ use crate::array::ArrayView; use crate::arrays::ListView; use crate::arrays::ListViewArray; use crate::arrays::dict::TakeReduce; +use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::ListViewRebuildMode; use crate::builtins::ArrayBuiltins; use crate::dtype::Nullability; diff --git a/vortex-array/src/arrays/listview/conversion.rs b/vortex-array/src/arrays/listview/conversion.rs index c6b0633bbcf..54571ecdf21 100644 --- a/vortex-array/src/arrays/listview/conversion.rs +++ b/vortex-array/src/arrays/listview/conversion.rs @@ -9,16 +9,18 @@ use crate::Canonical; use crate::ExecutionCtx; use crate::IntoArray; use crate::ToCanonical; -use crate::array::Array; -use crate::array::ArrayParts; use crate::arrays::ExtensionArray; use crate::arrays::FixedSizeListArray; -use crate::arrays::List; use crate::arrays::ListArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; +use crate::arrays::extension::ExtensionArrayExt; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; +use crate::arrays::list::ListArrayExt; +use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::ListViewRebuildMode; +use crate::arrays::struct_::StructArrayExt; use crate::builders::PrimitiveBuilder; use crate::dtype::IntegerPType; use crate::dtype::Nullability; @@ -38,11 +40,7 @@ pub fn list_view_from_list(list: ListArray, ctx: &mut ExecutionCtx) -> VortexRes // We reset the offsets here because mostly for convenience, and also because callers of this // function might not expect the output `ListViewArray` to have a bunch of leading and trailing // garbage data when they turn it back into a `ListArray`. - let data = list.reset_offsets(false).vortex_expect("This can't fail"); - let dtype = data.dtype(); - let len = data.len(); - let list: ListArray = - unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) }; + let list = list.reset_offsets(false).vortex_expect("This can't fail"); let list_offsets = list.offsets().clone(); @@ -303,6 +301,8 @@ mod tests { use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; use crate::arrays::VarBinViewArray; + use crate::arrays::list::ListArrayExt; + use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::list_from_list_view; use crate::arrays::listview::list_view_from_list; use crate::assert_arrays_eq; diff --git a/vortex-array/src/arrays/listview/mod.rs b/vortex-array/src/arrays/listview/mod.rs index 20dd6fe15d5..77f2a279160 100644 --- a/vortex-array/src/arrays/listview/mod.rs +++ b/vortex-array/src/arrays/listview/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::ListViewArrayExt; pub use array::ListViewData; pub use array::ListViewDataParts; pub use vtable::ListViewArray; diff --git a/vortex-array/src/arrays/listview/rebuild.rs b/vortex-array/src/arrays/listview/rebuild.rs index 7b37b82c03a..4bcfb8311f8 100644 --- a/vortex-array/src/arrays/listview/rebuild.rs +++ b/vortex-array/src/arrays/listview/rebuild.rs @@ -13,6 +13,7 @@ use crate::VortexSessionExecute; use crate::aggregate_fn::fns::min_max::min_max; use crate::arrays::ConstantArray; use crate::arrays::ListViewArray; +use crate::arrays::listview::ListViewArrayExt; use crate::builders::builder_with_capacity; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; @@ -377,6 +378,7 @@ mod tests { use crate::ToCanonical; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; + use crate::arrays::listview::ListViewArrayExt; use crate::assert_arrays_eq; use crate::dtype::Nullability; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/listview/tests/basic.rs b/vortex-array/src/arrays/listview/tests/basic.rs index 62e162f1677..5a1bf09d028 100644 --- a/vortex-array/src/arrays/listview/tests/basic.rs +++ b/vortex-array/src/arrays/listview/tests/basic.rs @@ -15,6 +15,7 @@ use crate::arrays::ConstantArray; use crate::arrays::ListArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; +use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::list_view_from_list; use crate::assert_arrays_eq; use crate::dtype::DType; diff --git a/vortex-array/src/arrays/listview/tests/filter.rs b/vortex-array/src/arrays/listview/tests/filter.rs index b63ceed1430..40eef10d825 100644 --- a/vortex-array/src/arrays/listview/tests/filter.rs +++ b/vortex-array/src/arrays/listview/tests/filter.rs @@ -15,6 +15,7 @@ use crate::ToCanonical; use crate::arrays::ConstantArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; +use crate::arrays::listview::ListViewArrayExt; use crate::assert_arrays_eq; use crate::compute::conformance::filter::test_filter_conformance; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/listview/tests/nested.rs b/vortex-array/src/arrays/listview/tests/nested.rs index d99c3aa55f5..e5ec85a690c 100644 --- a/vortex-array/src/arrays/listview/tests/nested.rs +++ b/vortex-array/src/arrays/listview/tests/nested.rs @@ -8,6 +8,7 @@ use crate::arrays::ListView; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; +use crate::arrays::listview::ListViewArrayExt; use crate::dtype::DType; use crate::dtype::FieldNames; use crate::dtype::Nullability; diff --git a/vortex-array/src/arrays/listview/tests/nullability.rs b/vortex-array/src/arrays/listview/tests/nullability.rs index 30b2a192ef1..aa56291e233 100644 --- a/vortex-array/src/arrays/listview/tests/nullability.rs +++ b/vortex-array/src/arrays/listview/tests/nullability.rs @@ -10,6 +10,7 @@ use crate::IntoArray; use crate::arrays::BoolArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; +use crate::arrays::listview::ListViewArrayExt; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; diff --git a/vortex-array/src/arrays/listview/tests/operations.rs b/vortex-array/src/arrays/listview/tests/operations.rs index f1972e67c28..09cf700bb98 100644 --- a/vortex-array/src/arrays/listview/tests/operations.rs +++ b/vortex-array/src/arrays/listview/tests/operations.rs @@ -20,6 +20,7 @@ use crate::arrays::ConstantArray; use crate::arrays::ListView; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; +use crate::arrays::listview::ListViewArrayExt; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; use crate::compute::conformance::mask::test_mask_conformance; diff --git a/vortex-array/src/arrays/listview/tests/take.rs b/vortex-array/src/arrays/listview/tests/take.rs index c002d7be140..042d93dfe59 100644 --- a/vortex-array/src/arrays/listview/tests/take.rs +++ b/vortex-array/src/arrays/listview/tests/take.rs @@ -14,6 +14,7 @@ use crate::ToCanonical; use crate::arrays::ConstantArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; +use crate::arrays::listview::ListViewArrayExt; use crate::assert_arrays_eq; use crate::compute::conformance::take::test_take_conformance; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index 9dae3483a16..187d3ca125c 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -1,13 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::hash::Hash; +use std::hash::Hasher; +use std::sync::Arc; + use prost::Message; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::ArrayEq; +use crate::ArrayHash; use crate::ArrayRef; use crate::ExecutionCtx; use crate::ExecutionResult; @@ -16,6 +23,7 @@ use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; +use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::ListViewData; use crate::arrays::listview::array::NUM_SLOTS; use crate::arrays::listview::array::SLOT_NAMES; @@ -24,8 +32,6 @@ use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; -use crate::hash::ArrayEq; -use crate::hash::ArrayHash; use crate::serde::ArrayChildren; use crate::validity::Validity; use crate::vtable; @@ -50,6 +56,18 @@ pub struct ListViewMetadata { size_ptype: i32, } +impl ArrayHash for ListViewData { + fn array_hash(&self, state: &mut H, _precision: Precision) { + self.is_zero_copy_to_list().hash(state); + } +} + +impl ArrayEq for ListViewData { + fn array_eq(&self, other: &Self, _precision: Precision) -> bool { + self.is_zero_copy_to_list() == other.is_zero_copy_to_list() + } +} + impl VTable for ListView { type ArrayData = ListViewData; @@ -60,20 +78,6 @@ impl VTable for ListView { Self::ID } - fn array_hash(array: &ListViewData, state: &mut H, precision: Precision) { - array.elements().array_hash(state, precision); - array.offsets().array_hash(state, precision); - array.sizes().array_hash(state, precision); - array.validity().array_hash(state, precision); - } - - fn array_eq(array: &ListViewData, other: &ListViewData, precision: Precision) -> bool { - array.elements().array_eq(other.elements(), precision) - && array.offsets().array_eq(other.offsets(), precision) - && array.sizes().array_eq(other.sizes(), precision) - && array.validity().array_eq(&other.validity(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -97,15 +101,35 @@ impl VTable for ListView { )) } - fn validate(&self, data: &ListViewData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + _data: &ListViewData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { vortex_ensure!( - data.len() == len, + slots.len() == NUM_SLOTS, + "ListViewArray expected {NUM_SLOTS} slots, found {}", + slots.len() + ); + let elements = slots[crate::arrays::listview::array::ELEMENTS_SLOT] + .as_ref() + .vortex_expect("ListViewArray elements slot"); + let offsets = slots[crate::arrays::listview::array::OFFSETS_SLOT] + .as_ref() + .vortex_expect("ListViewArray offsets slot"); + let sizes = slots[crate::arrays::listview::array::SIZES_SLOT] + .as_ref() + .vortex_expect("ListViewArray sizes slot"); + vortex_ensure!( + offsets.len() == len && sizes.len() == len, "ListViewArray length {} does not match outer length {}", - data.len(), + offsets.len(), len ); - let actual_dtype = data.dtype(); + let actual_dtype = DType::List(Arc::new(elements.dtype().clone()), dtype.nullability()); vortex_ensure!( &actual_dtype == dtype, "ListViewArray dtype {} does not match outer dtype {}", @@ -125,7 +149,7 @@ impl VTable for ListView { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = ListViewMetadata::decode(metadata)?; vortex_ensure!( buffers.is_empty(), @@ -169,28 +193,16 @@ impl VTable for ListView { len, )?; - ListViewData::try_new(elements, offsets, sizes, validity) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + ListViewData::validate(&elements, &offsets, &sizes, &validity)?; + let data = ListViewData::try_new()?; + let slots = ListViewData::make_slots(&elements, &offsets, &sizes, &validity, len); + Ok(crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "ListViewArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done(array)) } diff --git a/vortex-array/src/arrays/listview/vtable/operations.rs b/vortex-array/src/arrays/listview/vtable/operations.rs index b32bb87de1f..c899083027a 100644 --- a/vortex-array/src/arrays/listview/vtable/operations.rs +++ b/vortex-array/src/arrays/listview/vtable/operations.rs @@ -9,6 +9,7 @@ use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::ListView; +use crate::arrays::listview::ListViewArrayExt; use crate::scalar::Scalar; impl OperationsVTable for ListView { diff --git a/vortex-array/src/arrays/listview/vtable/validity.rs b/vortex-array/src/arrays/listview/vtable/validity.rs index 02be6bcedfd..64ede5cc142 100644 --- a/vortex-array/src/arrays/listview/vtable/validity.rs +++ b/vortex-array/src/arrays/listview/vtable/validity.rs @@ -5,11 +5,12 @@ use vortex_error::VortexResult; use crate::array::ArrayView; use crate::array::ValidityVTable; +use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::vtable::ListView; use crate::validity::Validity; impl ValidityVTable for ListView { fn validity(array: ArrayView<'_, ListView>) -> VortexResult { - Ok(array.data().validity()) + Ok(array.listview_validity()) } } diff --git a/vortex-array/src/arrays/masked/array.rs b/vortex-array/src/arrays/masked/array.rs index 9e011140930..1e90f83b344 100644 --- a/vortex-array/src/arrays/masked/array.rs +++ b/vortex-array/src/arrays/masked/array.rs @@ -8,11 +8,10 @@ use vortex_error::vortex_bail; use crate::ArrayRef; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Masked; -use crate::dtype::DType; -use crate::dtype::Nullability; use crate::validity::Validity; /// The underlying child array being masked. @@ -23,66 +22,55 @@ pub(super) const NUM_SLOTS: usize = 2; pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child", "validity"]; #[derive(Clone, Debug)] -pub struct MaskedData { - pub(super) slots: Vec>, +pub struct MaskedData; + +pub trait MaskedArrayExt: TypedArrayRef { + fn child(&self) -> &ArrayRef { + self.as_ref().slots()[CHILD_SLOT] + .as_ref() + .vortex_expect("validated masked child slot") + } + + fn validity_child(&self) -> Option<&ArrayRef> { + self.as_ref().slots()[VALIDITY_SLOT].as_ref() + } + + fn masked_validity(&self) -> Validity { + child_to_validity( + &self.as_ref().slots()[VALIDITY_SLOT], + self.as_ref().dtype().nullability(), + ) + } + + fn masked_validity_mask(&self) -> vortex_mask::Mask { + self.masked_validity().to_mask(self.as_ref().len()) + } } +impl> MaskedArrayExt for T {} impl MaskedData { - pub(crate) fn try_new(child: ArrayRef, validity: Validity) -> VortexResult { + pub(crate) fn try_new( + child_len: usize, + child_all_valid: bool, + validity: Validity, + ) -> VortexResult { if matches!(validity, Validity::NonNullable) { vortex_bail!("MaskedArray must have nullable validity, got {validity:?}") } - if !child.all_valid()? { + if !child_all_valid { vortex_bail!("MaskedArray children must not have nulls"); } if let Some(validity_len) = validity.maybe_len() - && validity_len != child.len() + && validity_len != child_len { vortex_bail!("Validity must be the same length as a MaskedArray's child"); } // MaskedArray's nullability is determined solely by its validity, not the child's dtype. // The child can have nullable dtype but must not have any actual null values. - let len = child.len(); - let validity_slot = validity_to_child(&validity, len); - - Ok(Self { - slots: vec![Some(child), validity_slot], - }) - } - - /// Returns the dtype of the array. - pub fn dtype(&self) -> DType { - self.child().dtype().as_nullable() - } - - /// Returns the length of the array. - pub fn len(&self) -> usize { - self.child().len() - } - - /// Returns `true` if the array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns the validity of the array. - #[allow(clippy::same_name_method)] - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], Nullability::Nullable) - } - - /// Returns the validity as a [`Mask`](vortex_mask::Mask). - pub fn validity_mask(&self) -> vortex_mask::Mask { - self.validity().to_mask(self.len()) - } - - pub fn child(&self) -> &ArrayRef { - self.slots[CHILD_SLOT] - .as_ref() - .vortex_expect("MaskedArray child slot") + Ok(Self) } } @@ -91,7 +79,13 @@ impl Array { pub fn try_new(child: ArrayRef, validity: Validity) -> VortexResult { let dtype = child.dtype().as_nullable(); let len = child.len(); - let data = MaskedData::try_new(child, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Masked, dtype, len, data)) }) + let validity_slot = validity_to_child(&validity, len); + let data = MaskedData::try_new(len, child.all_valid()?, validity)?; + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Masked, dtype, len, data) + .with_slots(vec![Some(child), validity_slot]), + ) + }) } } diff --git a/vortex-array/src/arrays/masked/compute/filter.rs b/vortex-array/src/arrays/masked/compute/filter.rs index 9d33ac6767d..fb3969b4bb8 100644 --- a/vortex-array/src/arrays/masked/compute/filter.rs +++ b/vortex-array/src/arrays/masked/compute/filter.rs @@ -10,6 +10,7 @@ use crate::array::ArrayView; use crate::arrays::Masked; use crate::arrays::MaskedArray; use crate::arrays::filter::FilterReduce; +use crate::arrays::masked::MaskedArrayExt; impl FilterReduce for Masked { fn filter(array: ArrayView<'_, Masked>, mask: &Mask) -> VortexResult> { diff --git a/vortex-array/src/arrays/masked/compute/mask.rs b/vortex-array/src/arrays/masked/compute/mask.rs index f822e909432..d8590a68d34 100644 --- a/vortex-array/src/arrays/masked/compute/mask.rs +++ b/vortex-array/src/arrays/masked/compute/mask.rs @@ -6,7 +6,8 @@ use vortex_error::VortexResult; use crate::ArrayRef; use crate::array::ArrayView; use crate::arrays::Masked; -use crate::arrays::scalar_fn::ScalarFnArrayExt; +use crate::arrays::masked::MaskedArrayExt; +use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::scalar_fn::EmptyOptions; use crate::scalar_fn::fns::mask::Mask as MaskExpr; use crate::scalar_fn::fns::mask::MaskReduce; diff --git a/vortex-array/src/arrays/masked/compute/slice.rs b/vortex-array/src/arrays/masked/compute/slice.rs index d6c9ee3c033..fe295bd6826 100644 --- a/vortex-array/src/arrays/masked/compute/slice.rs +++ b/vortex-array/src/arrays/masked/compute/slice.rs @@ -10,6 +10,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Masked; use crate::arrays::MaskedArray; +use crate::arrays::masked::MaskedArrayExt; use crate::arrays::slice::SliceReduce; impl SliceReduce for Masked { diff --git a/vortex-array/src/arrays/masked/compute/take.rs b/vortex-array/src/arrays/masked/compute/take.rs index c1107c76467..c02a88a08fe 100644 --- a/vortex-array/src/arrays/masked/compute/take.rs +++ b/vortex-array/src/arrays/masked/compute/take.rs @@ -9,6 +9,7 @@ use crate::array::ArrayView; use crate::arrays::Masked; use crate::arrays::MaskedArray; use crate::arrays::dict::TakeReduce; +use crate::arrays::masked::MaskedArrayExt; use crate::builtins::ArrayBuiltins; use crate::scalar::Scalar; diff --git a/vortex-array/src/arrays/masked/execute.rs b/vortex-array/src/arrays/masked/execute.rs index b4f8266aabc..f1304d79166 100644 --- a/vortex-array/src/arrays/masked/execute.rs +++ b/vortex-array/src/arrays/masked/execute.rs @@ -21,6 +21,12 @@ use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; use crate::arrays::VarBinViewArray; use crate::arrays::VariantArray; +use crate::arrays::bool::BoolArrayExt; +use crate::arrays::extension::ExtensionArrayExt; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; +use crate::arrays::listview::ListViewArrayExt; +use crate::arrays::struct_::StructArrayExt; +use crate::arrays::variant::VariantArrayExt; use crate::dtype::Nullability; use crate::executor::ExecutionCtx; use crate::match_each_decimal_value_type; diff --git a/vortex-array/src/arrays/masked/mod.rs b/vortex-array/src/arrays/masked/mod.rs index 3343468e593..ac437d30321 100644 --- a/vortex-array/src/arrays/masked/mod.rs +++ b/vortex-array/src/arrays/masked/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::MaskedArrayExt; pub use array::MaskedData; pub use vtable::MaskedArray; diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 4a79d53b78b..337ce1b8dc3 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -4,12 +4,17 @@ mod canonical; mod operations; mod validity; +use std::hash::Hasher; + +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::ArrayEq; +use crate::ArrayHash; use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; @@ -18,9 +23,11 @@ use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; +use crate::array::validity_to_child; use crate::arrays::ConstantArray; +use crate::arrays::masked::MaskedArrayExt; use crate::arrays::masked::MaskedData; -use crate::arrays::masked::array::NUM_SLOTS; +use crate::arrays::masked::array::CHILD_SLOT; use crate::arrays::masked::array::SLOT_NAMES; use crate::arrays::masked::compute::rules::PARENT_RULES; use crate::arrays::masked::mask_validity_canonical; @@ -28,8 +35,6 @@ use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; use crate::executor::ExecutionResult; -use crate::hash::ArrayEq; -use crate::hash::ArrayHash; use crate::scalar::Scalar; use crate::serde::ArrayChildren; use crate::validity::Validity; @@ -43,6 +48,16 @@ impl Masked { pub const ID: ArrayId = ArrayId::new_ref("vortex.masked"); } +impl ArrayHash for MaskedData { + fn array_hash(&self, _state: &mut H, _precision: Precision) {} +} + +impl ArrayEq for MaskedData { + fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { + true + } +} + impl VTable for Masked { type ArrayData = MaskedData; @@ -53,28 +68,28 @@ impl VTable for Masked { Self::ID } - fn validate(&self, data: &MaskedData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + _data: &MaskedData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { vortex_ensure!( - data.child().len() == len, - "MaskedArray child length mismatch" + slots[CHILD_SLOT].is_some(), + "MaskedArray child slot must be present" ); + let child = slots[CHILD_SLOT] + .as_ref() + .vortex_expect("validated child slot"); + vortex_ensure!(child.len() == len, "MaskedArray child length mismatch"); vortex_ensure!( - data.dtype() == *dtype, + child.dtype().as_nullable() == *dtype, "MaskedArray dtype does not match child and validity" ); Ok(()) } - fn array_hash(array: &MaskedData, state: &mut H, precision: Precision) { - array.child().array_hash(state, precision); - array.validity().array_hash(state, precision); - } - - fn array_eq(array: &MaskedData, other: &MaskedData, precision: Precision) -> bool { - array.child().array_eq(other.child(), precision) - && array.validity().array_eq(&other.validity(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -100,7 +115,7 @@ impl VTable for Masked { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { if !metadata.is_empty() { vortex_bail!( "MaskedArray expects empty metadata, got {} bytes", @@ -126,11 +141,16 @@ impl VTable for Masked { Validity::from(dtype.nullability()) }; - MaskedData::try_new(child, validity) + let validity_slot = validity_to_child(&validity, len); + let data = MaskedData::try_new(len, child.all_valid()?, validity)?; + Ok( + crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, data) + .with_slots(vec![Some(child), validity_slot]), + ) } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - let validity_mask = array.validity_mask()?; + let validity_mask = array.masked_validity_mask(); // Fast path: all masked means result is all nulls. if validity_mask.all_false() { @@ -159,25 +179,9 @@ impl VTable for Masked { ) -> VortexResult> { PARENT_RULES.evaluate(array, parent, child_idx) } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "MaskedArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } } #[cfg(test)] diff --git a/vortex-array/src/arrays/masked/vtable/operations.rs b/vortex-array/src/arrays/masked/vtable/operations.rs index e90b65f5315..eb3132cf957 100644 --- a/vortex-array/src/arrays/masked/vtable/operations.rs +++ b/vortex-array/src/arrays/masked/vtable/operations.rs @@ -7,6 +7,7 @@ use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::Masked; +use crate::arrays::masked::MaskedArrayExt; use crate::scalar::Scalar; impl OperationsVTable for Masked { diff --git a/vortex-array/src/arrays/masked/vtable/validity.rs b/vortex-array/src/arrays/masked/vtable/validity.rs index 6386d23dbb5..e79c882e003 100644 --- a/vortex-array/src/arrays/masked/vtable/validity.rs +++ b/vortex-array/src/arrays/masked/vtable/validity.rs @@ -5,11 +5,12 @@ use vortex_error::VortexResult; use crate::array::ArrayView; use crate::array::ValidityVTable; +use crate::arrays::masked::MaskedArrayExt; use crate::arrays::masked::vtable::Masked; use crate::validity::Validity; impl ValidityVTable for Masked { fn validity(array: ArrayView<'_, Masked>) -> VortexResult { - Ok(array.data().validity()) + Ok(array.masked_validity()) } } diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index 183a2008654..e2935a49116 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -1,11 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::hash::Hasher; + use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::ArrayEq; +use crate::ArrayHash; use crate::ArrayRef; use crate::ExecutionCtx; use crate::ExecutionResult; @@ -25,12 +29,23 @@ use crate::serde::ArrayChildren; use crate::validity::Validity; use crate::vtable; -const NUM_SLOTS: usize = 0; - pub(crate) mod compute; vtable!(Null, Null, NullData); +impl ArrayHash for NullData { + fn array_hash(&self, _state: &mut H, _precision: Precision) { + // len and dtype are hashed by ArrayInner; NullData has no additional fields. + } +} + +impl ArrayEq for NullData { + fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { + // len, dtype, and slots are compared by ArrayInner; NullData has no additional fields. + true + } +} + impl VTable for Null { type ArrayData = NullData; @@ -41,20 +56,17 @@ impl VTable for Null { Self::ID } - fn validate(&self, _data: &NullData, dtype: &DType, _len: usize) -> VortexResult<()> { + fn validate( + &self, + _data: &NullData, + dtype: &DType, + _len: usize, + _slots: &[Option], + ) -> VortexResult<()> { vortex_ensure!(*dtype == DType::Null, "NullArray dtype must be DType::Null"); Ok(()) } - fn array_hash(_array: &NullData, _state: &mut H, _precision: Precision) { - // len and dtype are hashed by ArrayInner; NullData has no additional fields. - } - - fn array_eq(_array: &NullData, _other: &NullData, _precision: Precision) -> bool { - // len and dtype are compared by ArrayInner; NullData has no additional fields. - true - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -67,44 +79,35 @@ impl VTable for Null { None } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { vortex_panic!("NullArray slot_name index {idx} out of bounds") } - fn with_slots(_array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "NullArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - Ok(()) - } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( &self, - _dtype: &DType, - _len: usize, + dtype: &DType, + len: usize, metadata: &[u8], _buffers: &[BufferHandle], _children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { vortex_ensure!( metadata.is_empty(), "NullArray expects empty metadata, got {} bytes", metadata.len() ); - Ok(NullData::new()) + Ok(ArrayParts::new( + self.clone(), + dtype.clone(), + len, + NullData::new(), + )) } fn reduce_parent( @@ -148,9 +151,7 @@ impl VTable for Null { /// # } /// ``` #[derive(Clone, Debug)] -pub struct NullData { - slots: Vec>, -} +pub struct NullData; #[derive(Clone, Debug)] pub struct Null; @@ -175,7 +176,7 @@ impl Default for NullData { impl NullData { pub fn new() -> Self { - Self { slots: vec![] } + Self } } impl OperationsVTable for Null { diff --git a/vortex-array/src/arrays/patched/array.rs b/vortex-array/src/arrays/patched/array.rs index 0f1d5ba4ca3..7a917a0daf3 100644 --- a/vortex-array/src/arrays/patched/array.rs +++ b/vortex-array/src/arrays/patched/array.rs @@ -16,11 +16,13 @@ use crate::ExecutionCtx; use crate::IntoArray; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::arrays::Patched; use crate::arrays::PrimitiveArray; use crate::arrays::patched::TransposedPatches; use crate::arrays::patched::patch_lanes; use crate::buffer::BufferHandle; +use crate::dtype::DType; use crate::dtype::IntegerPType; use crate::dtype::NativePType; use crate::dtype::PType; @@ -42,14 +44,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["inner", "lane_offsets", "patch_indices", "patch_values"]; #[derive(Debug, Clone)] -pub struct PatchedArray { - /// Child arrays stored as slots: - /// 0: inner - the inner array being patched - /// 1: lane_offsets - u32 array for indexing into indices/values - /// 2: indices - u16 array of chunk indices - /// 3: values - array of patch values - pub(super) slots: Vec>, - +pub struct PatchedData { /// Number of lanes the patch indices and values have been split into. Each of the `n_chunks` /// of 1024 values is split into `n_lanes` lanes horizontally, each lane having 1024 / n_lanes /// values that might be patched. @@ -63,37 +58,157 @@ pub struct PatchedArray { pub(super) offset: usize, } -impl IntoArray for PatchedArray { - fn into_array(self) -> ArrayRef { - let dtype = self.base_array().dtype().clone(); - let len = self.len(); - unsafe { - Array::::from_parts_unchecked(ArrayParts::new(Patched, dtype, len, self)) - } - .into_array() +impl PatchedData { + pub(crate) fn validate( + &self, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let base_array = base_array_from_slots(slots); + let patch_indices = patch_indices_from_slots(slots); + let patch_values = patch_values_from_slots(slots); + + vortex_ensure!( + base_array.dtype() == dtype, + "PatchedArray base dtype {} does not match outer dtype {}", + base_array.dtype(), + dtype + ); + vortex_ensure!( + base_array.len() == len, + "PatchedArray base len {} does not match outer len {}", + base_array.len(), + len + ); + vortex_ensure!( + patch_indices.len() == patch_values.len(), + "PatchedArray patch indices len {} does not match patch values len {}", + patch_indices.len(), + patch_values.len() + ); + Ok(()) } } -impl From for ArrayRef { - fn from(value: PatchedArray) -> Self { - value.into_array() +fn base_array_from_slots(slots: &[Option]) -> &ArrayRef { + slots[INNER_SLOT] + .as_ref() + .vortex_expect("PatchedArray inner slot") +} + +fn lane_offsets_from_slots(slots: &[Option]) -> &ArrayRef { + slots[LANE_OFFSETS_SLOT] + .as_ref() + .vortex_expect("PatchedArray lane_offsets slot") +} + +fn patch_indices_from_slots(slots: &[Option]) -> &ArrayRef { + slots[INDICES_SLOT] + .as_ref() + .vortex_expect("PatchedArray indices slot") +} + +fn patch_values_from_slots(slots: &[Option]) -> &ArrayRef { + slots[VALUES_SLOT] + .as_ref() + .vortex_expect("PatchedArray values slot") +} + +pub trait PatchedArrayExt: TypedArrayRef { + #[inline] + fn base_array(&self) -> &ArrayRef { + base_array_from_slots(self.as_ref().slots()) + } + + #[inline] + fn lane_offsets(&self) -> &ArrayRef { + lane_offsets_from_slots(self.as_ref().slots()) + } + + #[inline] + fn patch_indices(&self) -> &ArrayRef { + patch_indices_from_slots(self.as_ref().slots()) + } + + #[inline] + fn patch_values(&self) -> &ArrayRef { + patch_values_from_slots(self.as_ref().slots()) + } + + #[inline] + fn n_lanes(&self) -> usize { + self.n_lanes + } + + #[inline] + fn offset(&self) -> usize { + self.offset + } + + #[inline] + fn lane_range(&self, chunk: usize, lane: usize) -> VortexResult> { + assert!(chunk * 1024 <= self.as_ref().len() + self.offset()); + assert!(lane < self.n_lanes()); + + let start = self + .lane_offsets() + .scalar_at(chunk * self.n_lanes() + lane)?; + let stop = self + .lane_offsets() + .scalar_at(chunk * self.n_lanes() + lane + 1)?; + + let start = start + .as_primitive() + .as_::() + .ok_or_else(|| vortex_err!("could not cast lane_offset to usize"))?; + + let stop = stop + .as_primitive() + .as_::() + .ok_or_else(|| vortex_err!("could not cast lane_offset to usize"))?; + + Ok(start..stop) + } + + fn slice_chunks(&self, chunks: Range) -> VortexResult> { + let lane_offsets_start = chunks.start * self.n_lanes(); + let lane_offsets_stop = chunks.end * self.n_lanes() + 1; + + let sliced_lane_offsets = self + .lane_offsets() + .slice(lane_offsets_start..lane_offsets_stop)?; + let indices = self.patch_indices().clone(); + let values = self.patch_values().clone(); + + let begin = (chunks.start * 1024).saturating_sub(self.offset()); + let end = (chunks.end * 1024) + .saturating_sub(self.offset()) + .min(self.as_ref().len()); + + let offset = if chunks.start == 0 { self.offset() } else { 0 }; + let inner = self.base_array().slice(begin..end)?; + let len = inner.len(); + let dtype = self.as_ref().dtype().clone(); + let slots = vec![ + Some(inner), + Some(sliced_lane_offsets), + Some(indices), + Some(values), + ]; + + Ok(unsafe { Patched::new_unchecked(dtype, len, slots, self.n_lanes(), offset) }) } } -impl PatchedArray { - /// Create a new `PatchedArray` from a child array and a set of [`Patches`]. - /// - /// # Errors - /// - /// The `inner` array must be primitive type, and it must have the same `DType` as the patches. - /// - /// The patches cannot contain nulls themselves. Any nulls must be stored in the `inner` array's - /// validity. +impl> PatchedArrayExt for T {} + +impl Patched { pub fn from_array_and_patches( inner: ArrayRef, patches: &Patches, ctx: &mut ExecutionCtx, - ) -> VortexResult { + ) -> VortexResult> { vortex_ensure!( inner.dtype().eq_with_nullability_superset(patches.dtype()), "array DType must match patches DType" @@ -142,120 +257,25 @@ impl PatchedArray { ) .into_array(); - Ok(Self { - slots: vec![Some(inner), Some(lane_offsets), Some(indices), Some(values)], - n_lanes, - offset: 0, - }) - } -} - -impl PatchedArray { - /// Returns a reference to the base array being patched. - #[inline] - pub fn base_array(&self) -> &ArrayRef { - self.slots[INNER_SLOT] - .as_ref() - .vortex_expect("PatchedArray inner slot") - } - - /// Returns a reference to the lane offsets array (u32). - #[inline] - pub fn lane_offsets(&self) -> &ArrayRef { - self.slots[LANE_OFFSETS_SLOT] - .as_ref() - .vortex_expect("PatchedArray lane_offsets slot") - } - - /// Returns a reference to the indices array (u16). - #[inline] - pub fn patch_indices(&self) -> &ArrayRef { - self.slots[INDICES_SLOT] - .as_ref() - .vortex_expect("PatchedArray indices slot") - } - - /// Returns a reference to the patch values array. - #[inline] - pub fn patch_values(&self) -> &ArrayRef { - self.slots[VALUES_SLOT] - .as_ref() - .vortex_expect("PatchedArray values slot") - } - - #[inline] - pub fn len(&self) -> usize { - self.base_array().len() - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -impl PatchedArray { - /// Get a range of indices that can be used to access the `indices` and `values` children - /// to retrieve all patches for a specified lane. - /// - /// # Panics - /// - /// Note that this function will panic if the caller requests out of bounds chunk/lane ordinals. - pub(crate) fn lane_range(&self, chunk: usize, lane: usize) -> VortexResult> { - assert!(chunk * 1024 <= self.len() + self.offset); - assert!(lane < self.n_lanes); - - let start = self.lane_offsets().scalar_at(chunk * self.n_lanes + lane)?; - let stop = self - .lane_offsets() - .scalar_at(chunk * self.n_lanes + lane + 1)?; - - let start = start - .as_primitive() - .as_::() - .ok_or_else(|| vortex_err!("could not cast lane_offset to usize"))?; - - let stop = stop - .as_primitive() - .as_::() - .ok_or_else(|| vortex_err!("could not cast lane_offset to usize"))?; - - Ok(start..stop) + let dtype = inner.dtype().clone(); + let len = inner.len(); + let slots = vec![Some(inner), Some(lane_offsets), Some(indices), Some(values)]; + Ok(unsafe { Self::new_unchecked(dtype, len, slots, n_lanes, 0) }) } - /// Slice the array to just the patches and inner values that are within the chunk range. - pub(crate) fn slice_chunks(&self, chunks: Range) -> VortexResult { - let lane_offsets_start = chunks.start * self.n_lanes; - let lane_offsets_stop = chunks.end * self.n_lanes + 1; - - let sliced_lane_offsets = self - .lane_offsets() - .slice(lane_offsets_start..lane_offsets_stop)?; - let indices = self.patch_indices().clone(); - let values = self.patch_values().clone(); - - // Find the new start/end for slicing the inner array. - // The inner array has already been sliced to start at position `offset` in absolute terms, - // so we need to convert chunk boundaries to inner-relative coordinates. - let begin = (chunks.start * 1024).saturating_sub(self.offset); - let end = (chunks.end * 1024) - .saturating_sub(self.offset) - .min(self.len()); - - let offset = if chunks.start == 0 { self.offset } else { 0 }; - - let inner = self.base_array().slice(begin..end)?; - - Ok(PatchedArray { - slots: vec![ - Some(inner), - Some(sliced_lane_offsets), - Some(indices), - Some(values), - ], - n_lanes: self.n_lanes, - offset, - }) + pub(crate) unsafe fn new_unchecked( + dtype: DType, + len: usize, + slots: Vec>, + n_lanes: usize, + offset: usize, + ) -> Array { + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Patched, dtype, len, PatchedData { n_lanes, offset }) + .with_slots(slots), + ) + } } } @@ -318,10 +338,10 @@ fn transpose( let mut indices_buffer = BufferMut::with_capacity(indices_in.len()); let mut values_buffer = BufferMut::with_capacity(values_in.len()); - // number of patches in each chunk. + // Number of patches in each chunk/lane. let mut lane_offsets: BufferMut = BufferMut::zeroed(n_chunks * n_lanes + 1); - // Scan the index/values once to get chunk/lane counts + // Scan the index/value pairs once to get chunk/lane counts. for index in indices_in { let index = index.as_() - offset; let chunk = index / 1024; @@ -330,12 +350,11 @@ fn transpose( lane_offsets[chunk * n_lanes + lane + 1] += 1; } - // Prefix-sum sizes -> offsets for index in 1..lane_offsets.len() { lane_offsets[index] += lane_offsets[index - 1]; } - // Loop over patches, writing them to final positions + // Loop over patches, writing them to final positions. let indices_out = indices_buffer.spare_capacity_mut(); let values_out = values_buffer.spare_capacity_mut(); for (index, &value) in std::iter::zip(indices_in, values_in) { @@ -349,14 +368,11 @@ fn transpose( *position += 1; } - // SAFETY: we know there are exactly indices_in.len() indices/values, and we just - // set them to the appropriate values in the loop above. unsafe { indices_buffer.set_len(indices_in.len()); values_buffer.set_len(values_in.len()); } - // Now, pass over all the indices and values again and subtract out the position increments. for index in indices_in { let index = index.as_() - offset; let chunk = index / 1024; diff --git a/vortex-array/src/arrays/patched/compute/compare.rs b/vortex-array/src/arrays/patched/compute/compare.rs index 54c89c1744d..f491249a28d 100644 --- a/vortex-array/src/arrays/patched/compute/compare.rs +++ b/vortex-array/src/arrays/patched/compute/compare.rs @@ -10,11 +10,13 @@ use crate::Canonical; use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; +use crate::array::child_to_validity; use crate::arrays::BoolArray; use crate::arrays::ConstantArray; use crate::arrays::Patched; use crate::arrays::PrimitiveArray; use crate::arrays::bool::BoolDataParts; +use crate::arrays::patched::PatchedArrayExt; use crate::arrays::primitive::NativeValue; use crate::builtins::ArrayBuiltins; use crate::dtype::NativePType; @@ -50,23 +52,19 @@ impl CompareKernel for Patched { .execute::(ctx)? .into_bool(); - let result = result.into_parts(); - let BoolDataParts { - bits, - offset, - len, - validity, - } = result.data.into_parts(); + let validity = child_to_validity(&result.slots()[0], result.dtype().nullability()); + let len = result.len(); + let BoolDataParts { bits, offset, len } = result.into_data().into_parts(len); let mut bits = BitBufferMut::from_buffer(bits.unwrap_host().into_mut(), offset, len); let lane_offsets = lhs.lane_offsets().clone().execute::(ctx)?; let indices = lhs.patch_indices().clone().execute::(ctx)?; let values = lhs.patch_values().clone().execute::(ctx)?; - let n_lanes = lhs.n_lanes; + let n_lanes = lhs.n_lanes(); match_each_native_ptype!(values.ptype(), |V| { - let offset = lhs.offset; + let offset = lhs.offset(); let indices = indices.as_slice::(); let values = values.as_slice::(); let constant = constant @@ -168,7 +166,6 @@ mod tests { use crate::arrays::BoolArray; use crate::arrays::ConstantArray; use crate::arrays::Patched; - use crate::arrays::PatchedArray; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; use crate::optimizer::ArrayOptimizer; @@ -191,10 +188,10 @@ mod tests { let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let lhs = PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx) + let lhs = Patched::from_array_and_patches(lhs, &patches, &mut ctx) .unwrap() .into_array() - .try_into::() + .try_downcast::() .unwrap(); let rhs = ConstantArray::new(u32::MAX, 512).into_array(); @@ -224,10 +221,10 @@ mod tests { let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let lhs = PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx).unwrap(); + let lhs = Patched::from_array_and_patches(lhs, &patches, &mut ctx).unwrap(); // Slice the array so that the first patch should be skipped. let lhs_ref = lhs.into_array().slice(10..512).unwrap().optimize().unwrap(); - let lhs = lhs_ref.try_into::().unwrap(); + let lhs = lhs_ref.try_downcast::().unwrap(); assert_eq!(lhs.len(), 502); @@ -260,9 +257,9 @@ mod tests { )?; let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let lhs = PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx)? + let lhs = Patched::from_array_and_patches(lhs, &patches, &mut ctx)? .into_array() - .try_into::() + .try_downcast::() .unwrap(); let rhs = ConstantArray::new(subnormal, 512).into_array(); @@ -294,9 +291,9 @@ mod tests { )?; let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let lhs = PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx)? + let lhs = Patched::from_array_and_patches(lhs, &patches, &mut ctx)? .into_array() - .try_into::() + .try_downcast::() .unwrap(); let rhs = ConstantArray::new(0.0f32, 10).into_array(); diff --git a/vortex-array/src/arrays/patched/compute/filter.rs b/vortex-array/src/arrays/patched/compute/filter.rs index 0a2060964aa..d2c60941059 100644 --- a/vortex-array/src/arrays/patched/compute/filter.rs +++ b/vortex-array/src/arrays/patched/compute/filter.rs @@ -11,6 +11,7 @@ use crate::array::ArrayView; use crate::arrays::FilterArray; use crate::arrays::Patched; use crate::arrays::filter::FilterReduce; +use crate::arrays::patched::PatchedArrayExt; impl FilterReduce for Patched { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { @@ -31,13 +32,13 @@ impl FilterReduce for Patched { // Convert mask indices to absolute positions by adding offset ( - (array.offset + first) / 1024, - (array.offset + last).div_ceil(1024), + (array.offset() + first) / 1024, + (array.offset() + last).div_ceil(1024), ) } }; - let n_chunks = (array.offset + array.len()).div_ceil(1024); + let n_chunks = (array.offset() + array.len()).div_ceil(1024); // If all chunks already covered, there is nothing to do. if chunk_start == 0 && chunk_stop == n_chunks { @@ -48,9 +49,9 @@ impl FilterReduce for Patched { // Slice the mask according to if the chunk is sliced. // Convert chunk bounds back to mask indices by subtracting offset. - let mask_start = (chunk_start * 1024).saturating_sub(array.offset); + let mask_start = (chunk_start * 1024).saturating_sub(array.offset()); let mask_end = (chunk_stop * 1024) - .saturating_sub(array.offset) + .saturating_sub(array.offset()) .min(array.len()); let remainder = mask.slice(mask_start..mask_end); @@ -69,7 +70,7 @@ mod tests { use crate::ExecutionCtx; use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::arrays::PatchedArray; + use crate::arrays::Patched; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; use crate::optimizer::ArrayOptimizer; @@ -86,7 +87,7 @@ mod tests { let patches = Patches::new(5, 0, patched_indices, patched_values, None)?; - let array = PatchedArray::from_array_and_patches(array, &patches, &mut ctx)?.into_array(); + let array = Patched::from_array_and_patches(array, &patches, &mut ctx)?.into_array(); let mask = Mask::from_iter([true, false, false, false, true]); let filtered = array @@ -113,7 +114,7 @@ mod tests { let patches = Patches::new(4096, 0, patched_indices, patched_values, None)?; - let array = PatchedArray::from_array_and_patches(array, &patches, &mut ctx)?.into_array(); + let array = Patched::from_array_and_patches(array, &patches, &mut ctx)?.into_array(); let sliced = array.slice(5..4096)?.optimize()?; @@ -141,7 +142,7 @@ mod tests { let patches = Patches::new(4096, 0, patched_indices, patched_values, None)?; - let array = PatchedArray::from_array_and_patches(array, &patches, &mut ctx)?.into_array(); + let array = Patched::from_array_and_patches(array, &patches, &mut ctx)?.into_array(); // Filter that only touches the middle 2 chunks. let mask = Mask::from_indices(4096, vec![1024, 1025, 3000]); @@ -168,7 +169,7 @@ mod tests { let patches = Patches::new(4096, 1, patched_indices, patched_values, None)?; - let array = PatchedArray::from_array_and_patches(array, &patches, &mut ctx)?.into_array(); + let array = Patched::from_array_and_patches(array, &patches, &mut ctx)?.into_array(); // Filter that only touches the middle 2 chunks. let mask = Mask::from_indices(4096, vec![1024, 1025, 3000]); @@ -198,7 +199,7 @@ mod tests { let patches = Patches::new(6144, 0, patched_indices, patched_values, None)?; - let patched = PatchedArray::from_array_and_patches(array, &patches, &mut ctx)?; + let patched = Patched::from_array_and_patches(array, &patches, &mut ctx)?; // Slice at chunk boundary to create offset > 0. After slicing [1024..5120], // we have 4096 elements and patches are at relative indices 1024 and 1025. @@ -236,7 +237,7 @@ mod tests { let patched_values = buffer![9999u16, 8888].into_array(); let patches = Patches::new(4096, 0, patched_indices, patched_values, None)?; - let array = PatchedArray::from_array_and_patches(array, &patches, &mut ctx)?.into_array(); + let array = Patched::from_array_and_patches(array, &patches, &mut ctx)?.into_array(); // Slice to create offset > 0. // After slice(5..4096), logical position 0 = original position 5 (patched to 9999). @@ -272,7 +273,7 @@ mod tests { let patches = Patches::new(6144, 0, patched_indices, patched_values, None)?; - let patched = PatchedArray::from_array_and_patches(array, &patches, &mut ctx)?; + let patched = Patched::from_array_and_patches(array, &patches, &mut ctx)?; // Slice at chunk boundary to create offset > 0. // Slice [1024..6144] gives us 5120 elements (5 chunks). diff --git a/vortex-array/src/arrays/patched/compute/take.rs b/vortex-array/src/arrays/patched/compute/take.rs index 1257eec5169..b69984e4405 100644 --- a/vortex-array/src/arrays/patched/compute/take.rs +++ b/vortex-array/src/arrays/patched/compute/take.rs @@ -12,6 +12,7 @@ use crate::array::ArrayView; use crate::arrays::Patched; use crate::arrays::PrimitiveArray; use crate::arrays::dict::TakeExecute; +use crate::arrays::patched::PatchedArrayExt; use crate::arrays::primitive::PrimitiveDataParts; use crate::dtype::IntegerPType; use crate::dtype::NativePType; @@ -39,7 +40,7 @@ impl TakeExecute for Patched { buffer, validity, ptype, - } = inner.into_data().into_parts(); + } = inner.into_data_parts(); let indices_ptype = indices.dtype().as_ptype(); @@ -62,9 +63,9 @@ impl TakeExecute for Patched { take_map( output.as_mut(), indices.as_slice::(), - array.offset, + array.offset(), array.len(), - array.n_lanes, + array.n_lanes(), lane_offsets.as_slice::(), patch_indices.as_slice::(), patch_values.as_slice::(), @@ -136,7 +137,7 @@ mod tests { use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; - use crate::arrays::PatchedArray; + use crate::arrays::Patched; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; use crate::patches::Patches; @@ -159,7 +160,7 @@ mod tests { let session = VortexSession::empty(); let mut ctx = ExecutionCtx::new(session); - PatchedArray::from_array_and_patches(values, &patches, &mut ctx)? + Patched::from_array_and_patches(values, &patches, &mut ctx)? .into_array() .slice(slice) } diff --git a/vortex-array/src/arrays/patched/vtable/mod.rs b/vortex-array/src/arrays/patched/vtable/mod.rs index 0e592452e27..89086dfcbc3 100644 --- a/vortex-array/src/arrays/patched/vtable/mod.rs +++ b/vortex-array/src/arrays/patched/vtable/mod.rs @@ -2,6 +2,9 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use prost::Message; + +use crate::ArrayEq; +use crate::ArrayHash; mod kernels; mod operations; mod slice; @@ -12,12 +15,9 @@ use std::hash::Hasher; use vortex_buffer::Buffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; -use crate::ArrayEq; -use crate::ArrayHash; use crate::ArrayRef; use crate::Canonical; use crate::ExecutionCtx; @@ -26,13 +26,14 @@ use crate::IntoArray; use crate::Precision; use crate::array::Array; use crate::array::ArrayId; +use crate::array::ArrayParts; use crate::array::ArrayView; use crate::array::VTable; use crate::array::ValidityChild; use crate::array::ValidityVTableFromChild; use crate::arrays::PrimitiveArray; -use crate::arrays::patched::PatchedArray; -use crate::arrays::patched::array::NUM_SLOTS; +use crate::arrays::patched::PatchedArrayExt; +use crate::arrays::patched::PatchedData; use crate::arrays::patched::array::SLOT_NAMES; use crate::arrays::patched::compute::rules::PARENT_RULES; use crate::arrays::patched::vtable::kernels::PARENT_KERNELS; @@ -47,14 +48,14 @@ use crate::match_each_native_ptype; use crate::serde::ArrayChildren; use crate::vtable; -vtable!(Patched); +vtable!(Patched, Patched, PatchedData); #[derive(Clone, Debug)] pub struct Patched; impl ValidityChild for Patched { - fn validity_child(array: &PatchedArray) -> &ArrayRef { - array.base_array() + fn validity_child(array: ArrayView<'_, Patched>) -> ArrayRef { + array.base_array().clone() } } @@ -75,8 +76,21 @@ pub struct PatchedMetadata { pub(crate) offset: u32, } +impl ArrayHash for PatchedData { + fn array_hash(&self, state: &mut H, _precision: Precision) { + self.offset.hash(state); + self.n_lanes.hash(state); + } +} + +impl ArrayEq for PatchedData { + fn array_eq(&self, other: &Self, _precision: Precision) -> bool { + self.offset == other.offset && self.n_lanes == other.n_lanes + } +} + impl VTable for Patched { - type ArrayData = PatchedArray; + type ArrayData = PatchedData; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; @@ -84,54 +98,18 @@ impl VTable for Patched { ArrayId::new_ref("vortex.patched") } - fn array_hash(array: &PatchedArray, state: &mut H, precision: Precision) { - array.offset.hash(state); - array.n_lanes.hash(state); - array.base_array().array_hash(state, precision); - array.lane_offsets().array_hash(state, precision); - array.patch_indices().array_hash(state, precision); - array.patch_values().array_hash(state, precision); - } - - fn array_eq(array: &PatchedArray, other: &PatchedArray, precision: Precision) -> bool { - array.offset == other.offset - && array.n_lanes == other.n_lanes - && array.base_array().array_eq(other.base_array(), precision) - && array - .lane_offsets() - .array_eq(other.lane_offsets(), precision) - && array - .patch_indices() - .array_eq(other.patch_indices(), precision) - && array - .patch_values() - .array_eq(other.patch_values(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } - fn validate(&self, data: &PatchedArray, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.base_array().dtype() == dtype, - "PatchedArray base dtype {} does not match outer dtype {}", - data.base_array().dtype(), - dtype - ); - vortex_ensure!( - data.len() == len, - "PatchedArray base len {} does not match outer len {}", - data.len(), - len - ); - vortex_ensure!( - data.patch_indices().len() == data.patch_values().len(), - "PatchedArray patch indices len {} does not match patch values len {}", - data.patch_indices().len(), - data.patch_values().len() - ); - Ok(()) + fn validate( + &self, + data: &PatchedData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + data.validate(dtype, len, slots) } fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { @@ -156,8 +134,8 @@ impl VTable for Patched { Ok(Some( PatchedMetadata { n_patches: u32::try_from(array.patch_indices().len())?, - n_lanes: u32::try_from(array.n_lanes)?, - offset: u32::try_from(array.offset)?, + n_lanes: u32::try_from(array.n_lanes())?, + offset: u32::try_from(array.offset())?, } .encode_to_vec(), )) @@ -171,7 +149,7 @@ impl VTable for Patched { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = PatchedMetadata::decode(metadata)?; let n_patches = metadata.n_patches as usize; let n_lanes = metadata.n_lanes as usize; @@ -186,11 +164,15 @@ impl VTable for Patched { let indices = children.get(2, PType::U16.into(), n_patches)?; let values = children.get(3, dtype, n_patches)?; - Ok(PatchedArray { - slots: vec![Some(inner), Some(lane_offsets), Some(indices), Some(values)], - n_lanes, - offset, - }) + let data = PatchedData { n_lanes, offset }; + Ok( + ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(vec![ + Some(inner), + Some(lane_offsets), + Some(indices), + Some(values), + ]), + ) } fn append_to_builder( @@ -217,7 +199,7 @@ impl VTable for Patched { array.base_array().append_to_builder(builder, ctx)?; - let offset = array.offset; + let offset = array.offset(); let lane_offsets = array .lane_offsets() .clone() @@ -246,7 +228,7 @@ impl VTable for Patched { &mut output[trailer..], offset, len, - array.n_lanes, + array.n_lanes(), lane_offsets.as_slice::(), indices.as_slice::(), values.as_slice::(), @@ -256,25 +238,10 @@ impl VTable for Patched { Ok(()) } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "PatchedArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { let inner = array .base_array() @@ -286,7 +253,7 @@ impl VTable for Patched { buffer, ptype, validity, - } = inner.into_data().into_parts(); + } = inner.into_data_parts(); let lane_offsets = array .lane_offsets() @@ -304,7 +271,7 @@ impl VTable for Patched { .execute::(ctx)?; let patched_values = match_each_native_ptype!(values.ptype(), |V| { - let offset = array.offset; + let offset = array.offset(); let len = array.len(); let mut output = Buffer::::from_byte_buffer(buffer.unwrap_host()).into_mut(); @@ -313,7 +280,7 @@ impl VTable for Patched { &mut output, offset, len, - array.n_lanes, + array.n_lanes(), lane_offsets.as_slice::(), indices.as_slice::(), values.as_slice::(), @@ -392,6 +359,7 @@ mod tests { use crate::arrays::Patched; use crate::arrays::PatchedArray; use crate::arrays::PrimitiveArray; + use crate::arrays::patched::array::PatchedArrayExt; use crate::assert_arrays_eq; use crate::builders::builder_with_capacity; use crate::patches::Patches; @@ -414,7 +382,7 @@ mod tests { let session = VortexSession::empty(); let mut ctx = ExecutionCtx::new(session); - let array = PatchedArray::from_array_and_patches(values, &patches, &mut ctx) + let array = Patched::from_array_and_patches(values, &patches, &mut ctx) .unwrap() .into_array(); @@ -447,7 +415,7 @@ mod tests { let session = VortexSession::empty(); let mut ctx = ExecutionCtx::new(session); - let array = PatchedArray::from_array_and_patches(values, &patches, &mut ctx) + let array = Patched::from_array_and_patches(values, &patches, &mut ctx) .unwrap() .into_array() .slice(3..1024) @@ -480,7 +448,7 @@ mod tests { let session = VortexSession::empty(); let mut ctx = ExecutionCtx::new(session); - let array = PatchedArray::from_array_and_patches(values, &patches, &mut ctx) + let array = Patched::from_array_and_patches(values, &patches, &mut ctx) .unwrap() .into_array(); @@ -513,7 +481,7 @@ mod tests { let session = VortexSession::empty(); let mut ctx = ExecutionCtx::new(session); - let array = PatchedArray::from_array_and_patches(values, &patches, &mut ctx) + let array = Patched::from_array_and_patches(values, &patches, &mut ctx) .unwrap() .into_array() .slice(3..1024) @@ -550,7 +518,7 @@ mod tests { let session = VortexSession::empty(); let mut ctx = ExecutionCtx::new(session); - let array = PatchedArray::from_array_and_patches(values, &patches, &mut ctx) + let array = Patched::from_array_and_patches(values, &patches, &mut ctx) .unwrap() .into_array(); @@ -594,7 +562,7 @@ mod tests { let session = VortexSession::empty(); let mut ctx = ExecutionCtx::new(session); - PatchedArray::from_array_and_patches(array, &patches, &mut ctx) + Patched::from_array_and_patches(array, &patches, &mut ctx) } #[rstest] diff --git a/vortex-array/src/arrays/patched/vtable/operations.rs b/vortex-array/src/arrays/patched/vtable/operations.rs index 87b29392cb7..35e8bad117b 100644 --- a/vortex-array/src/arrays/patched/vtable/operations.rs +++ b/vortex-array/src/arrays/patched/vtable/operations.rs @@ -8,6 +8,7 @@ use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::PrimitiveArray; use crate::arrays::patched::Patched; +use crate::arrays::patched::PatchedArrayExt; use crate::optimizer::ArrayOptimizer; use crate::scalar::Scalar; @@ -17,15 +18,15 @@ impl OperationsVTable for Patched { index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { - let chunk = (index + array.offset) / 1024; + let chunk = (index + array.offset()) / 1024; #[expect( clippy::cast_possible_truncation, reason = "N % 1024 always fits in u16" )] - let chunk_index = ((index + array.offset) % 1024) as u16; + let chunk_index = ((index + array.offset()) % 1024) as u16; - let lane = (index + array.offset) % array.n_lanes; + let lane = (index + array.offset()) % array.n_lanes(); let range = array.lane_range(chunk, lane)?; @@ -58,7 +59,6 @@ mod tests { use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::Patched; - use crate::arrays::PatchedArray; use crate::dtype::Nullability; use crate::optimizer::ArrayOptimizer; use crate::patches::Patches; @@ -79,7 +79,7 @@ mod tests { let session = VortexSession::empty(); let mut ctx = ExecutionCtx::new(session); - let array = PatchedArray::from_array_and_patches(values, &patches, &mut ctx) + let array = Patched::from_array_and_patches(values, &patches, &mut ctx) .unwrap() .into_array(); @@ -116,7 +116,7 @@ mod tests { let session = VortexSession::empty(); let mut ctx = ExecutionCtx::new(session); - let array = PatchedArray::from_array_and_patches(values, &patches, &mut ctx) + let array = Patched::from_array_and_patches(values, &patches, &mut ctx) .unwrap() .into_array(); @@ -146,7 +146,7 @@ mod tests { let session = VortexSession::empty(); let mut ctx = ExecutionCtx::new(session); - let array = PatchedArray::from_array_and_patches(values, &patches, &mut ctx) + let array = Patched::from_array_and_patches(values, &patches, &mut ctx) .unwrap() .into_array() .slice(3..4096) diff --git a/vortex-array/src/arrays/patched/vtable/slice.rs b/vortex-array/src/arrays/patched/vtable/slice.rs index bbdb4df2464..ef3212ab820 100644 --- a/vortex-array/src/arrays/patched/vtable/slice.rs +++ b/vortex-array/src/arrays/patched/vtable/slice.rs @@ -9,19 +9,19 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Patched; -use crate::arrays::PatchedArray; +use crate::arrays::patched::PatchedArrayExt; use crate::arrays::slice::SliceReduce; impl SliceReduce for Patched { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // We **always** slice the patches at 1024-element chunk boundaries. We keep the offset + len // around so that when we execute we know how much to chop off. - let new_offset = (range.start + array.offset) % 1024; - let chunk_start = (range.start + array.offset) / 1024; - let chunk_stop = (range.end + array.offset).div_ceil(1024); + let new_offset = (range.start + array.offset()) % 1024; + let chunk_start = (range.start + array.offset()) / 1024; + let chunk_stop = (range.end + array.offset()).div_ceil(1024); let sliced_lane_offsets = array .lane_offsets() - .slice((chunk_start * array.n_lanes)..(chunk_stop * array.n_lanes) + 1)?; + .slice((chunk_start * array.n_lanes())..(chunk_stop * array.n_lanes()) + 1)?; // Unlike the patches, we slice the inner to the exact range. This is handled // at execution time by making sure to skip patch indices that are < offset @@ -29,15 +29,19 @@ impl SliceReduce for Patched { let inner = array.base_array().slice(range.start..range.end)?; Ok(Some( - PatchedArray { - slots: vec![ - Some(inner), - Some(sliced_lane_offsets), - Some(array.patch_indices().clone()), - Some(array.patch_values().clone()), - ], - n_lanes: array.n_lanes, - offset: new_offset, + unsafe { + Patched::new_unchecked( + array.dtype().clone(), + inner.len(), + vec![ + Some(inner), + Some(sliced_lane_offsets), + Some(array.patch_indices().clone()), + Some(array.patch_values().clone()), + ], + array.n_lanes(), + new_offset, + ) } .into_array(), )) @@ -58,7 +62,7 @@ mod tests { use crate::ExecutionCtx; use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::arrays::PatchedArray; + use crate::arrays::Patched; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; use crate::dtype::NativePType; @@ -73,7 +77,7 @@ mod tests { let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let patched_array = PatchedArray::from_array_and_patches(values, &patches, &mut ctx)?; + let patched_array = Patched::from_array_and_patches(values, &patches, &mut ctx)?; let sliced = patched_array.into_array().slice(1..10)?; @@ -119,10 +123,9 @@ mod tests { let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let patched_array = - PatchedArray::from_array_and_patches(inner.into_array(), &patches, &mut ctx) - .unwrap() - .into_array(); + let patched_array = Patched::from_array_and_patches(inner.into_array(), &patches, &mut ctx) + .unwrap() + .into_array(); // Verify that applying slice first yields same result as applying slice at end. let slice_first = patched_array @@ -152,7 +155,7 @@ mod tests { let patches = Patches::new(10_000, 0, patched_indices, patched_values, None).unwrap(); let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let patched_array = PatchedArray::from_array_and_patches(values, &patches, &mut ctx) + let patched_array = Patched::from_array_and_patches(values, &patches, &mut ctx) .unwrap() .into_array(); diff --git a/vortex-array/src/arrays/primitive/array/cast.rs b/vortex-array/src/arrays/primitive/array/cast.rs index 65406688afe..0023ea71fb7 100644 --- a/vortex-array/src/arrays/primitive/array/cast.rs +++ b/vortex-array/src/arrays/primitive/array/cast.rs @@ -1,22 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use vortex_buffer::Buffer; use vortex_error::VortexExpect; -use vortex_error::VortexResult; use vortex_error::vortex_panic; use super::PrimitiveData; -use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::ToCanonical; -use crate::VortexSessionExecute; -use crate::aggregate_fn::fns::min_max::min_max; -use crate::arrays::PrimitiveArray; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; use crate::dtype::NativePType; -use crate::dtype::PType; impl PrimitiveData { /// Return a slice of the array's buffer. @@ -44,127 +33,6 @@ impl PrimitiveData { // SAFETY: alignment of Buffer is checked on construction unsafe { std::slice::from_raw_parts(raw_slice.cast(), byte_buffer.len() / size_of::()) } } - - pub fn reinterpret_cast(&self, ptype: PType) -> Self { - if self.ptype() == ptype { - return self.clone(); - } - - assert_eq!( - self.ptype().byte_width(), - ptype.byte_width(), - "can't reinterpret cast between integers of two different widths" - ); - - PrimitiveData::from_buffer_handle(self.buffer_handle().clone(), ptype, self.validity()) - } -} - -impl PrimitiveArray { - pub fn reinterpret_cast(&self, ptype: PType) -> Self { - if self.ptype() == ptype { - return self.clone(); - } - - assert_eq!( - self.ptype().byte_width(), - ptype.byte_width(), - "can't reinterpret cast between integers of two different widths" - ); - - PrimitiveArray::from_buffer_handle( - self.buffer_handle().clone(), - ptype, - self.validity() - .vortex_expect("primitive validity should be derivable"), - ) - } - - /// Narrow the array to the smallest possible integer type that can represent all values. - pub fn narrow(&self) -> VortexResult { - if !self.ptype().is_int() { - return Ok(self.clone()); - } - - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let Some(min_max) = min_max(&self.clone().into_array(), &mut ctx)? else { - return Ok(PrimitiveArray::new( - Buffer::::zeroed(self.len()), - self.validity()?, - )); - }; - - // If we can't cast to i64, then leave the array as its original type. - // It's too big to downcast anyway. - let Ok(min) = min_max - .min - .cast(&PType::I64.into()) - .and_then(|s| i64::try_from(&s)) - else { - return Ok(self.clone()); - }; - let Ok(max) = min_max - .max - .cast(&PType::I64.into()) - .and_then(|s| i64::try_from(&s)) - else { - return Ok(self.clone()); - }; - - if min < 0 || max < 0 { - // Signed - if min >= i8::MIN as i64 && max <= i8::MAX as i64 { - return Ok(self - .clone() - .into_array() - .cast(DType::Primitive(PType::I8, self.dtype().nullability()))? - .to_primitive()); - } - - if min >= i16::MIN as i64 && max <= i16::MAX as i64 { - return Ok(self - .clone() - .into_array() - .cast(DType::Primitive(PType::I16, self.dtype().nullability()))? - .to_primitive()); - } - - if min >= i32::MIN as i64 && max <= i32::MAX as i64 { - return Ok(self - .clone() - .into_array() - .cast(DType::Primitive(PType::I32, self.dtype().nullability()))? - .to_primitive()); - } - } else { - // Unsigned - if max <= u8::MAX as i64 { - return Ok(self - .clone() - .into_array() - .cast(DType::Primitive(PType::U8, self.dtype().nullability()))? - .to_primitive()); - } - - if max <= u16::MAX as i64 { - return Ok(self - .clone() - .into_array() - .cast(DType::Primitive(PType::U16, self.dtype().nullability()))? - .to_primitive()); - } - - if max <= u32::MAX as i64 { - return Ok(self - .clone() - .into_array() - .cast(DType::Primitive(PType::U32, self.dtype().nullability()))? - .to_primitive()); - } - } - - Ok(self.clone()) - } } #[cfg(test)] @@ -174,6 +42,7 @@ mod tests { use vortex_buffer::buffer; use crate::arrays::PrimitiveArray; + use crate::arrays::primitive::PrimitiveArrayExt; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index 25cbf8c755b..270e3b17ab5 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -13,10 +13,14 @@ use vortex_error::VortexResult; use vortex_error::vortex_err; use vortex_error::vortex_panic; +use crate::LEGACY_SESSION; use crate::ToCanonical; +use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::arrays::Primitive; +use crate::arrays::PrimitiveArray; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::Nullability; @@ -34,9 +38,12 @@ pub use patch::chunk_range; pub use patch::patch_chunk; use crate::ArrayRef; +use crate::aggregate_fn::fns::min_max::min_max; use crate::array::child_to_validity; use crate::array::validity_to_child; +use crate::arrays::bool::BoolArrayExt; use crate::buffer::BufferHandle; +use crate::builtins::ArrayBuiltins; /// The validity bitmap indicating which elements are non-null. pub(super) const VALIDITY_SLOT: usize = 0; @@ -75,9 +82,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; /// ``` #[derive(Clone, Debug)] pub struct PrimitiveData { - pub(super) slots: Vec>, pub(super) ptype: PType, - pub(super) nullability: Nullability, pub(super) buffer: BufferHandle, } @@ -87,6 +92,135 @@ pub struct PrimitiveDataParts { pub validity: Validity, } +pub trait PrimitiveArrayExt: TypedArrayRef { + fn ptype(&self) -> PType { + match self.as_ref().dtype() { + DType::Primitive(ptype, _) => *ptype, + _ => unreachable!("PrimitiveArrayExt requires a primitive dtype"), + } + } + + fn nullability(&self) -> Nullability { + match self.as_ref().dtype() { + DType::Primitive(_, nullability) => *nullability, + _ => unreachable!("PrimitiveArrayExt requires a primitive dtype"), + } + } + + fn validity_child(&self) -> Option<&ArrayRef> { + self.as_ref().slots()[VALIDITY_SLOT].as_ref() + } + + fn validity(&self) -> Validity { + child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) + } + + fn validity_mask(&self) -> vortex_mask::Mask { + self.validity().to_mask(self.as_ref().len()) + } + + fn buffer_handle(&self) -> &BufferHandle { + &self.buffer + } + + fn reinterpret_cast(&self, ptype: PType) -> PrimitiveArray { + if self.ptype() == ptype { + return self.to_owned(); + } + + assert_eq!( + self.ptype().byte_width(), + ptype.byte_width(), + "can't reinterpret cast between integers of two different widths" + ); + + PrimitiveArray::from_buffer_handle(self.buffer_handle().clone(), ptype, self.validity()) + } + + /// Narrow the array to the smallest possible integer type that can represent all values. + fn narrow(&self) -> VortexResult { + if !self.ptype().is_int() { + return Ok(self.to_owned()); + } + + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let Some(min_max) = min_max(self.as_ref(), &mut ctx)? else { + return Ok(PrimitiveArray::new( + Buffer::::zeroed(self.len()), + self.validity(), + )); + }; + + // If we can't cast to i64, then leave the array as its original type. + // It's too big to downcast anyway. + let Ok(min) = min_max + .min + .cast(&PType::I64.into()) + .and_then(|s| i64::try_from(&s)) + else { + return Ok(self.to_owned()); + }; + let Ok(max) = min_max + .max + .cast(&PType::I64.into()) + .and_then(|s| i64::try_from(&s)) + else { + return Ok(self.to_owned()); + }; + + let nullability = self.as_ref().dtype().nullability(); + + if min < 0 || max < 0 { + // Signed + if min >= i8::MIN as i64 && max <= i8::MAX as i64 { + return Ok(self + .as_ref() + .cast(DType::Primitive(PType::I8, nullability))? + .to_primitive()); + } + + if min >= i16::MIN as i64 && max <= i16::MAX as i64 { + return Ok(self + .as_ref() + .cast(DType::Primitive(PType::I16, nullability))? + .to_primitive()); + } + + if min >= i32::MIN as i64 && max <= i32::MAX as i64 { + return Ok(self + .as_ref() + .cast(DType::Primitive(PType::I32, nullability))? + .to_primitive()); + } + } else { + // Unsigned + if max <= u8::MAX as i64 { + return Ok(self + .as_ref() + .cast(DType::Primitive(PType::U8, nullability))? + .to_primitive()); + } + + if max <= u16::MAX as i64 { + return Ok(self + .as_ref() + .cast(DType::Primitive(PType::U16, nullability))? + .to_primitive()); + } + + if max <= u32::MAX as i64 { + return Ok(self + .as_ref() + .cast(DType::Primitive(PType::U32, nullability))? + .to_primitive()); + } + } + + Ok(self.to_owned()) + } +} +impl> PrimitiveArrayExt for T {} + // TODO(connor): There are a lot of places where we could be using `new_unchecked` in the codebase. impl PrimitiveData { /// Build the slots vector for this array. @@ -103,15 +237,11 @@ impl PrimitiveData { pub unsafe fn new_unchecked_from_handle( handle: BufferHandle, ptype: PType, - validity: Validity, + _validity: Validity, ) -> Self { - let len = handle.len() / ptype.byte_width(); - let slots = Self::make_slots(&validity, len); Self { - slots, - buffer: handle, ptype, - nullability: validity.nullability(), + buffer: handle, } } @@ -155,17 +285,13 @@ impl PrimitiveData { /// /// - If `validity` is [`Validity::Array`], its length must exactly equal `buffer.len()`. #[inline] - pub unsafe fn new_unchecked(buffer: Buffer, validity: Validity) -> Self { + pub unsafe fn new_unchecked(buffer: Buffer, _validity: Validity) -> Self { #[cfg(debug_assertions)] - Self::validate(&buffer, &validity) + Self::validate(&buffer, &_validity) .vortex_expect("[Debug Assertion]: Invalid `PrimitiveArray` parameters"); - let len = buffer.len(); - let slots = Self::make_slots(&validity, len); Self { - slots, ptype: T::PTYPE, - nullability: validity.nullability(), buffer: BufferHandle::new_host(buffer.into_byte_buffer()), } } @@ -198,7 +324,12 @@ impl Array { let dtype = DType::Primitive(T::PTYPE, nullability); let len = 0; let data = PrimitiveData::empty::(nullability); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } + let slots = PrimitiveData::make_slots(&Validity::from(nullability), len); + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Primitive, dtype, len, data).with_slots(slots), + ) + } } /// Creates a new `PrimitiveArray`. @@ -210,16 +341,26 @@ impl Array { let buffer = buffer.into(); let dtype = DType::Primitive(T::PTYPE, validity.nullability()); let len = buffer.len(); + let slots = PrimitiveData::make_slots(&validity, len); let data = PrimitiveData::new(buffer, validity); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Primitive, dtype, len, data).with_slots(slots), + ) + } } /// Constructs a new `PrimitiveArray`. pub fn try_new(buffer: Buffer, validity: Validity) -> VortexResult { let dtype = DType::Primitive(T::PTYPE, validity.nullability()); let len = buffer.len(); + let slots = PrimitiveData::make_slots(&validity, len); let data = PrimitiveData::try_new(buffer, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) }) + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Primitive, dtype, len, data).with_slots(slots), + ) + }) } /// Creates a new `PrimitiveArray` without validation. @@ -230,8 +371,13 @@ impl Array { pub unsafe fn new_unchecked(buffer: Buffer, validity: Validity) -> Self { let dtype = DType::Primitive(T::PTYPE, validity.nullability()); let len = buffer.len(); + let slots = PrimitiveData::make_slots(&validity, len); let data = unsafe { PrimitiveData::new_unchecked(buffer, validity) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Primitive, dtype, len, data).with_slots(slots), + ) + } } /// Create a new array from a buffer handle. @@ -246,16 +392,22 @@ impl Array { ) -> Self { let dtype = DType::Primitive(ptype, validity.nullability()); let len = handle.len() / ptype.byte_width(); + let slots = PrimitiveData::make_slots(&validity, len); let data = unsafe { PrimitiveData::new_unchecked_from_handle(handle, ptype, validity) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Primitive, dtype, len, data).with_slots(slots), + ) + } } /// Creates a new `PrimitiveArray` from a [`BufferHandle`]. pub fn from_buffer_handle(handle: BufferHandle, ptype: PType, validity: Validity) -> Self { let dtype = DType::Primitive(ptype, validity.nullability()); let len = handle.len() / ptype.byte_width(); + let slots = PrimitiveData::make_slots(&validity, len); let data = PrimitiveData::from_buffer_handle(handle, ptype, validity); - Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data).with_slots(slots)) .vortex_expect("PrimitiveData is always valid") } @@ -263,8 +415,13 @@ impl Array { pub fn from_byte_buffer(buffer: ByteBuffer, ptype: PType, validity: Validity) -> Self { let dtype = DType::Primitive(ptype, validity.nullability()); let len = buffer.len() / ptype.byte_width(); + let slots = PrimitiveData::make_slots(&validity, len); let data = PrimitiveData::from_byte_buffer(buffer, ptype, validity); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Primitive, dtype, len, data).with_slots(slots), + ) + } } /// Create a PrimitiveArray from a byte buffer containing only the valid elements. @@ -276,55 +433,66 @@ impl Array { ) -> Self { let dtype = DType::Primitive(ptype, validity.nullability()); let len = n_rows; + let slots = PrimitiveData::make_slots(&validity, len); let data = PrimitiveData::from_values_byte_buffer(valid_elems_buffer, ptype, validity, n_rows); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Primitive, dtype, len, data).with_slots(slots), + ) + } } /// Validates the components that would be used to create a `PrimitiveArray`. pub fn validate(buffer: &Buffer, validity: &Validity) -> VortexResult<()> { PrimitiveData::validate(buffer, validity) } -} -impl PrimitiveData { - /// Consume the primitive array and returns its component parts. - pub fn into_parts(self) -> PrimitiveDataParts { - let ptype = self.ptype(); - let validity = self.validity(); + pub fn into_data_parts(self) -> PrimitiveDataParts { + let validity = PrimitiveArrayExt::validity(&self); + let ptype = PrimitiveArrayExt::ptype(&self); + let data = self.into_data(); PrimitiveDataParts { ptype, - buffer: self.buffer, + buffer: data.buffer, validity, } } -} -impl PrimitiveData { - /// Returns the dtype of the array. - pub fn dtype(&self) -> DType { - DType::Primitive(self.ptype, self.nullability) + pub fn map_each_with_validity(self, f: F) -> VortexResult + where + T: NativePType, + R: NativePType, + F: FnMut((T, bool)) -> R, + { + let validity = PrimitiveArrayExt::validity(&self); + let data = self.into_data(); + let buf_iter = data.to_buffer::().into_iter(); + + let buffer = match &validity { + Validity::NonNullable | Validity::AllValid => { + BufferMut::::from_iter(buf_iter.zip(iter::repeat(true)).map(f)) + } + Validity::AllInvalid => { + BufferMut::::from_iter(buf_iter.zip(iter::repeat(false)).map(f)) + } + Validity::Array(val) => { + let val = val.to_bool().into_bit_buffer(); + BufferMut::::from_iter(buf_iter.zip(val.iter()).map(f)) + } + }; + Ok(PrimitiveArray::new(buffer.freeze(), validity)) } +} - /// Returns the length of the array. +impl PrimitiveData { pub fn len(&self) -> usize { - self.buffer.len() / self.ptype().byte_width() + self.buffer.len() / self.ptype.byte_width() } /// Returns `true` if the array is empty. pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Reconstructs the validity from the slot state. - #[allow(clippy::same_name_method)] - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) - } - - /// Returns the validity as a [`Mask`](vortex_mask::Mask). - pub fn validity_mask(&self) -> vortex_mask::Mask { - self.validity().to_mask(self.len()) + self.buffer.is_empty() } pub fn ptype(&self) -> PType { @@ -336,14 +504,10 @@ impl PrimitiveData { &self.buffer } - pub fn from_buffer_handle(handle: BufferHandle, ptype: PType, validity: Validity) -> Self { - let len = handle.len() / ptype.byte_width(); - let slots = Self::make_slots(&validity, len); + pub fn from_buffer_handle(handle: BufferHandle, ptype: PType, _validity: Validity) -> Self { Self { - slots, - buffer: handle, ptype, - nullability: validity.nullability(), + buffer: handle, } } @@ -394,55 +558,6 @@ impl PrimitiveData { Buffer::from_byte_buffer(self.buffer_handle().to_host_sync()) } - /// Map each element in the array to a new value. - /// - /// This ignores validity and maps over all maybe-null elements. - /// - /// TODO(ngates): we could be smarter here if validity is sparse and only run the function - /// over the valid elements. - pub fn map_each(self, f: F) -> Self - where - T: NativePType, - R: NativePType, - F: FnMut(T) -> R, - { - let validity = self.validity(); - let buffer = match self.try_into_buffer_mut() { - Ok(buffer_mut) => buffer_mut.map_each_in_place(f), - Err(buffer) => BufferMut::from_iter(buffer.iter().copied().map(f)), - }; - PrimitiveData::new(buffer.freeze(), validity) - } - - /// Map each element in the array to a new value. - /// - /// This doesn't ignore validity and maps over all maybe-null elements, with a bool true if - /// valid and false otherwise. - pub fn map_each_with_validity(self, f: F) -> VortexResult - where - T: NativePType, - R: NativePType, - F: FnMut((T, bool)) -> R, - { - let validity = self.validity(); - - let buf_iter = self.to_buffer::().into_iter(); - - let buffer = match &validity { - Validity::NonNullable | Validity::AllValid => { - BufferMut::::from_iter(buf_iter.zip(iter::repeat(true)).map(f)) - } - Validity::AllInvalid => { - BufferMut::::from_iter(buf_iter.zip(iter::repeat(false)).map(f)) - } - Validity::Array(val) => { - let val = val.to_bool().into_bit_buffer(); - BufferMut::::from_iter(buf_iter.zip(val.iter()).map(f)) - } - }; - Ok(PrimitiveData::new(buffer.freeze(), validity)) - } - /// Consume the array and get a host Buffer containing the data values. pub fn into_buffer(self) -> Buffer { if T::PTYPE != self.ptype() { diff --git a/vortex-array/src/arrays/primitive/compute/cast.rs b/vortex-array/src/arrays/primitive/compute/cast.rs index f2467f6f1ab..9053afaf66f 100644 --- a/vortex-array/src/arrays/primitive/compute/cast.rs +++ b/vortex-array/src/arrays/primitive/compute/cast.rs @@ -16,6 +16,7 @@ use crate::aggregate_fn; use crate::array::ArrayView; use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; +use crate::arrays::primitive::PrimitiveArrayExt; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::Nullability; diff --git a/vortex-array/src/arrays/primitive/compute/take/portable.rs b/vortex-array/src/arrays/primitive/compute/take/portable.rs index 29c36ec3cc4..2a9fec9cfb8 100644 --- a/vortex-array/src/arrays/primitive/compute/take/portable.rs +++ b/vortex-array/src/arrays/primitive/compute/take/portable.rs @@ -25,6 +25,7 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::PrimitiveArray; +use crate::arrays::primitive::PrimitiveArrayExt; use crate::arrays::primitive::compute::take::TakeImpl; use crate::arrays::primitive::vtable::Primitive; use crate::dtype::NativePType; diff --git a/vortex-array/src/arrays/primitive/mod.rs b/vortex-array/src/arrays/primitive/mod.rs index 7c526c2cfbb..4d62f1da517 100644 --- a/vortex-array/src/arrays/primitive/mod.rs +++ b/vortex-array/src/arrays/primitive/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::PrimitiveArrayExt; pub use array::PrimitiveData; pub use array::PrimitiveDataParts; pub use array::chunk_range; diff --git a/vortex-array/src/arrays/primitive/vtable/mod.rs b/vortex-array/src/arrays/primitive/vtable/mod.rs index ab29d6305f5..180d01e8c35 100644 --- a/vortex-array/src/arrays/primitive/vtable/mod.rs +++ b/vortex-array/src/arrays/primitive/vtable/mod.rs @@ -31,7 +31,6 @@ use vortex_session::VortexSession; use crate::Precision; use crate::array::ArrayId; -use crate::arrays::primitive::array::NUM_SLOTS; use crate::arrays::primitive::array::SLOT_NAMES; use crate::arrays::primitive::compute::rules::RULES; use crate::hash::ArrayEq; @@ -39,6 +38,18 @@ use crate::hash::ArrayHash; vtable!(Primitive, Primitive, PrimitiveData); +impl ArrayHash for PrimitiveData { + fn array_hash(&self, state: &mut H, precision: Precision) { + self.buffer.array_hash(state, precision); + } +} + +impl ArrayEq for PrimitiveData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + self.buffer.array_eq(&other.buffer, precision) + } +} + impl VTable for Primitive { type ArrayData = PrimitiveData; @@ -49,16 +60,6 @@ impl VTable for Primitive { Self::ID } - fn array_hash(array: &PrimitiveData, state: &mut H, precision: Precision) { - array.buffer.array_hash(state, precision); - array.validity().array_hash(state, precision); - } - - fn array_eq(array: &PrimitiveData, other: &PrimitiveData, precision: Precision) -> bool { - array.buffer.array_eq(&other.buffer, precision) - && array.validity().array_eq(&other.validity(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 1 } @@ -81,21 +82,31 @@ impl VTable for Primitive { Ok(Some(vec![])) } - fn validate(&self, data: &PrimitiveData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &PrimitiveData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let DType::Primitive(_, nullability) = dtype else { + vortex_bail!("Expected primitive dtype, got {dtype:?}"); + }; vortex_ensure!( data.len() == len, "PrimitiveArray length {} does not match outer length {}", data.len(), len ); - - let actual_dtype = data.dtype(); - vortex_ensure!( - &actual_dtype == dtype, - "PrimitiveArray dtype {} does not match outer dtype {}", - actual_dtype, - dtype - ); + let validity = crate::array::child_to_validity(&slots[0], *nullability); + if let Some(validity_len) = validity.maybe_len() { + vortex_ensure!( + validity_len == len, + "PrimitiveArray validity len {} does not match outer length {}", + validity_len, + len + ); + } Ok(()) } @@ -109,7 +120,7 @@ impl VTable for Primitive { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { if !metadata.is_empty() { vortex_bail!( "PrimitiveArray expects empty metadata, got {} bytes", @@ -155,33 +166,15 @@ impl VTable for Primitive { ); // SAFETY: checked ahead of time - unsafe { - Ok(PrimitiveData::new_unchecked_from_handle( - buffer, ptype, validity, - )) - } - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let slots = PrimitiveData::make_slots(&validity, len); + let data = unsafe { PrimitiveData::new_unchecked_from_handle(buffer, ptype, validity) }; + Ok(crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "PrimitiveArray expects {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - - array.slots = slots; - Ok(()) - } - fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done(array)) } diff --git a/vortex-array/src/arrays/primitive/vtable/validity.rs b/vortex-array/src/arrays/primitive/vtable/validity.rs index 87659128064..c6891ba0f4b 100644 --- a/vortex-array/src/arrays/primitive/vtable/validity.rs +++ b/vortex-array/src/arrays/primitive/vtable/validity.rs @@ -5,11 +5,12 @@ use vortex_error::VortexResult; use crate::array::ArrayView; use crate::array::ValidityVTable; +use crate::arrays::primitive::PrimitiveArrayExt; use crate::arrays::primitive::vtable::Primitive; use crate::validity::Validity; impl ValidityVTable for Primitive { fn validity(array: ArrayView<'_, Primitive>) -> VortexResult { - Ok(array.data().validity()) + Ok(PrimitiveArrayExt::validity(&array)) } } diff --git a/vortex-array/src/arrays/scalar_fn/array.rs b/vortex-array/src/arrays/scalar_fn/array.rs index 1fc9accc886..4326ca3c11d 100644 --- a/vortex-array/src/arrays/scalar_fn/array.rs +++ b/vortex-array/src/arrays/scalar_fn/array.rs @@ -8,6 +8,7 @@ use vortex_error::vortex_ensure; use crate::ArrayRef; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::arrays::ScalarFnVTable; use crate::scalar_fn::ScalarFnRef; @@ -16,12 +17,11 @@ use crate::scalar_fn::ScalarFnRef; #[derive(Clone, Debug)] pub struct ScalarFnData { pub(super) scalar_fn: ScalarFnRef, - pub(super) slots: Vec>, } impl ScalarFnData { /// Create a new ScalarFnArray from a scalar function and its children. - pub fn try_new( + pub fn build( scalar_fn: ScalarFnRef, children: Vec, len: usize, @@ -30,10 +30,7 @@ impl ScalarFnData { children.iter().all(|c| c.len() == len), "ScalarFnArray must have children equal to the array length" ); - - let slots = children.into_iter().map(Some).collect(); - - Ok(Self { scalar_fn, slots }) + Ok(Self { scalar_fn }) } /// Get the scalar function bound to this array. @@ -42,46 +39,44 @@ impl ScalarFnData { pub fn scalar_fn(&self) -> &ScalarFnRef { &self.scalar_fn } +} + +pub trait ScalarFnArrayExt: TypedArrayRef { + fn scalar_fn(&self) -> &ScalarFnRef { + &self.scalar_fn + } - /// Get a child array by index. - pub fn get_child(&self, idx: usize) -> &ArrayRef { - self.slots[idx] + fn child_at(&self, idx: usize) -> &ArrayRef { + self.as_ref().slots()[idx] .as_ref() .vortex_expect("ScalarFnArray child slot") } - /// Get the number of children. - pub fn nchildren(&self) -> usize { - self.slots.len() + fn child_count(&self) -> usize { + self.as_ref().slots().len() } - /// Iterate over the children arrays without allocation. - pub fn iter_children(&self) -> impl Iterator + '_ { - self.slots - .iter() - .map(|s| s.as_ref().vortex_expect("ScalarFnArray child slot")) + #[allow(clippy::same_name_method)] + fn nchildren(&self) -> usize { + self.child_count() } - /// Get the children arrays of this scalar function array. - pub fn children(&self) -> Vec { - self.iter_children().cloned().collect() + #[allow(clippy::same_name_method)] + fn get_child(&self, idx: usize) -> &ArrayRef { + self.child_at(idx) } -} -impl Array { - /// Get the scalar function bound to this array. - #[allow(clippy::same_name_method)] - #[inline(always)] - pub fn scalar_fn(&self) -> &ScalarFnRef { - self.data().scalar_fn() + fn iter_children(&self) -> impl Iterator + '_ { + (0..self.child_count()).map(|idx| self.child_at(idx)) } - /// Get the children arrays of this scalar function array. - #[allow(clippy::same_name_method)] - pub fn children(&self) -> Vec { - self.data().children() + fn children(&self) -> Vec { + self.iter_children().cloned().collect() } +} +impl> ScalarFnArrayExt for T {} +impl Array { /// Create a new ScalarFnArray from a scalar function and its children. pub fn try_new( scalar_fn: ScalarFnRef, @@ -90,8 +85,13 @@ impl Array { ) -> VortexResult { let arg_dtypes: Vec<_> = children.iter().map(|c| c.dtype().clone()).collect(); let dtype = scalar_fn.return_dtype(&arg_dtypes)?; - let data = ScalarFnData::try_new(scalar_fn.clone(), children, len)?; + let data = ScalarFnData::build(scalar_fn.clone(), children.clone(), len)?; let vtable = ScalarFnVTable { scalar_fn }; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(vtable, dtype, len, data)) }) + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new(vtable, dtype, len, data) + .with_slots(children.into_iter().map(Some).collect()), + ) + }) } } diff --git a/vortex-array/src/arrays/scalar_fn/mod.rs b/vortex-array/src/arrays/scalar_fn/mod.rs index 778075b9fe0..9d2e9e66ed8 100644 --- a/vortex-array/src/arrays/scalar_fn/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/mod.rs @@ -5,5 +5,7 @@ mod array; mod rules; mod vtable; +pub use array::ScalarFnArrayExt; pub use array::ScalarFnData; +pub use vtable::ScalarFnFactoryExt; pub use vtable::*; diff --git a/vortex-array/src/arrays/scalar_fn/rules.rs b/vortex-array/src/arrays/scalar_fn/rules.rs index 2c1aa67cdaa..d0cb0efdbf4 100644 --- a/vortex-array/src/arrays/scalar_fn/rules.rs +++ b/vortex-array/src/arrays/scalar_fn/rules.rs @@ -19,6 +19,7 @@ use crate::arrays::ScalarFnArray; use crate::arrays::ScalarFnVTable; use crate::arrays::Slice; use crate::arrays::StructArray; +use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::dtype::DType; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ArrayReduceRule; diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index c74a53d7730..656d90e8b37 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -26,6 +26,7 @@ use crate::array::ArrayId; use crate::array::ArrayParts; use crate::array::ArrayView; use crate::array::VTable; +use crate::arrays::scalar_fn::array::ScalarFnArrayExt; use crate::arrays::scalar_fn::array::ScalarFnData; use crate::arrays::scalar_fn::rules::PARENT_RULES; use crate::arrays::scalar_fn::rules::RULES; @@ -53,6 +54,18 @@ pub struct ScalarFnVTable { pub(super) scalar_fn: ScalarFnRef, } +impl ArrayHash for ScalarFnData { + fn array_hash(&self, state: &mut H, _precision: Precision) { + self.scalar_fn().hash(state); + } +} + +impl ArrayEq for ScalarFnData { + fn array_eq(&self, other: &Self, _precision: Precision) -> bool { + self.scalar_fn() == other.scalar_fn() + } +} + impl VTable for ScalarFnVTable { type ArrayData = ScalarFnData; type OperationsVTable = Self; @@ -62,18 +75,25 @@ impl VTable for ScalarFnVTable { self.scalar_fn.id() } - fn validate(&self, data: &ScalarFnData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &ScalarFnData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { vortex_ensure!( data.scalar_fn == self.scalar_fn, "ScalarFnArray data scalar_fn does not match vtable" ); vortex_ensure!( - data.iter_children().all(|c| c.len() == len), + slots.iter().flatten().all(|c| c.len() == len), "All child arrays must have the same length as the scalar function array" ); - let child_dtypes = data - .iter_children() + let child_dtypes = slots + .iter() + .flatten() .map(|c| c.dtype().clone()) .collect_vec(); vortex_ensure!( @@ -83,25 +103,6 @@ impl VTable for ScalarFnVTable { Ok(()) } - fn array_hash(array: &ScalarFnData, state: &mut H, precision: Precision) { - array.scalar_fn().hash(state); - for child in array.iter_children() { - child.array_hash(state, precision); - } - } - - fn array_eq(array: &ScalarFnData, other: &ScalarFnData, precision: Precision) -> bool { - if array.scalar_fn() != other.scalar_fn() { - return false; - } - for (child, other_child) in array.iter_children().zip(other.iter_children()) { - if !child.array_eq(other_child, precision) { - return false; - } - } - true - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -128,14 +129,10 @@ impl VTable for ScalarFnVTable { _buffers: &[BufferHandle], _children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { vortex_bail!("Deserialization of ScalarFnVTable metadata is not supported"); } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(array: ArrayView<'_, Self>, idx: usize) -> String { array .scalar_fn() @@ -145,11 +142,6 @@ impl VTable for ScalarFnVTable { .to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - array.slots = slots; - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { ctx.log(format_args!("scalar_fn({}): executing", array.scalar_fn())); let args = VecExecutionArgs::new(array.children(), array.len()); @@ -173,7 +165,7 @@ impl VTable for ScalarFnVTable { } /// Array factory functions for scalar functions. -pub trait ScalarFnArrayExt: scalar_fn::ScalarFnVTable { +pub trait ScalarFnFactoryExt: scalar_fn::ScalarFnVTable { fn try_new_array( &self, len: usize, @@ -193,16 +185,18 @@ pub trait ScalarFnArrayExt: scalar_fn::ScalarFnVTable { let data = ScalarFnData { scalar_fn: scalar_fn.clone(), - slots: children.into_iter().map(Some).collect(), }; let vtable = ScalarFnVTable { scalar_fn }; - Ok( - unsafe { Array::from_parts_unchecked(ArrayParts::new(vtable, dtype, len, data)) } - .into_array(), - ) + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new(vtable, dtype, len, data) + .with_slots(children.into_iter().map(Some).collect()), + ) + } + .into_array()) } } -impl ScalarFnArrayExt for V {} +impl ScalarFnFactoryExt for V {} /// A matcher that matches any scalar function expression. #[derive(Debug)] diff --git a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs index c697d784ea8..bcb0368772a 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs @@ -10,6 +10,7 @@ use crate::VortexSessionExecute; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::ConstantArray; +use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::vtable::ScalarFnVTable; use crate::columnar::Columnar; use crate::scalar::Scalar; diff --git a/vortex-array/src/arrays/scalar_fn/vtable/validity.rs b/vortex-array/src/arrays/scalar_fn/vtable/validity.rs index 0bd27c9b4f1..2884530f8ba 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/validity.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/validity.rs @@ -9,6 +9,7 @@ use crate::LEGACY_SESSION; use crate::VortexSessionExecute; use crate::array::ArrayView; use crate::array::ValidityVTable; +use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::vtable::ArrayExpr; use crate::arrays::scalar_fn::vtable::FakeEq; use crate::arrays::scalar_fn::vtable::ScalarFnVTable; diff --git a/vortex-array/src/arrays/shared/array.rs b/vortex-array/src/arrays/shared/array.rs index cf4df660520..3a212cf74df 100644 --- a/vortex-array/src/arrays/shared/array.rs +++ b/vortex-array/src/arrays/shared/array.rs @@ -15,8 +15,8 @@ use crate::Canonical; use crate::IntoArray; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::arrays::Shared; -use crate::dtype::DType; /// The source array that is shared and lazily computed. pub(super) const SOURCE_SLOT: usize = 0; @@ -29,42 +29,26 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["source"]; /// After materialization (via `get_or_compute`), operations delegate to the cached result. #[derive(Debug, Clone)] pub struct SharedData { - pub(super) slots: Vec>, cached: Arc>>, async_compute_lock: Arc>, } -impl SharedData { - pub fn new(source: ArrayRef) -> Self { - Self { - slots: vec![Some(source)], - cached: Arc::new(OnceLock::new()), - async_compute_lock: Arc::new(AsyncMutex::new(())), - } - } - - /// Returns the source array reference. - pub(super) fn source(&self) -> &ArrayRef { - self.slots[SOURCE_SLOT] +#[allow(async_fn_in_trait)] +pub trait SharedArrayExt: TypedArrayRef { + fn source(&self) -> &ArrayRef { + self.as_ref().slots()[SOURCE_SLOT] .as_ref() - .vortex_expect("SharedArray source slot") + .vortex_expect("validated shared source slot") } - /// Returns the current array reference. - /// - /// After materialization, returns the cached result. Otherwise, returns the source. - /// If materialization failed, falls back to the source. - pub(super) fn current_array_ref(&self) -> &ArrayRef { + fn current_array_ref(&self) -> &ArrayRef { match self.cached.get() { Some(Ok(arr)) => arr, _ => self.source(), } } - /// Compute and cache the result. The computation runs exactly once via `OnceLock`. - /// - /// If the computation fails, the error is cached and returned on all subsequent calls. - pub fn get_or_compute( + fn get_or_compute( &self, f: impl FnOnce(&ArrayRef) -> VortexResult, ) -> VortexResult { @@ -74,21 +58,17 @@ impl SharedData { result.clone().map_err(Into::into) } - /// Async version of `get_or_compute`. - pub async fn get_or_compute_async(&self, f: F) -> VortexResult + async fn get_or_compute_async(&self, f: F) -> VortexResult where F: FnOnce(ArrayRef) -> Fut, Fut: Future>, { - // Fast path: already computed. if let Some(result) = self.cached.get() { return result.clone().map_err(Into::into); } - // Serialize async computation to prevent redundant work. let _guard = self.async_compute_lock.lock().await; - // Double-check after acquiring the lock. if let Some(result) = self.cached.get() { return result.clone().map_err(Into::into); } @@ -101,20 +81,21 @@ impl SharedData { let result = self.cached.get_or_init(|| computed); result.clone().map_err(Into::into) } +} +impl> SharedArrayExt for T {} - /// Returns the length of this array. - pub fn len(&self) -> usize { - self.current_array_ref().len() - } - - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - self.current_array_ref().dtype() +impl SharedData { + pub fn new() -> Self { + Self { + cached: Arc::new(OnceLock::new()), + async_compute_lock: Arc::new(AsyncMutex::new(())), + } } +} - /// Returns `true` if this array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 +impl Default for SharedData { + fn default() -> Self { + Self::new() } } @@ -124,20 +105,10 @@ impl Array { let dtype = source.dtype().clone(); let len = source.len(); unsafe { - Array::from_parts_unchecked(ArrayParts::new( - Shared, - dtype, - len, - SharedData::new(source), - )) + Array::from_parts_unchecked( + ArrayParts::new(Shared, dtype, len, SharedData::new()) + .with_slots(vec![Some(source)]), + ) } } } - -impl SharedData { - pub(super) fn set_source(&mut self, source: Option) { - self.slots = vec![source]; - self.cached = Arc::new(OnceLock::new()); - self.async_compute_lock = Arc::new(AsyncMutex::new(())); - } -} diff --git a/vortex-array/src/arrays/shared/mod.rs b/vortex-array/src/arrays/shared/mod.rs index 34108bcd619..5d947388a5b 100644 --- a/vortex-array/src/arrays/shared/mod.rs +++ b/vortex-array/src/arrays/shared/mod.rs @@ -4,6 +4,7 @@ mod array; mod vtable; +pub use array::SharedArrayExt; pub use array::SharedData; pub use vtable::Shared; pub use vtable::SharedArray; diff --git a/vortex-array/src/arrays/shared/tests.rs b/vortex-array/src/arrays/shared/tests.rs index b828aa8191a..359a0919965 100644 --- a/vortex-array/src/arrays/shared/tests.rs +++ b/vortex-array/src/arrays/shared/tests.rs @@ -10,6 +10,7 @@ use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::PrimitiveArray; use crate::arrays::SharedArray; +use crate::arrays::shared::SharedArrayExt; use crate::hash::ArrayEq; use crate::hash::Precision as HashPrecision; use crate::session::ArraySession; diff --git a/vortex-array/src/arrays/shared/vtable.rs b/vortex-array/src/arrays/shared/vtable.rs index abd4723ca5d..a368d74226b 100644 --- a/vortex-array/src/arrays/shared/vtable.rs +++ b/vortex-array/src/arrays/shared/vtable.rs @@ -1,11 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::hash::Hasher; + use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::ArrayEq; +use crate::ArrayHash; use crate::ArrayRef; use crate::Canonical; use crate::ExecutionCtx; @@ -17,12 +21,11 @@ use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::array::VTable; use crate::array::ValidityVTable; +use crate::arrays::shared::SharedArrayExt; use crate::arrays::shared::SharedData; use crate::arrays::shared::array::SLOT_NAMES; use crate::buffer::BufferHandle; use crate::dtype::DType; -use crate::hash::ArrayEq; -use crate::hash::ArrayHash; use crate::scalar::Scalar; use crate::validity::Validity; use crate::vtable; @@ -38,6 +41,16 @@ impl Shared { pub const ID: ArrayId = ArrayId::new_ref("vortex.shared"); } +impl ArrayHash for SharedData { + fn array_hash(&self, _state: &mut H, _precision: Precision) {} +} + +impl ArrayEq for SharedData { + fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { + true + } +} + impl VTable for Shared { type ArrayData = SharedData; type OperationsVTable = Self; @@ -47,23 +60,21 @@ impl VTable for Shared { Self::ID } - fn validate(&self, data: &SharedData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_error::vortex_ensure!(data.source().dtype() == dtype, "SharedArray dtype mismatch"); - vortex_error::vortex_ensure!(data.source().len() == len, "SharedArray len mismatch"); + fn validate( + &self, + _data: &SharedData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let source = slots[0] + .as_ref() + .vortex_expect("SharedArray source slot must be present"); + vortex_error::vortex_ensure!(source.dtype() == dtype, "SharedArray dtype mismatch"); + vortex_error::vortex_ensure!(source.len() == len, "SharedArray len mismatch"); Ok(()) } - fn array_hash(array: &SharedData, state: &mut H, precision: Precision) { - let current = array.current_array_ref(); - current.array_hash(state, precision); - } - - fn array_eq(array: &SharedData, other: &SharedData, precision: Precision) -> bool { - let current = array.current_array_ref(); - let other_current = other.current_array_ref(); - current.array_eq(other_current, precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -76,28 +87,10 @@ impl VTable for Shared { None } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_error::vortex_ensure!( - slots.len() == 1, - "SharedArray expects exactly 1 slot, got {}", - slots.len() - ); - let slot = slots - .into_iter() - .next() - .vortex_expect("slots length already validated"); - array.set_source(slot); - Ok(()) - } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { vortex_error::vortex_bail!("Shared array is not serializable") } @@ -111,7 +104,7 @@ impl VTable for Shared { _buffers: &[BufferHandle], _children: &dyn crate::serde::ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { vortex_error::vortex_bail!("Shared array is not serializable") } diff --git a/vortex-array/src/arrays/slice/array.rs b/vortex-array/src/arrays/slice/array.rs index 2d438c3fe9c..eac2cffd2ca 100644 --- a/vortex-array/src/arrays/slice/array.rs +++ b/vortex-array/src/arrays/slice/array.rs @@ -10,8 +10,8 @@ use vortex_error::vortex_panic; use crate::ArrayRef; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::arrays::Slice; -use crate::dtype::DType; /// The underlying child array being sliced. pub(super) const CHILD_SLOT: usize = 0; @@ -20,32 +20,36 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; #[derive(Clone, Debug)] pub struct SliceData { - pub(super) slots: Vec>, pub(super) range: Range, } pub struct SliceDataParts { - pub child: ArrayRef, pub range: Range, } +pub trait SliceArrayExt: TypedArrayRef { + fn child(&self) -> &ArrayRef { + self.as_ref().slots()[CHILD_SLOT] + .as_ref() + .vortex_expect("validated slice child slot") + } +} +impl> SliceArrayExt for T {} + impl SliceData { - fn try_new(child: ArrayRef, range: Range) -> VortexResult { - if range.end > child.len() { + fn try_new(child_len: usize, range: Range) -> VortexResult { + if range.end > child_len { vortex_panic!( "SliceArray range out of bounds: range {:?} exceeds child array length {}", range, - child.len() + child_len ); } - Ok(Self { - slots: vec![Some(child)], - range, - }) + Ok(Self { range }) } - pub fn new(child: ArrayRef, range: Range) -> Self { - Self::try_new(child, range).vortex_expect("failed") + pub fn new(range: Range) -> Self { + Self { range } } /// Returns the length of this array. @@ -53,11 +57,6 @@ impl SliceData { self.range.len() } - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - self.child().dtype() - } - /// Returns `true` if this array is empty. pub fn is_empty(&self) -> bool { self.len() == 0 @@ -68,20 +67,8 @@ impl SliceData { &self.range } - /// The child array being sliced. - pub fn child(&self) -> &ArrayRef { - self.slots[CHILD_SLOT] - .as_ref() - .vortex_expect("SliceArray child slot") - } - - pub fn into_parts(mut self) -> SliceDataParts { - SliceDataParts { - child: self.slots[CHILD_SLOT] - .take() - .vortex_expect("SliceArray child slot"), - range: self.range, - } + pub fn into_parts(self) -> SliceDataParts { + SliceDataParts { range: self.range } } } @@ -90,15 +77,23 @@ impl Array { pub fn try_new(child: ArrayRef, range: Range) -> VortexResult { let len = range.len(); let dtype = child.dtype().clone(); - let data = SliceData::try_new(child, range)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Slice, dtype, len, data)) }) + let data = SliceData::try_new(child.len(), range)?; + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Slice, dtype, len, data).with_slots(vec![Some(child)]), + ) + }) } /// Constructs a new `SliceArray`. pub fn new(child: ArrayRef, range: Range) -> Self { let len = range.len(); let dtype = child.dtype().clone(); - let data = SliceData::new(child, range); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Slice, dtype, len, data)) } + let data = SliceData::new(range); + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Slice, dtype, len, data).with_slots(vec![Some(child)]), + ) + } } } diff --git a/vortex-array/src/arrays/slice/mod.rs b/vortex-array/src/arrays/slice/mod.rs index f701a274a14..8eec9d85310 100644 --- a/vortex-array/src/arrays/slice/mod.rs +++ b/vortex-array/src/arrays/slice/mod.rs @@ -15,6 +15,7 @@ mod vtable; use std::ops::Range; +pub use array::SliceArrayExt; pub use array::SliceData; pub use array::SliceDataParts; use vortex_error::VortexResult; diff --git a/vortex-array/src/arrays/slice/slice_.rs b/vortex-array/src/arrays/slice/slice_.rs index 23011952ff0..0f5fba2d80b 100644 --- a/vortex-array/src/arrays/slice/slice_.rs +++ b/vortex-array/src/arrays/slice/slice_.rs @@ -10,6 +10,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Slice; use crate::arrays::SliceArray; +use crate::arrays::slice::SliceArrayExt; use crate::arrays::slice::SliceReduce; impl SliceReduce for Slice { diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index 562fcd4c117..b5552cdec3c 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -7,6 +7,7 @@ use std::hash::Hash; use std::hash::Hasher; use std::ops::Range; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -26,7 +27,8 @@ use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::array::VTable; use crate::array::ValidityVTable; -use crate::arrays::slice::array::NUM_SLOTS; +use crate::arrays::slice::SliceArrayExt; +use crate::arrays::slice::array::CHILD_SLOT; use crate::arrays::slice::array::SLOT_NAMES; use crate::arrays::slice::array::SliceData; use crate::arrays::slice::rules::PARENT_RULES; @@ -48,6 +50,19 @@ impl Slice { pub const ID: ArrayId = ArrayId::new_ref("vortex.slice"); } +impl ArrayHash for SliceData { + fn array_hash(&self, state: &mut H, _precision: Precision) { + self.range.start.hash(state); + self.range.end.hash(state); + } +} + +impl ArrayEq for SliceData { + fn array_eq(&self, other: &Self, _precision: Precision) -> bool { + self.range == other.range + } +} + impl VTable for Slice { type ArrayData = SliceData; type OperationsVTable = Self; @@ -57,11 +72,24 @@ impl VTable for Slice { Slice::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { vortex_ensure!( - data.child().dtype() == dtype, + slots[CHILD_SLOT].is_some(), + "SliceArray child slot must be present" + ); + let child = slots[CHILD_SLOT] + .as_ref() + .vortex_expect("validated child slot"); + vortex_ensure!( + child.dtype() == dtype, "SliceArray dtype {} does not match outer dtype {}", - data.child().dtype(), + child.dtype(), dtype ); vortex_ensure!( @@ -71,24 +99,14 @@ impl VTable for Slice { len ); vortex_ensure!( - data.range.end <= data.child().len(), + data.range.end <= child.len(), "SliceArray range {:?} exceeds child length {}", data.range, - data.child().len() + child.len() ); Ok(()) } - fn array_hash(array: &SliceData, state: &mut H, precision: Precision) { - array.child().array_hash(state, precision); - array.range.start.hash(state); - array.range.end.hash(state); - } - - fn array_eq(array: &SliceData, other: &SliceData, precision: Precision) -> bool { - array.child().array_eq(other.child(), precision) && array.range == other.range - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -101,10 +119,6 @@ impl VTable for Slice { None } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots - } - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } @@ -123,21 +137,10 @@ impl VTable for Slice { _buffers: &[BufferHandle], _children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { vortex_bail!("Slice array is not serializable") } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "SliceArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { // Execute the child to get canonical form, then slice it let Some(canonical) = array.child().as_opt::() else { diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index 86ea3aef1f4..d69e7ae748d 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -14,6 +14,7 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Struct; @@ -83,6 +84,7 @@ pub(super) const FIELDS_OFFSET: usize = 1; /// /// ``` /// use vortex_array::arrays::StructArray; +/// use vortex_array::arrays::struct_::StructArrayExt; /// use vortex_array::validity::Validity; /// use vortex_array::dtype::FieldNames; /// use vortex_array::IntoArray; @@ -123,6 +125,7 @@ pub(super) const FIELDS_OFFSET: usize = 1; /// /// ``` /// use vortex_array::arrays::{StructArray, PrimitiveArray}; +/// use vortex_array::arrays::struct_::StructArrayExt; /// use vortex_array::validity::Validity; /// use vortex_array::dtype::FieldNames; /// use vortex_array::IntoArray; @@ -150,9 +153,7 @@ pub(super) const FIELDS_OFFSET: usize = 1; #[derive(Clone, Debug)] pub struct StructData { pub(super) names: FieldNames, - pub(super) nullability: crate::dtype::Nullability, pub(super) fieldless_len: Option, - pub(super) slots: Vec>, } pub struct StructDataParts { @@ -162,85 +163,23 @@ pub struct StructDataParts { } impl StructData { - /// Returns the length of this array. - pub fn len(&self) -> usize { - self.slots[FIELDS_OFFSET..] - .first() - .and_then(|slot| slot.as_ref()) - .map_or_else(|| self.fieldless_len.unwrap_or(0), |field| field.len()) - } - - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - DType::Struct(self.struct_fields(), self.nullability) - } - - /// Returns `true` if this array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Reconstructs the validity from the slots. - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) - } - - /// Return an iterator over the struct fields without the validity of the struct applied. - pub fn iter_unmasked_fields(&self) -> impl Iterator + '_ { - self.slots[FIELDS_OFFSET..] - .iter() - .map(|s| s.as_ref().vortex_expect("StructArray field slot")) - } - - /// Return the struct fields without the validity of the struct applied. - pub fn unmasked_fields(&self) -> Arc<[ArrayRef]> { - self.iter_unmasked_fields().cloned().collect() - } - - /// Return the struct field at the given index without the validity of the struct applied. - pub fn unmasked_field(&self, idx: usize) -> &ArrayRef { - self.slots[FIELDS_OFFSET + idx] - .as_ref() - .vortex_expect("StructArray field slot") - } - - /// Return the struct field without the validity of the struct applied - pub fn unmasked_field_by_name(&self, name: impl AsRef) -> VortexResult<&ArrayRef> { - let name = name.as_ref(); - self.unmasked_field_by_name_opt(name).ok_or_else(|| { - vortex_err!( - "Field {name} not found in struct array with names {:?}", - self.names() - ) - }) - } - - /// Return the struct field without the validity of the struct applied - pub fn unmasked_field_by_name_opt(&self, name: impl AsRef) -> Option<&ArrayRef> { - let name = name.as_ref(); - self.names.find(name).map(|idx| { - self.slots[FIELDS_OFFSET + idx] - .as_ref() - .vortex_expect("StructArray field slot") - }) - } - pub fn names(&self) -> &FieldNames { &self.names } - pub fn struct_fields(&self) -> StructFields { - StructFields::new( - self.names.clone(), - self.iter_unmasked_fields() - .map(|field| field.dtype().clone()) - .collect(), - ) + pub(crate) fn make_slots( + fields: &[ArrayRef], + validity: &Validity, + length: usize, + ) -> Vec> { + once(validity_to_child(validity, length)) + .chain(fields.iter().cloned().map(Some)) + .collect() } /// Create a new `StructArray` with the given length, but without any fields. pub fn new_fieldless_with_len(len: usize) -> Self { - Self::try_new( + Self::try_build( FieldNames::default(), Vec::new(), len, @@ -255,13 +194,13 @@ impl StructData { /// /// Panics if the provided components do not satisfy the invariants documented /// in `StructArray::new_unchecked`. - pub fn new( + pub fn build( names: FieldNames, fields: impl Into>, length: usize, validity: Validity, ) -> Self { - Self::try_new(names, fields, length, validity) + Self::try_build(names, fields, length, validity) .vortex_expect("StructArray construction failed") } @@ -273,7 +212,7 @@ impl StructData { /// /// Returns an error if the provided components do not satisfy the invariants documented in /// `StructArray::new_unchecked`. - pub(crate) fn try_new( + pub(crate) fn try_build( names: FieldNames, fields: impl Into>, length: usize, @@ -286,7 +225,7 @@ impl StructData { Self::validate(&fields, &dtype, length, &validity)?; // SAFETY: validate ensures all invariants are met. - Ok(unsafe { Self::new_unchecked(fields, dtype, length, validity) }) + Ok(unsafe { Self::new_unchecked(dtype, fields.is_empty().then_some(length)) }) } /// Creates a new `StructArray` without validation from these components: @@ -314,28 +253,10 @@ impl StructData { /// ## Validity Requirements /// /// - If `validity` is [`Validity::Array`], its length must exactly equal `length`. - pub unsafe fn new_unchecked( - fields: impl Into>, - dtype: StructFields, - length: usize, - validity: Validity, - ) -> Self { - let fields = fields.into(); - - #[cfg(debug_assertions)] - Self::validate(&fields, &dtype, length, &validity) - .vortex_expect("[Debug Assertion]: Invalid `StructArray` parameters"); - - let validity_slot = validity_to_child(&validity, length); - let slots = once(validity_slot) - .chain(fields.iter().map(|f| Some(f.clone()))) - .collect(); - + pub unsafe fn new_unchecked(dtype: StructFields, fieldless_len: Option) -> Self { Self { names: dtype.names().clone(), - nullability: validity.nullability(), - fieldless_len: fields.is_empty().then_some(length), - slots, + fieldless_len, } } @@ -402,116 +323,67 @@ impl StructData { Self::validate(&fields, &dtype, length, &validity)?; // SAFETY: validate ensures all invariants are met. - Ok(unsafe { Self::new_unchecked(fields, dtype, length, validity) }) + Ok(unsafe { Self::new_unchecked(dtype, fields.is_empty().then_some(length)) }) } +} - pub fn into_parts(self) -> StructDataParts { - let StructData { - names, - slots, - nullability, - .. - } = self; - let validity = child_to_validity(&slots[VALIDITY_SLOT], nullability); - let struct_fields = StructFields::new( - names, - slots - .iter() - .skip(FIELDS_OFFSET) - .map(|slot| { - slot.as_ref() - .vortex_expect("StructArray field slot") - .dtype() - .clone() - }) - .collect(), - ); - let fields: Arc<[ArrayRef]> = slots - .into_iter() - .skip(FIELDS_OFFSET) - .map(|s| s.vortex_expect("StructArray field slot")) - .collect(); - StructDataParts { - struct_fields, - fields, - validity, +pub trait StructArrayExt: TypedArrayRef { + fn nullability(&self) -> crate::dtype::Nullability { + match self.as_ref().dtype() { + DType::Struct(_, nullability) => *nullability, + _ => unreachable!("StructArrayExt requires a struct dtype"), } } - pub fn into_fields(self) -> Vec { - self.into_parts().fields.to_vec() + fn names(&self) -> &FieldNames { + StructData::names(self) } - pub fn from_fields>(items: &[(N, ArrayRef)]) -> VortexResult { - Self::try_from_iter(items.iter().map(|(a, b)| (a, b.clone()))) + fn struct_validity(&self) -> Validity { + child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - pub fn try_from_iter_with_validity< - N: AsRef, - A: IntoArray, - T: IntoIterator, - >( - iter: T, - validity: Validity, - ) -> VortexResult { - let (names, fields): (Vec, Vec) = iter - .into_iter() - .map(|(name, fields)| (FieldName::from(name.as_ref()), fields.into_array())) - .unzip(); - let len = fields - .first() - .map(|f| f.len()) - .ok_or_else(|| vortex_err!("StructArray cannot be constructed from an empty slice of arrays because the length is unspecified"))?; - - Self::try_new(FieldNames::from_iter(names), fields, len, validity) + fn iter_unmasked_fields(&self) -> impl Iterator + '_ { + self.as_ref().slots()[FIELDS_OFFSET..] + .iter() + .map(|s| s.as_ref().vortex_expect("StructArray field slot")) } - pub fn try_from_iter, A: IntoArray, T: IntoIterator>( - iter: T, - ) -> VortexResult { - Self::try_from_iter_with_validity(iter, Validity::NonNullable) + fn unmasked_fields(&self) -> Arc<[ArrayRef]> { + self.iter_unmasked_fields().cloned().collect() } - /// Removes and returns a column from the struct array by name. - /// If the column does not exist, returns `None`. - pub fn remove_column(&mut self, name: impl Into) -> Option { - let name = name.into(); - - let struct_dtype = self.struct_fields(); - let len = self.len(); - - let position = struct_dtype - .names() - .iter() - .position(|field_name| field_name.as_ref() == name.as_ref())?; - - let slot_position = FIELDS_OFFSET + position; - let field = self.slots[slot_position] + fn unmasked_field(&self, idx: usize) -> &ArrayRef { + self.as_ref().slots()[FIELDS_OFFSET + idx] .as_ref() .vortex_expect("StructArray field slot") - .clone(); - let new_slots: Vec> = self - .slots - .iter() - .enumerate() - .filter(|(i, _)| *i != slot_position) - .map(|(_, s)| s.clone()) - .collect(); + } - if let Ok(new_dtype) = struct_dtype.without_field(position) { - self.slots = new_slots; - self.names = new_dtype.names().clone(); - self.fieldless_len = self - .slots - .get(FIELDS_OFFSET) - .and_then(|slot| slot.as_ref()) - .map(|_| None) - .unwrap_or(Some(len)); - return Some(field); - } - None + fn unmasked_field_by_name_opt(&self, name: impl AsRef) -> Option<&ArrayRef> { + let name = name.as_ref(); + self.names().find(name).map(|idx| self.unmasked_field(idx)) + } + + fn unmasked_field_by_name(&self, name: impl AsRef) -> VortexResult<&ArrayRef> { + let name = name.as_ref(); + self.unmasked_field_by_name_opt(name).ok_or_else(|| { + vortex_err!( + "Field {name} not found in struct array with names {:?}", + self.names() + ) + }) + } + + fn struct_fields(&self) -> StructFields { + StructFields::new( + self.names().clone(), + self.iter_unmasked_fields() + .map(|field| field.dtype().clone()) + .collect(), + ) } } +impl> StructArrayExt for T {} impl Array { /// Creates a new `StructArray`. @@ -521,10 +393,19 @@ impl Array { length: usize, validity: Validity, ) -> Self { - let data = StructData::new(names, fields, length, validity); - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) } + let fields = fields.into(); + let field_dtypes: Vec<_> = fields.iter().map(|d| d.dtype().clone()).collect(); + let dtype = DType::Struct( + StructFields::new(names.clone(), field_dtypes), + validity.nullability(), + ); + let slots = StructData::make_slots(&fields, &validity, length); + let data = StructData::build(names, fields, length, validity); + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Struct, dtype, length, data).with_slots(slots), + ) + } } /// Constructs a new `StructArray`. @@ -534,10 +415,19 @@ impl Array { length: usize, validity: Validity, ) -> VortexResult { - let data = StructData::try_new(names, fields, length, validity)?; - let dtype = data.dtype(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) + let fields = fields.into(); + let field_dtypes: Vec<_> = fields.iter().map(|d| d.dtype().clone()).collect(); + let dtype = DType::Struct( + StructFields::new(names.clone(), field_dtypes), + validity.nullability(), + ); + let slots = StructData::make_slots(&fields, &validity, length); + let data = StructData::try_build(names, fields, length, validity)?; + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Struct, dtype, length, data).with_slots(slots), + ) + }) } /// Creates a new `StructArray` without validation. @@ -551,10 +441,15 @@ impl Array { length: usize, validity: Validity, ) -> Self { - let data = unsafe { StructData::new_unchecked(fields, dtype, length, validity) }; - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) } + let fields = fields.into(); + let outer_dtype = DType::Struct(dtype.clone(), validity.nullability()); + let slots = StructData::make_slots(&fields, &validity, length); + let data = unsafe { StructData::new_unchecked(dtype, fields.is_empty().then_some(length)) }; + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Struct, outer_dtype, length, data).with_slots(slots), + ) + } } /// Constructs a new `StructArray` with an explicit dtype. @@ -564,18 +459,20 @@ impl Array { length: usize, validity: Validity, ) -> VortexResult { + let fields = fields.into(); + let outer_dtype = DType::Struct(dtype.clone(), validity.nullability()); + let slots = StructData::make_slots(&fields, &validity, length); let data = StructData::try_new_with_dtype(fields, dtype, length, validity)?; - let dtype = data.dtype(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) + Ok(unsafe { + Array::from_parts_unchecked( + ArrayParts::new(Struct, outer_dtype, length, data).with_slots(slots), + ) + }) } /// Construct a `StructArray` from named fields. pub fn from_fields>(items: &[(N, ArrayRef)]) -> VortexResult { - let data = StructData::from_fields(items)?; - let dtype = data.dtype(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) + Self::try_from_iter(items.iter().map(|(a, b)| (a, b.clone()))) } /// Create a `StructArray` from an iterator of (name, array) pairs with validity. @@ -587,20 +484,37 @@ impl Array { iter: T, validity: Validity, ) -> VortexResult { - let data = StructData::try_from_iter_with_validity(iter, validity)?; - let dtype = data.dtype(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) + let (names, fields): (Vec, Vec) = iter + .into_iter() + .map(|(name, fields)| (FieldName::from(name.as_ref()), fields.into_array())) + .unzip(); + let len = fields + .first() + .map(|f| f.len()) + .ok_or_else(|| vortex_err!("StructArray cannot be constructed from an empty slice of arrays because the length is unspecified"))?; + + Self::try_new(FieldNames::from_iter(names), fields, len, validity) } /// Create a `StructArray` from an iterator of (name, array) pairs. pub fn try_from_iter, A: IntoArray, T: IntoIterator>( iter: T, ) -> VortexResult { - let data = StructData::try_from_iter(iter)?; - let dtype = data.dtype(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) + let (names, fields): (Vec, Vec) = iter + .into_iter() + .map(|(name, field)| (FieldName::from(name.as_ref()), field.into_array())) + .unzip(); + let len = fields + .first() + .map(ArrayRef::len) + .ok_or_else(|| vortex_err!("StructArray cannot be constructed from an empty slice of arrays because the length is unspecified"))?; + + Self::try_new( + FieldNames::from_iter(names), + fields, + len, + Validity::NonNullable, + ) } // TODO(aduffy): Add equivalent function to support field masks for nested column access. @@ -636,13 +550,82 @@ impl Array { /// Create a fieldless `StructArray` with the given length. pub fn new_fieldless_with_len(len: usize) -> Self { let data = StructData::new_fieldless_with_len(len); - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) } + let dtype = DType::Struct( + StructFields::new(FieldNames::default(), Vec::new()), + crate::dtype::Nullability::NonNullable, + ); + let slots = StructData::make_slots(&[], &Validity::NonNullable, len); + unsafe { + Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data).with_slots(slots)) + } + } + + pub fn into_data_parts(self) -> StructDataParts { + let fields: Arc<[ArrayRef]> = self.slots()[FIELDS_OFFSET..] + .iter() + .map(|s| s.as_ref().vortex_expect("StructArray field slot").clone()) + .collect(); + let names = self.data().names.clone(); + let validity = self.validity().vortex_expect("StructArray validity"); + let struct_fields = StructFields::new( + names, + fields.iter().map(|field| field.dtype().clone()).collect(), + ); + StructDataParts { + struct_fields, + fields, + validity, + } + } + + pub fn remove_column(&self, name: impl Into) -> Option<(Self, ArrayRef)> { + let name = name.into(); + let struct_dtype = self.struct_fields(); + let len = self.len(); + + let position = struct_dtype + .names() + .iter() + .position(|field_name| field_name.as_ref() == name.as_ref())?; + + let slot_position = FIELDS_OFFSET + position; + let field = self.slots()[slot_position] + .as_ref() + .vortex_expect("StructArray field slot") + .clone(); + let new_slots: Vec> = self + .slots() + .iter() + .enumerate() + .filter(|(i, _)| *i != slot_position) + .map(|(_, s)| s.clone()) + .collect(); + + let new_dtype = struct_dtype.without_field(position).ok()?; + let data = StructData { + names: new_dtype.names().clone(), + fieldless_len: new_slots + .get(FIELDS_OFFSET) + .and_then(|slot| slot.as_ref()) + .map(|_| None) + .unwrap_or(Some(len)), + }; + let new_array = unsafe { + Array::from_parts_unchecked( + ArrayParts::new( + Struct, + DType::Struct(new_dtype, self.dtype().nullability()), + len, + data, + ) + .with_slots(new_slots), + ) + }; + Some((new_array, field)) } } -impl StructData { +impl Array { pub fn with_column(&self, name: impl Into, array: ArrayRef) -> VortexResult { let name = name.into(); let struct_dtype = self.struct_fields(); @@ -651,12 +634,16 @@ impl StructData { let types = struct_dtype.fields().chain(once(array.dtype().clone())); let new_fields = StructFields::new(names.collect(), types.collect()); - let children: Arc<[ArrayRef]> = self.slots[FIELDS_OFFSET..] + let children: Arc<[ArrayRef]> = self.slots()[FIELDS_OFFSET..] .iter() .map(|s| s.as_ref().vortex_expect("StructArray field slot").clone()) .chain(once(array)) .collect(); - Self::try_new_with_dtype(children, new_fields, self.len(), self.validity()) + Self::try_new_with_dtype(children, new_fields, self.len(), self.validity()?) + } + + pub fn remove_column_owned(&self, name: impl Into) -> Option<(Self, ArrayRef)> { + self.remove_column(name) } } diff --git a/vortex-array/src/arrays/struct_/compute/cast.rs b/vortex-array/src/arrays/struct_/compute/cast.rs index 5b5bd8abfe0..5e63cdc2092 100644 --- a/vortex-array/src/arrays/struct_/compute/cast.rs +++ b/vortex-array/src/arrays/struct_/compute/cast.rs @@ -12,6 +12,7 @@ use crate::array::ArrayView; use crate::arrays::ConstantArray; use crate::arrays::Struct; use crate::arrays::StructArray; +use crate::arrays::struct_::StructArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::scalar::Scalar; @@ -94,6 +95,7 @@ mod tests { use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; use crate::arrays::VarBinArray; + use crate::arrays::struct_::StructArrayExt; use crate::builtins::ArrayBuiltins; use crate::compute::conformance::cast::test_cast_conformance; use crate::dtype::DType; diff --git a/vortex-array/src/arrays/struct_/compute/mask.rs b/vortex-array/src/arrays/struct_/compute/mask.rs index a6572d29937..180b08a540f 100644 --- a/vortex-array/src/arrays/struct_/compute/mask.rs +++ b/vortex-array/src/arrays/struct_/compute/mask.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Struct; use crate::arrays::StructArray; +use crate::arrays::struct_::StructArrayExt; use crate::scalar_fn::fns::mask::MaskReduce; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/struct_/compute/rules.rs b/vortex-array/src/arrays/struct_/compute/rules.rs index 1672352afe1..2c09bb3acd4 100644 --- a/vortex-array/src/arrays/struct_/compute/rules.rs +++ b/vortex-array/src/arrays/struct_/compute/rules.rs @@ -13,9 +13,10 @@ use crate::arrays::Struct; use crate::arrays::StructArray; use crate::arrays::dict::TakeReduceAdaptor; use crate::arrays::scalar_fn::ExactScalarFn; -use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; +use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::arrays::slice::SliceReduceAdaptor; +use crate::arrays::struct_::StructArrayExt; use crate::builtins::ArrayBuiltins; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; @@ -142,6 +143,7 @@ mod tests { use crate::IntoArray; use crate::arrays::StructArray; use crate::arrays::VarBinViewArray; + use crate::arrays::struct_::StructArrayExt; use crate::arrays::struct_::compute::rules::ConstantArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; diff --git a/vortex-array/src/arrays/struct_/compute/slice.rs b/vortex-array/src/arrays/struct_/compute/slice.rs index b5b29a397da..011315f35aa 100644 --- a/vortex-array/src/arrays/struct_/compute/slice.rs +++ b/vortex-array/src/arrays/struct_/compute/slice.rs @@ -12,6 +12,7 @@ use crate::array::ArrayView; use crate::arrays::Struct; use crate::arrays::StructArray; use crate::arrays::slice::SliceReduce; +use crate::arrays::struct_::StructArrayExt; impl SliceReduce for Struct { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { diff --git a/vortex-array/src/arrays/struct_/compute/take.rs b/vortex-array/src/arrays/struct_/compute/take.rs index 817bd4d287f..6ae544ffc96 100644 --- a/vortex-array/src/arrays/struct_/compute/take.rs +++ b/vortex-array/src/arrays/struct_/compute/take.rs @@ -9,6 +9,7 @@ use crate::array::ArrayView; use crate::arrays::Struct; use crate::arrays::StructArray; use crate::arrays::dict::TakeReduce; +use crate::arrays::struct_::StructArrayExt; use crate::builtins::ArrayBuiltins; use crate::scalar::Scalar; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/struct_/compute/zip.rs b/vortex-array/src/arrays/struct_/compute/zip.rs index 6e2615cbef7..cf7b7b91d6d 100644 --- a/vortex-array/src/arrays/struct_/compute/zip.rs +++ b/vortex-array/src/arrays/struct_/compute/zip.rs @@ -13,6 +13,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Struct; use crate::arrays::StructArray; +use crate::arrays::struct_::StructArrayExt; use crate::builtins::ArrayBuiltins; use crate::scalar_fn::fns::zip::ZipKernel; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/struct_/mod.rs b/vortex-array/src/arrays/struct_/mod.rs index 65b53e2d603..3ad34f225a1 100644 --- a/vortex-array/src/arrays/struct_/mod.rs +++ b/vortex-array/src/arrays/struct_/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::StructArrayExt; pub use array::StructData; pub use array::StructDataParts; pub use vtable::StructArray; diff --git a/vortex-array/src/arrays/struct_/tests.rs b/vortex-array/src/arrays/struct_/tests.rs index 6fb3078c993..38b12074900 100644 --- a/vortex-array/src/arrays/struct_/tests.rs +++ b/vortex-array/src/arrays/struct_/tests.rs @@ -10,6 +10,7 @@ use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; use crate::arrays::VarBinArray; +use crate::arrays::struct_::StructArrayExt; use crate::assert_arrays_eq; use crate::dtype::DType; use crate::dtype::FieldName; @@ -68,8 +69,7 @@ fn test_remove_column() { ) .unwrap(); - let mut data = struct_a.into_data(); - let removed = data.remove_column("xs").unwrap(); + let (data, removed) = struct_a.remove_column("xs").unwrap(); assert_eq!( removed.dtype(), &DType::Primitive(PType::I64, Nullability::NonNullable) diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index 3d32534cd3f..629e115f6d0 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::hash::Hasher; + use itertools::Itertools; use kernel::PARENT_KERNELS; use vortex_error::VortexExpect; @@ -9,12 +11,15 @@ use vortex_error::vortex_bail; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::ArrayEq; +use crate::ArrayHash; use crate::ArrayRef; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::array::Array; use crate::array::ArrayView; use crate::array::VTable; +use crate::array::child_to_validity; use crate::arrays::struct_::StructData; use crate::arrays::struct_::array::FIELDS_OFFSET; use crate::arrays::struct_::array::VALIDITY_SLOT; @@ -30,11 +35,19 @@ mod validity; use crate::Precision; use crate::array::ArrayId; -use crate::hash::ArrayEq; -use crate::hash::ArrayHash; vtable!(Struct, Struct, StructData); +impl ArrayHash for StructData { + fn array_hash(&self, _state: &mut H, _precision: Precision) {} +} + +impl ArrayEq for StructData { + fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { + true + } +} + impl VTable for Struct { type ArrayData = StructData; @@ -45,46 +58,86 @@ impl VTable for Struct { Self::ID } - fn array_hash(array: &StructData, state: &mut H, precision: Precision) { - for field in array.iter_unmasked_fields() { - field.array_hash(state, precision); - } - array.validity().array_hash(state, precision); - } - - fn array_eq(array: &StructData, other: &StructData, precision: Precision) -> bool { - array.slots.len() == other.slots.len() - && array - .iter_unmasked_fields() - .zip(other.iter_unmasked_fields()) - .all(|(a, b)| a.array_eq(b, precision)) - && array.validity().array_eq(&other.validity(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } - fn validate(&self, data: &StructData, dtype: &DType, len: usize) -> VortexResult<()> { - match dtype { - DType::Struct(..) => {} - _ => vortex_bail!("Expected struct dtype, found {:?}", dtype), + fn validate( + &self, + data: &StructData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + let DType::Struct(struct_dtype, nullability) = dtype else { + vortex_bail!("Expected struct dtype, found {:?}", dtype) + }; + + if data.names() != struct_dtype.names() { + vortex_bail!( + InvalidArgument: "StructArray field names {:?} do not match dtype names {:?}", + data.names(), + struct_dtype.names() + ); } - if data.len() != len { + + let expected_slots = struct_dtype.nfields() + 1; + if slots.len() != expected_slots { vortex_bail!( - InvalidArgument: "StructArray length {} does not match outer length {}", - data.len(), - len + InvalidArgument: "StructArray has {} slots but expected {}", + slots.len(), + expected_slots ); } - let data_dtype = data.dtype(); - if &data_dtype != dtype { + + let validity = child_to_validity(&slots[VALIDITY_SLOT], *nullability); + if let Some(validity_len) = validity.maybe_len() + && validity_len != len + { vortex_bail!( - InvalidArgument: "StructArray dtype {} does not match outer dtype {}", - data_dtype, - dtype + InvalidArgument: "StructArray validity length {} does not match outer length {}", + validity_len, + len ); } + + let field_slots = &slots[FIELDS_OFFSET..]; + if field_slots.is_empty() { + if data.fieldless_len != Some(len) { + vortex_bail!( + InvalidArgument: "Fieldless StructArray length {:?} does not match outer length {}", + data.fieldless_len, + len + ); + } + return Ok(()); + } + + if data.fieldless_len.is_some() { + vortex_bail!("StructArray cannot have fieldless length and field slots"); + } + + for (idx, (slot, field_dtype)) in field_slots.iter().zip(struct_dtype.fields()).enumerate() + { + let field = slot + .as_ref() + .ok_or_else(|| vortex_error::vortex_err!("StructArray missing field slot {idx}"))?; + if field.len() != len { + vortex_bail!( + InvalidArgument: "StructArray field {idx} has length {} but expected {}", + field.len(), + len + ); + } + if field.dtype() != &field_dtype { + vortex_bail!( + InvalidArgument: "StructArray field {idx} has dtype {} but expected {}", + field.dtype(), + field_dtype + ); + } + } + Ok(()) } @@ -109,7 +162,7 @@ impl VTable for Struct { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { if !metadata.is_empty() { vortex_bail!( "StructArray expects empty metadata, got {} bytes", @@ -143,11 +196,10 @@ impl VTable for Struct { }) .try_collect()?; - StructData::try_new_with_dtype(field_children, struct_dtype.clone(), len, validity) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let slots = StructData::make_slots(&field_children, &validity, len); + let data = + StructData::try_new_with_dtype(field_children, struct_dtype.clone(), len, validity)?; + Ok(crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(array: ArrayView<'_, Self>, idx: usize) -> String { @@ -158,11 +210,6 @@ impl VTable for Struct { } } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - array.slots = slots; - Ok(()) - } - fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done(array)) } diff --git a/vortex-array/src/arrays/struct_/vtable/operations.rs b/vortex-array/src/arrays/struct_/vtable/operations.rs index ba7b3c6e315..021d93f3c2f 100644 --- a/vortex-array/src/arrays/struct_/vtable/operations.rs +++ b/vortex-array/src/arrays/struct_/vtable/operations.rs @@ -7,6 +7,7 @@ use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::Struct; +use crate::arrays::struct_::StructArrayExt; use crate::scalar::Scalar; impl OperationsVTable for Struct { diff --git a/vortex-array/src/arrays/struct_/vtable/validity.rs b/vortex-array/src/arrays/struct_/vtable/validity.rs index 53c31b1994e..1545162bae5 100644 --- a/vortex-array/src/arrays/struct_/vtable/validity.rs +++ b/vortex-array/src/arrays/struct_/vtable/validity.rs @@ -5,11 +5,12 @@ use vortex_error::VortexResult; use crate::array::ArrayView; use crate::array::ValidityVTable; +use crate::arrays::struct_::StructArrayExt; use crate::arrays::struct_::vtable::Struct; use crate::validity::Validity; impl ValidityVTable for Struct { fn validity(array: ArrayView<'_, Struct>) -> VortexResult { - Ok(array.data().validity()) + Ok(array.struct_validity()) } } diff --git a/vortex-array/src/arrays/varbin/accessor.rs b/vortex-array/src/arrays/varbin/accessor.rs index 7421f09fbbe..aa84ac64421 100644 --- a/vortex-array/src/arrays/varbin/accessor.rs +++ b/vortex-array/src/arrays/varbin/accessor.rs @@ -8,6 +8,7 @@ use vortex_error::VortexExpect; use crate::ToCanonical; use crate::accessor::ArrayAccessor; use crate::arrays::VarBinArray; +use crate::arrays::varbin::VarBinArrayExt; use crate::match_each_integer_ptype; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index 329cf3da04c..4f6c93d4ced 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -5,7 +5,6 @@ use num_traits::AsPrimitive; use vortex_buffer::ByteBuffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_mask::Mask; @@ -14,6 +13,7 @@ use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::VarBin; @@ -34,37 +34,25 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["offsets", "validity"]; #[derive(Clone, Debug)] pub struct VarBinData { - pub(super) is_utf8: bool, - pub(super) nullability: Nullability, pub(super) bytes: BufferHandle, - pub(super) slots: Vec>, } -impl VarBinData { - fn dtype_parts(dtype: &DType) -> VortexResult<(bool, Nullability)> { - match dtype { - DType::Utf8(nullability) => Ok((true, *nullability)), - DType::Binary(nullability) => Ok((false, *nullability)), - _ => vortex_bail!(MismatchedTypes: "utf8 or binary", dtype), - } - } - - fn make_dtype(is_utf8: bool, nullability: Nullability) -> DType { - if is_utf8 { - DType::Utf8(nullability) - } else { - DType::Binary(nullability) - } - } +pub struct VarBinDataParts { + pub dtype: DType, + pub bytes: BufferHandle, + pub offsets: ArrayRef, + pub validity: Validity, +} +impl VarBinData { /// Creates a new `VarBinArray`. /// /// # Panics /// /// Panics if the provided components do not satisfy the invariants documented /// in `VarBinArray::new_unchecked`. - pub fn new(offsets: ArrayRef, bytes: ByteBuffer, dtype: DType, validity: Validity) -> Self { - Self::try_new(offsets, bytes, dtype, validity).vortex_expect("VarBinArray new") + pub fn build(offsets: ArrayRef, bytes: ByteBuffer, dtype: DType, validity: Validity) -> Self { + Self::try_build(offsets, bytes, dtype, validity).vortex_expect("VarBinArray new") } /// Creates a new `VarBinArray`. @@ -73,13 +61,21 @@ impl VarBinData { /// /// Panics if the provided components do not satisfy the invariants documented /// in `VarBinArray::new_unchecked`. - pub fn new_from_handle( + pub fn build_from_handle( offset: ArrayRef, bytes: BufferHandle, dtype: DType, validity: Validity, ) -> Self { - Self::try_new_from_handle(offset, bytes, dtype, validity).vortex_expect("VarBinArray new") + Self::try_build_from_handle(offset, bytes, dtype, validity).vortex_expect("VarBinArray new") + } + + pub(crate) fn make_slots( + offsets: ArrayRef, + validity: &Validity, + len: usize, + ) -> Vec> { + vec![Some(offsets), validity_to_child(validity, len)] } /// Constructs a new `VarBinArray`. @@ -90,7 +86,7 @@ impl VarBinData { /// /// Returns an error if the provided components do not satisfy the invariants documented in /// `VarBinArray::new_unchecked`. - pub fn try_new( + pub fn try_build( offsets: ArrayRef, bytes: ByteBuffer, dtype: DType, @@ -100,7 +96,7 @@ impl VarBinData { Self::validate(&offsets, &bytes, &dtype, &validity)?; // SAFETY: validate ensures all invariants are met. - Ok(unsafe { Self::new_unchecked_from_handle(offsets, bytes, dtype, validity) }) + Ok(unsafe { Self::new_unchecked_from_handle(bytes) }) } /// Constructs a new `VarBinArray` from a `BufferHandle` of memory that may exist @@ -112,7 +108,7 @@ impl VarBinData { /// /// Returns an error if the provided components do not satisfy the invariants documented in /// `VarBinArray::new_unchecked`. - pub fn try_new_from_handle( + pub fn try_build_from_handle( offsets: ArrayRef, bytes: BufferHandle, dtype: DType, @@ -121,7 +117,7 @@ impl VarBinData { Self::validate(&offsets, &bytes, &dtype, &validity)?; // SAFETY: validate ensures all invariants are met. - Ok(unsafe { Self::new_unchecked_from_handle(offsets, bytes, dtype, validity) }) + Ok(unsafe { Self::new_unchecked_from_handle(bytes) }) } /// Creates a new `VarBinArray` without validation from these components: @@ -152,17 +148,10 @@ impl VarBinData { /// ## Validity Requirements /// /// - If `validity` is [`Validity::Array`], its length must exactly equal `offsets.len() - 1`. - pub unsafe fn new_unchecked( - offsets: ArrayRef, - bytes: ByteBuffer, - dtype: DType, - validity: Validity, - ) -> Self { + pub unsafe fn new_unchecked(bytes: ByteBuffer) -> Self { // SAFETY: `new_unchecked_from_handle` has same invariants which should be checked // by caller. - unsafe { - Self::new_unchecked_from_handle(offsets, BufferHandle::new_host(bytes), dtype, validity) - } + unsafe { Self::new_unchecked_from_handle(BufferHandle::new_host(bytes)) } } /// Creates a new `VarBinArray` without validation from its components, with string data @@ -171,27 +160,8 @@ impl VarBinData { /// # Safety /// /// The caller must ensure all the invariants documented in `new_unchecked` are satisfied. - pub unsafe fn new_unchecked_from_handle( - offsets: ArrayRef, - bytes: BufferHandle, - dtype: DType, - validity: Validity, - ) -> Self { - #[cfg(debug_assertions)] - Self::validate(&offsets, &bytes, &dtype, &validity) - .vortex_expect("[Debug Assertion]: Invalid `VarBinArray` parameters"); - - let len = offsets.len().saturating_sub(1); - let validity_slot = validity_to_child(&validity, len); - let (is_utf8, nullability) = - Self::dtype_parts(&dtype).vortex_expect("VarBinArray dtype must be utf8 or binary"); - - Self { - is_utf8, - nullability, - bytes, - slots: vec![Some(offsets), validity_slot], - } + pub unsafe fn new_unchecked_from_handle(bytes: BufferHandle) -> Self { + Self { bytes } } /// Validates the components that would be used to create a `VarBinArray`. @@ -288,45 +258,12 @@ impl VarBinData { Ok(()) } - /// Returns the length of this array. - pub fn len(&self) -> usize { - self.offsets().len().saturating_sub(1) - } - - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - Self::make_dtype(self.is_utf8, self.nullability) - } - - /// Returns `true` if this array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns the [`Validity`] of this array. - #[allow(clippy::same_name_method)] - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) - } - - /// Returns the validity as a [`Mask`]. - pub fn validity_mask(&self) -> Mask { - self.validity().to_mask(self.len()) - } - - #[inline] - pub fn offsets(&self) -> &ArrayRef { - self.slots[OFFSETS_SLOT] - .as_ref() - .vortex_expect("VarBinArray offsets slot") - } - /// Access the value bytes child buffer /// /// # Note /// /// Bytes child buffer is never sliced when the array is sliced so this can include values - /// that are not logically present in the array. Users should prefer [sliced_bytes][Self::sliced_bytes] + /// that are not logically present in the array. Users should prefer `sliced_bytes` /// unless they're resolving values via the offset child array. #[inline] pub fn bytes(&self) -> &ByteBuffer { @@ -338,16 +275,74 @@ impl VarBinData { pub fn bytes_handle(&self) -> &BufferHandle { &self.bytes } +} - /// Access value bytes child array limited to values that are logically present in - /// the array unlike [bytes][Self::bytes]. - pub fn sliced_bytes(&self) -> ByteBuffer { - let first_offset: usize = self.offset_at(0); - let last_offset = self.offset_at(self.len()); +pub trait VarBinArrayExt: TypedArrayRef { + fn offsets(&self) -> &ArrayRef { + self.as_ref().slots()[OFFSETS_SLOT] + .as_ref() + .vortex_expect("VarBinArray offsets slot") + } + + fn validity_child(&self) -> Option<&ArrayRef> { + self.as_ref().slots()[VALIDITY_SLOT].as_ref() + } + + fn dtype_parts(&self) -> (bool, Nullability) { + match self.as_ref().dtype() { + DType::Utf8(nullability) => (true, *nullability), + DType::Binary(nullability) => (false, *nullability), + _ => unreachable!("VarBinArrayExt requires a utf8 or binary dtype"), + } + } + + fn is_utf8(&self) -> bool { + self.dtype_parts().0 + } + + fn nullability(&self) -> Nullability { + self.dtype_parts().1 + } + + fn varbin_validity(&self) -> Validity { + child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) + } + + fn varbin_validity_mask(&self) -> Mask { + self.varbin_validity().to_mask(self.as_ref().len()) + } + + fn offset_at(&self, index: usize) -> usize { + assert!( + index <= self.as_ref().len(), + "Index {index} out of bounds 0..={}", + self.as_ref().len() + ); + (&self + .offsets() + .scalar_at(index) + .vortex_expect("offsets must support scalar_at")) + .try_into() + .vortex_expect("Failed to convert offset to usize") + } + + fn bytes_at(&self, index: usize) -> ByteBuffer { + let start = self.offset_at(index); + let end = self.offset_at(index + 1); + self.bytes().slice(start..end) + } + + fn sliced_bytes(&self) -> ByteBuffer { + let first_offset: usize = self.offset_at(0); + let last_offset = self.offset_at(self.as_ref().len()); self.bytes().slice(first_offset..last_offset) } +} +impl> VarBinArrayExt for T {} +/// Forwarding constructors for `VarBinArray` (= `Array`). +impl Array { pub fn from_vec>(vec: Vec, dtype: DType) -> Self { let size: usize = vec.iter().map(|v| v.as_ref().len()).sum(); if size < u32::MAX as usize { @@ -357,18 +352,6 @@ impl VarBinData { } } - fn from_vec_sized(vec: Vec, dtype: DType) -> Self - where - O: IntegerPType, - T: AsRef<[u8]>, - { - let mut builder = VarBinBuilder::::with_capacity(vec.len()); - for v in vec { - builder.append_value(v.as_ref()); - } - builder.finish(dtype).into_data() - } - #[expect( clippy::same_name_method, reason = "intentionally named from_iter like Iterator::from_iter" @@ -382,7 +365,7 @@ impl VarBinData { for v in iter { builder.append(v.as_ref().map(|o| o.as_ref())); } - builder.finish(dtype).into_data() + builder.finish(dtype) } pub fn from_iter_nonnull, I: IntoIterator>( @@ -394,39 +377,19 @@ impl VarBinData { for v in iter { builder.append_value(v); } - builder.finish(dtype).into_data() - } -} - -/// Forwarding constructors for `VarBinArray` (= `Array`). -impl Array { - #[inline] - fn from_prevalidated_data(data: VarBinData) -> Self { - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(VarBin, dtype, len, data)) } + builder.finish(dtype) } - pub fn from_vec>(vec: Vec, dtype: DType) -> Self { - Self::from_prevalidated_data(VarBinData::from_vec(vec, dtype)) - } - - #[expect( - clippy::same_name_method, - reason = "intentionally named from_iter like Iterator::from_iter" - )] - pub fn from_iter, I: IntoIterator>>( - iter: I, - dtype: DType, - ) -> Self { - Self::from_prevalidated_data(VarBinData::from_iter(iter, dtype)) - } - - pub fn from_iter_nonnull, I: IntoIterator>( - iter: I, - dtype: DType, - ) -> Self { - Self::from_prevalidated_data(VarBinData::from_iter_nonnull(iter, dtype)) + fn from_vec_sized(vec: Vec, dtype: DType) -> Self + where + O: IntegerPType, + T: AsRef<[u8]>, + { + let mut builder = VarBinBuilder::::with_capacity(vec.len()); + for v in vec { + builder.append_value(v.as_ref()); + } + builder.finish(dtype) } /// Create from a vector of string slices. @@ -448,54 +411,38 @@ impl Array { pub fn from_nullable_bytes(value: Vec>) -> Self { Self::from_iter(value, DType::Binary(Nullability::Nullable)) } -} -impl VarBinData { - /// Get value offset at a given index - /// - /// Note: There's 1 more offsets than the elements in the array, thus last offset is at array length index - /// - /// Panics if index is out of bounds - pub fn offset_at(&self, index: usize) -> usize { - assert!( - index <= self.len(), - "Index {index} out of bounds 0..={}", - self.len() - ); - - (&self - .offsets() - .scalar_at(index) - .vortex_expect("offsets must support scalar_at")) - .try_into() - .vortex_expect("Failed to convert offset to usize") - } - - /// Access value bytes at a given index - /// - /// Will return buffer referencing underlying data without performing a copy - pub fn bytes_at(&self, index: usize) -> ByteBuffer { - let start = self.offset_at(index); - let end = self.offset_at(index + 1); - - self.bytes().slice(start..end) - } - - /// Consumes self, returning a tuple containing the `DType`, the `bytes` array, - /// the `offsets` array, and the `validity`. - pub fn into_parts(mut self) -> (DType, BufferHandle, ArrayRef, Validity) { - let validity = self.validity(); - let offsets = self.slots[OFFSETS_SLOT] - .take() - .vortex_expect("VarBinArray offsets slot"); - (self.dtype(), self.bytes, offsets, validity) + pub fn into_data_parts(self) -> VarBinDataParts { + let dtype = self.dtype().clone(); + let validity = self.varbin_validity(); + let offsets = self.offsets().clone(); + let data = self.into_data(); + VarBinDataParts { + dtype, + bytes: data.bytes, + offsets, + validity, + } } } impl Array { /// Creates a new `VarBinArray`. pub fn new(offsets: ArrayRef, bytes: ByteBuffer, dtype: DType, validity: Validity) -> Self { - Self::from_prevalidated_data(VarBinData::new(offsets, bytes, dtype, validity)) + let len = offsets.len().saturating_sub(1); + let slots = VarBinData::make_slots(offsets, &validity, len); + let data = VarBinData::build( + slots[OFFSETS_SLOT] + .as_ref() + .vortex_expect("VarBinArray offsets slot") + .clone(), + bytes, + dtype.clone(), + validity, + ); + unsafe { + Array::from_parts_unchecked(ArrayParts::new(VarBin, dtype, len, data).with_slots(slots)) + } } /// Creates a new `VarBinArray` without validation. @@ -509,9 +456,12 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Self::from_prevalidated_data(unsafe { - VarBinData::new_unchecked(offsets, bytes, dtype, validity) - }) + let len = offsets.len().saturating_sub(1); + let slots = VarBinData::make_slots(offsets, &validity, len); + let data = unsafe { VarBinData::new_unchecked(bytes) }; + unsafe { + Array::from_parts_unchecked(ArrayParts::new(VarBin, dtype, len, data).with_slots(slots)) + } } /// Creates a new `VarBinArray` without validation from a [`BufferHandle`]. @@ -525,9 +475,12 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Self::from_prevalidated_data(unsafe { - VarBinData::new_unchecked_from_handle(offsets, bytes, dtype, validity) - }) + let len = offsets.len().saturating_sub(1); + let slots = VarBinData::make_slots(offsets, &validity, len); + let data = unsafe { VarBinData::new_unchecked_from_handle(bytes) }; + unsafe { + Array::from_parts_unchecked(ArrayParts::new(VarBin, dtype, len, data).with_slots(slots)) + } } /// Constructs a new `VarBinArray`. @@ -537,154 +490,91 @@ impl Array { dtype: DType, validity: Validity, ) -> VortexResult { - Ok(Self::from_prevalidated_data(VarBinData::try_new( - offsets, bytes, dtype, validity, - )?)) - } -} - -impl From> for VarBinData { - fn from(value: Vec<&[u8]>) -> Self { - Self::from_vec(value, DType::Binary(Nullability::NonNullable)) - } -} - -impl From>> for VarBinData { - fn from(value: Vec>) -> Self { - Self::from_vec(value, DType::Binary(Nullability::NonNullable)) - } -} - -impl From> for VarBinData { - fn from(value: Vec) -> Self { - Self::from_vec(value, DType::Utf8(Nullability::NonNullable)) - } -} - -impl From> for VarBinData { - fn from(value: Vec<&str>) -> Self { - Self::from_vec(value, DType::Utf8(Nullability::NonNullable)) - } -} - -impl From>> for VarBinData { - fn from(value: Vec>) -> Self { - Self::from_iter(value, DType::Binary(Nullability::Nullable)) - } -} - -impl From>>> for VarBinData { - fn from(value: Vec>>) -> Self { - Self::from_iter(value, DType::Binary(Nullability::Nullable)) - } -} - -impl From>> for VarBinData { - fn from(value: Vec>) -> Self { - Self::from_iter(value, DType::Utf8(Nullability::Nullable)) - } -} - -impl From>> for VarBinData { - fn from(value: Vec>) -> Self { - Self::from_iter(value, DType::Utf8(Nullability::Nullable)) - } -} - -impl<'a> FromIterator> for VarBinData { - fn from_iter>>(iter: T) -> Self { - Self::from_iter(iter, DType::Binary(Nullability::Nullable)) - } -} - -impl FromIterator>> for VarBinData { - fn from_iter>>>(iter: T) -> Self { - Self::from_iter(iter, DType::Binary(Nullability::Nullable)) - } -} - -impl FromIterator> for VarBinData { - fn from_iter>>(iter: T) -> Self { - Self::from_iter(iter, DType::Utf8(Nullability::Nullable)) - } -} - -impl<'a> FromIterator> for VarBinData { - fn from_iter>>(iter: T) -> Self { - Self::from_iter(iter, DType::Utf8(Nullability::Nullable)) + let len = offsets.len().saturating_sub(1); + let slots = VarBinData::make_slots(offsets, &validity, len); + let data = VarBinData::try_build( + slots[OFFSETS_SLOT] + .as_ref() + .vortex_expect("VarBinArray offsets slot") + .clone(), + bytes, + dtype.clone(), + validity, + )?; + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(VarBin, dtype, len, data).with_slots(slots)) + }) } } -// --- From and FromIterator forwarding for Array --- - impl From> for Array { fn from(value: Vec<&[u8]>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Self::from_vec(value, DType::Binary(Nullability::NonNullable)) } } impl From>> for Array { fn from(value: Vec>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Self::from_vec(value, DType::Binary(Nullability::NonNullable)) } } impl From> for Array { fn from(value: Vec) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Self::from_vec(value, DType::Utf8(Nullability::NonNullable)) } } impl From> for Array { fn from(value: Vec<&str>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Self::from_vec(value, DType::Utf8(Nullability::NonNullable)) } } impl From>> for Array { fn from(value: Vec>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Self::from_iter(value, DType::Binary(Nullability::Nullable)) } } impl From>>> for Array { fn from(value: Vec>>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Self::from_iter(value, DType::Binary(Nullability::Nullable)) } } impl From>> for Array { fn from(value: Vec>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Self::from_iter(value, DType::Utf8(Nullability::Nullable)) } } impl From>> for Array { fn from(value: Vec>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Self::from_iter(value, DType::Utf8(Nullability::Nullable)) } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Self::from_iter(iter, DType::Binary(Nullability::Nullable)) } } impl FromIterator>> for Array { fn from_iter>>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Self::from_iter(iter, DType::Binary(Nullability::Nullable)) } } impl FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Self::from_iter(iter, DType::Utf8(Nullability::Nullable)) } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Self::from_iter(iter, DType::Utf8(Nullability::Nullable)) } } diff --git a/vortex-array/src/arrays/varbin/builder.rs b/vortex-array/src/arrays/varbin/builder.rs index 8dd5066a889..9b63889e70f 100644 --- a/vortex-array/src/arrays/varbin/builder.rs +++ b/vortex-array/src/arrays/varbin/builder.rs @@ -122,6 +122,7 @@ impl VarBinBuilder { mod tests { use vortex_error::VortexResult; + use crate::arrays::varbin::VarBinArrayExt; use crate::arrays::varbin::builder::VarBinBuilder; use crate::dtype::DType; use crate::dtype::Nullability::Nullable; diff --git a/vortex-array/src/arrays/varbin/compute/cast.rs b/vortex-array/src/arrays/varbin/compute/cast.rs index a474168a0a2..6b02283aec8 100644 --- a/vortex-array/src/arrays/varbin/compute/cast.rs +++ b/vortex-array/src/arrays/varbin/compute/cast.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::VarBin; use crate::arrays::VarBinArray; +use crate::arrays::varbin::VarBinArrayExt; use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; diff --git a/vortex-array/src/arrays/varbin/compute/compare.rs b/vortex-array/src/arrays/varbin/compute/compare.rs index abce39a4045..b4ad470760e 100644 --- a/vortex-array/src/arrays/varbin/compute/compare.rs +++ b/vortex-array/src/arrays/varbin/compute/compare.rs @@ -18,6 +18,7 @@ use crate::arrays::BoolArray; use crate::arrays::PrimitiveArray; use crate::arrays::VarBin; use crate::arrays::VarBinViewArray; +use crate::arrays::varbin::VarBinArrayExt; use crate::arrow::Datum; use crate::arrow::from_arrow_array_with_len; use crate::builtins::ArrayBuiltins; @@ -148,6 +149,7 @@ mod test { use crate::arrays::ConstantArray; use crate::arrays::VarBinArray; use crate::arrays::VarBinViewArray; + use crate::arrays::bool::BoolArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::Nullability; diff --git a/vortex-array/src/arrays/varbin/compute/filter.rs b/vortex-array/src/arrays/varbin/compute/filter.rs index b1117b35bd9..8f4b6793a72 100644 --- a/vortex-array/src/arrays/varbin/compute/filter.rs +++ b/vortex-array/src/arrays/varbin/compute/filter.rs @@ -19,6 +19,7 @@ use crate::arrays::PrimitiveArray; use crate::arrays::VarBin; use crate::arrays::VarBinArray; use crate::arrays::filter::FilterKernel; +use crate::arrays::varbin::VarBinArrayExt; use crate::arrays::varbin::builder::VarBinBuilder; use crate::dtype::DType; use crate::dtype::IntegerPType; @@ -67,7 +68,7 @@ fn filter_select_var_bin_by_slice( offsets.as_slice::(), values.bytes().as_slice(), mask_slices, - values.validity_mask(), + values.varbin_validity_mask(), selection_count, ) }) diff --git a/vortex-array/src/arrays/varbin/compute/mask.rs b/vortex-array/src/arrays/varbin/compute/mask.rs index 8a205acaab5..1c5567adea7 100644 --- a/vortex-array/src/arrays/varbin/compute/mask.rs +++ b/vortex-array/src/arrays/varbin/compute/mask.rs @@ -8,6 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::VarBin; use crate::arrays::VarBinArray; +use crate::arrays::varbin::VarBinArrayExt; use crate::scalar_fn::fns::mask::MaskReduce; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/varbin/compute/slice.rs b/vortex-array/src/arrays/varbin/compute/slice.rs index c356c0338d9..b3ab91cc578 100644 --- a/vortex-array/src/arrays/varbin/compute/slice.rs +++ b/vortex-array/src/arrays/varbin/compute/slice.rs @@ -11,6 +11,7 @@ use crate::array::ArrayView; use crate::arrays::VarBin; use crate::arrays::VarBinArray; use crate::arrays::slice::SliceReduce; +use crate::arrays::varbin::VarBinArrayExt; impl SliceReduce for VarBin { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { diff --git a/vortex-array/src/arrays/varbin/compute/take.rs b/vortex-array/src/arrays/varbin/compute/take.rs index d742b4cda51..de8b1ffff77 100644 --- a/vortex-array/src/arrays/varbin/compute/take.rs +++ b/vortex-array/src/arrays/varbin/compute/take.rs @@ -16,6 +16,7 @@ use crate::arrays::PrimitiveArray; use crate::arrays::VarBin; use crate::arrays::VarBinArray; use crate::arrays::dict::TakeExecute; +use crate::arrays::varbin::VarBinArrayExt; use crate::dtype::DType; use crate::dtype::IntegerPType; use crate::executor::ExecutionCtx; @@ -36,7 +37,7 @@ impl TakeExecute for VarBin { .dtype() .clone() .union_nullability(indices.dtype().nullability()); - let array_validity = array.validity_mask(); + let array_validity = array.varbin_validity_mask(); let indices_validity = indices.validity_mask()?; let array = match_each_integer_ptype!(indices.ptype(), |I| { diff --git a/vortex-array/src/arrays/varbin/mod.rs b/vortex-array/src/arrays/varbin/mod.rs index dc3fa89f746..a8f321fec9a 100644 --- a/vortex-array/src/arrays/varbin/mod.rs +++ b/vortex-array/src/arrays/varbin/mod.rs @@ -2,7 +2,9 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::VarBinArrayExt; pub use array::VarBinData; +pub use array::VarBinDataParts; pub use vtable::VarBinArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/varbin/vtable/canonical.rs b/vortex-array/src/arrays/varbin/vtable/canonical.rs index b7ee968c90c..726bbfd8dcd 100644 --- a/vortex-array/src/arrays/varbin/vtable/canonical.rs +++ b/vortex-array/src/arrays/varbin/vtable/canonical.rs @@ -23,21 +23,23 @@ pub(crate) fn varbin_to_canonical( array: ArrayView<'_, VarBin>, ctx: &mut ExecutionCtx, ) -> VortexResult { - let (dtype, bytes, offsets, validity) = array.into_owned().into_data().into_parts(); + let parts = array.into_owned().into_data_parts(); - let offsets = offsets.execute::(ctx)?; + let offsets = parts.offsets.execute::(ctx)?; match_each_integer_ptype!(offsets.ptype(), |P| { let offsets_slice = offsets.as_slice::

(); let first: usize = offsets_slice[0].as_(); let last: usize = offsets_slice[offsets_slice.len() - 1].as_(); - let bytes = bytes.unwrap_host().slice(first..last).into_mut(); + let bytes = parts.bytes.unwrap_host().slice(first..last).into_mut(); let lens = offsets_to_lengths(offsets_slice); let (buffers, views) = build_views(0, MAX_BUFFER_LEN, bytes, lens.as_slice()); // SAFETY: views are correctly computed from valid offsets - Ok(unsafe { VarBinViewArray::new_unchecked(views, Arc::from(buffers), dtype, validity) }) + Ok(unsafe { + VarBinViewArray::new_unchecked(views, Arc::from(buffers), parts.dtype, parts.validity) + }) }) } diff --git a/vortex-array/src/arrays/varbin/vtable/mod.rs b/vortex-array/src/arrays/varbin/vtable/mod.rs index 0b6813ddc99..b1fd1410b68 100644 --- a/vortex-array/src/arrays/varbin/vtable/mod.rs +++ b/vortex-array/src/arrays/varbin/vtable/mod.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::hash::Hasher; + use prost::Message; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -16,6 +18,7 @@ use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; +use crate::arrays::varbin::VarBinArrayExt; use crate::arrays::varbin::VarBinData; use crate::arrays::varbin::array::NUM_SLOTS; use crate::arrays::varbin::array::SLOT_NAMES; @@ -48,6 +51,18 @@ pub struct VarBinMetadata { pub(crate) offsets_ptype: i32, } +impl ArrayHash for VarBinData { + fn array_hash(&self, state: &mut H, precision: Precision) { + self.bytes().array_hash(state, precision); + } +} + +impl ArrayEq for VarBinData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + self.bytes().array_eq(other.bytes(), precision) + } +} + impl VTable for VarBin { type ArrayData = VarBinData; @@ -58,34 +73,34 @@ impl VTable for VarBin { Self::ID } - fn array_hash(array: &VarBinData, state: &mut H, precision: Precision) { - array.bytes().array_hash(state, precision); - array.offsets().array_hash(state, precision); - array.validity().array_hash(state, precision); - } - - fn array_eq(array: &VarBinData, other: &VarBinData, precision: Precision) -> bool { - array.bytes().array_eq(other.bytes(), precision) - && array.offsets().array_eq(other.offsets(), precision) - && array.validity().array_eq(&other.validity(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 1 } - fn validate(&self, data: &VarBinData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + _data: &VarBinData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { vortex_ensure!( - data.len() == len, + slots.len() == NUM_SLOTS, + "VarBinArray expected {NUM_SLOTS} slots, found {}", + slots.len() + ); + let offsets = slots[crate::arrays::varbin::array::OFFSETS_SLOT] + .as_ref() + .vortex_expect("VarBinArray offsets slot"); + vortex_ensure!( + offsets.len().saturating_sub(1) == len, "VarBinArray length {} does not match outer length {}", - data.len(), + offsets.len().saturating_sub(1), len ); vortex_ensure!( - data.dtype() == *dtype, - "VarBinArray dtype {} does not match outer dtype {}", - data.dtype(), - dtype + matches!(dtype, DType::Binary(_) | DType::Utf8(_)), + "VarBinArray dtype must be binary or utf8, got {dtype}" ); Ok(()) } @@ -123,7 +138,7 @@ impl VTable for VarBin { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { let metadata = VarBinMetadata::decode(metadata)?; let validity = if children.len() == 1 { Validity::from(dtype.nullability()) @@ -145,28 +160,15 @@ impl VTable for VarBin { } let bytes = buffers[0].clone().try_to_host_sync()?; - VarBinData::try_new(offsets, bytes, dtype.clone(), validity) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let data = VarBinData::try_build(offsets.clone(), bytes, dtype.clone(), validity.clone())?; + let slots = VarBinData::make_slots(offsets, &validity, len); + Ok(crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "VarBinArray expects exactly {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn reduce_parent( array: ArrayView<'_, Self>, parent: &ArrayRef, diff --git a/vortex-array/src/arrays/varbin/vtable/operations.rs b/vortex-array/src/arrays/varbin/vtable/operations.rs index c58c301a0ff..e11043e605a 100644 --- a/vortex-array/src/arrays/varbin/vtable/operations.rs +++ b/vortex-array/src/arrays/varbin/vtable/operations.rs @@ -7,6 +7,7 @@ use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::VarBin; +use crate::arrays::varbin::VarBinArrayExt; use crate::arrays::varbin::varbin_scalar; use crate::scalar::Scalar; diff --git a/vortex-array/src/arrays/varbin/vtable/validity.rs b/vortex-array/src/arrays/varbin/vtable/validity.rs index 0102cc3b73e..4af0a9a0b0e 100644 --- a/vortex-array/src/arrays/varbin/vtable/validity.rs +++ b/vortex-array/src/arrays/varbin/vtable/validity.rs @@ -5,11 +5,12 @@ use vortex_error::VortexResult; use crate::array::ArrayView; use crate::array::ValidityVTable; +use crate::arrays::varbin::VarBinArrayExt; use crate::arrays::varbin::vtable::VarBin; use crate::validity::Validity; impl ValidityVTable for VarBin { fn validity(array: ArrayView<'_, VarBin>) -> VortexResult { - Ok(array.data().validity()) + Ok(array.varbin_validity()) } } diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index fd61c204feb..21c63346d10 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -18,6 +18,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::VarBinView; @@ -95,9 +96,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; /// ``` #[derive(Clone, Debug)] pub struct VarBinViewData { - pub(super) slots: Vec>, - pub(super) is_utf8: bool, - pub(super) nullability: Nullability, pub(super) buffers: Arc<[BufferHandle]>, pub(super) views: BufferHandle, } @@ -118,14 +116,6 @@ impl VarBinViewData { } } - fn make_dtype(is_utf8: bool, nullability: Nullability) -> DType { - if is_utf8 { - DType::Utf8(nullability) - } else { - DType::Binary(nullability) - } - } - /// Build the slots vector for this array. pub(super) fn make_slots(validity: &Validity, len: usize) -> Vec> { vec![validity_to_child(validity, len)] @@ -275,19 +265,11 @@ impl VarBinViewData { views: BufferHandle, buffers: Arc<[BufferHandle]>, dtype: DType, - validity: Validity, + _validity: Validity, ) -> Self { - let len = views.len() / size_of::(); - let slots = Self::make_slots(&validity, len); - let (is_utf8, nullability) = + let _ = Self::dtype_parts(&dtype).vortex_expect("VarBinViewArray dtype must be utf8 or binary"); - Self { - slots, - is_utf8, - nullability, - buffers, - views, - } + Self { buffers, views } } /// Validates the components that would be used to create a `VarBinViewArray`. @@ -384,38 +366,11 @@ impl VarBinViewData { self.views.len() / size_of::() } - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - Self::make_dtype(self.is_utf8, self.nullability) - } - /// Returns `true` if this array is empty. pub fn is_empty(&self) -> bool { self.len() == 0 } - /// Returns the [`Validity`] of this array. - #[allow(clippy::same_name_method)] - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) - } - - /// Returns the validity as a [`Mask`]. - pub fn validity_mask(&self) -> Mask { - self.validity().to_mask(self.len()) - } - - /// Splits the array into owned parts - pub fn into_parts(self) -> VarBinViewDataParts { - let validity = self.validity(); - VarBinViewDataParts { - dtype: self.dtype(), - buffers: self.buffers, - views: self.views, - validity, - } - } - /// Access to the primitive views buffer. /// /// Variable-sized binary view buffer contain a "view" child array, with 16-byte entries that @@ -569,12 +524,38 @@ impl VarBinViewData { } } +pub trait VarBinViewArrayExt: TypedArrayRef { + fn dtype_parts(&self) -> (bool, Nullability) { + match self.as_ref().dtype() { + DType::Utf8(nullability) => (true, *nullability), + DType::Binary(nullability) => (false, *nullability), + _ => unreachable!("VarBinViewArrayExt requires a utf8 or binary dtype"), + } + } + + fn varbinview_validity(&self) -> Validity { + child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.dtype_parts().1) + } + + fn varbinview_validity_mask(&self) -> Mask { + self.varbinview_validity().to_mask(self.as_ref().len()) + } +} +impl> VarBinViewArrayExt for T {} + impl Array { #[inline] - fn from_prevalidated_data(data: VarBinViewData) -> Self { - let dtype = data.dtype(); + fn from_prevalidated_data( + dtype: DType, + data: VarBinViewData, + slots: Vec>, + ) -> Self { let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(VarBinView, dtype, len, data)) } + unsafe { + Array::from_parts_unchecked( + ArrayParts::new(VarBinView, dtype, len, data).with_slots(slots), + ) + } } /// Construct a `VarBinViewArray` from an iterator of optional byte slices. @@ -586,27 +567,73 @@ impl Array { iter: I, dtype: DType, ) -> Self { - Self::from_prevalidated_data(VarBinViewData::from_iter(iter, dtype)) + let iter = iter.into_iter(); + let mut builder = VarBinViewBuilder::with_capacity(dtype, iter.size_hint().0); + for value in iter { + match value { + Some(value) => builder.append_value(value), + None => builder.append_null(), + } + } + builder.finish_into_varbinview() } pub fn from_iter_str, I: IntoIterator>(iter: I) -> Self { - Self::from_prevalidated_data(VarBinViewData::from_iter_str(iter)) + let iter = iter.into_iter(); + let mut builder = VarBinViewBuilder::with_capacity( + DType::Utf8(Nullability::NonNullable), + iter.size_hint().0, + ); + for value in iter { + builder.append_value(value.as_ref()); + } + builder.finish_into_varbinview() } pub fn from_iter_nullable_str, I: IntoIterator>>( iter: I, ) -> Self { - Self::from_prevalidated_data(VarBinViewData::from_iter_nullable_str(iter)) + let iter = iter.into_iter(); + let mut builder = VarBinViewBuilder::with_capacity( + DType::Utf8(Nullability::Nullable), + iter.size_hint().0, + ); + for value in iter { + match value { + Some(value) => builder.append_value(value.as_ref()), + None => builder.append_null(), + } + } + builder.finish_into_varbinview() } pub fn from_iter_bin, I: IntoIterator>(iter: I) -> Self { - Self::from_prevalidated_data(VarBinViewData::from_iter_bin(iter)) + let iter = iter.into_iter(); + let mut builder = VarBinViewBuilder::with_capacity( + DType::Binary(Nullability::NonNullable), + iter.size_hint().0, + ); + for value in iter { + builder.append_value(value.as_ref()); + } + builder.finish_into_varbinview() } pub fn from_iter_nullable_bin, I: IntoIterator>>( iter: I, ) -> Self { - Self::from_prevalidated_data(VarBinViewData::from_iter_nullable_bin(iter)) + let iter = iter.into_iter(); + let mut builder = VarBinViewBuilder::with_capacity( + DType::Binary(Nullability::Nullable), + iter.size_hint().0, + ); + for value in iter { + match value { + Some(value) => builder.append_value(value.as_ref()), + None => builder.append_null(), + } + } + builder.finish_into_varbinview() } /// Creates a new `VarBinViewArray`. @@ -616,9 +643,9 @@ impl Array { dtype: DType, validity: Validity, ) -> VortexResult { - Ok(Self::from_prevalidated_data(VarBinViewData::try_new( - views, buffers, dtype, validity, - )?)) + let data = VarBinViewData::try_new(views, buffers, dtype.clone(), validity.clone())?; + let slots = VarBinViewData::make_slots(&validity, data.len()); + Ok(Self::from_prevalidated_data(dtype, data, slots)) } /// Creates a new `VarBinViewArray` without validation. @@ -632,9 +659,11 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Self::from_prevalidated_data(unsafe { - VarBinViewData::new_unchecked(views, buffers, dtype, validity) - }) + let data = unsafe { + VarBinViewData::new_unchecked(views, buffers, dtype.clone(), validity.clone()) + }; + let slots = VarBinViewData::make_slots(&validity, data.len()); + Self::from_prevalidated_data(dtype, data, slots) } /// Creates a new `VarBinViewArray` with device or host memory. @@ -644,7 +673,9 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Self::from_prevalidated_data(VarBinViewData::new_handle(views, buffers, dtype, validity)) + let data = VarBinViewData::new_handle(views, buffers, dtype.clone(), validity.clone()); + let slots = VarBinViewData::make_slots(&validity, data.len()); + Self::from_prevalidated_data(dtype, data, slots) } /// Construct a new array from `BufferHandle`s without validation. @@ -658,9 +689,23 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Self::from_prevalidated_data(unsafe { - VarBinViewData::new_handle_unchecked(views, buffers, dtype, validity) - }) + let data = unsafe { + VarBinViewData::new_handle_unchecked(views, buffers, dtype.clone(), validity.clone()) + }; + let slots = VarBinViewData::make_slots(&validity, data.len()); + Self::from_prevalidated_data(dtype, data, slots) + } + + pub fn into_data_parts(self) -> VarBinViewDataParts { + let dtype = self.dtype().clone(); + let validity = self.varbinview_validity(); + let data = self.into_data(); + VarBinViewDataParts { + dtype, + buffers: data.buffers, + views: data.views, + validity, + } } } @@ -692,24 +737,24 @@ impl<'a> FromIterator> for VarBinViewData { impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Self::from_iter(iter, DType::Binary(Nullability::Nullable)) } } impl FromIterator>> for Array { fn from_iter>>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Self::from_iter(iter, DType::Binary(Nullability::Nullable)) } } impl FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Self::from_iter_nullable_str(iter) } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Self::from_iter_nullable_str(iter) } } diff --git a/vortex-array/src/arrays/varbinview/compute/zip.rs b/vortex-array/src/arrays/varbinview/compute/zip.rs index 559d796088e..dfa2ae8e71c 100644 --- a/vortex-array/src/arrays/varbinview/compute/zip.rs +++ b/vortex-array/src/arrays/varbinview/compute/zip.rs @@ -16,6 +16,7 @@ use crate::array::ArrayView; use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::BinaryView; +use crate::arrays::varbinview::VarBinViewArrayExt; use crate::builders::DeduplicatedBuffers; use crate::builders::LazyBitBufferBuilder; use crate::scalar_fn::fns::zip::ZipKernel; @@ -53,8 +54,8 @@ impl ZipKernel for VarBinView { let mut views_builder = BufferMut::::with_capacity(len); let mut validity_builder = LazyBitBufferBuilder::new(len); - let true_validity = if_true.validity_mask(); - let false_validity = if_false.validity_mask(); + let true_validity = if_true.varbinview_validity_mask(); + let false_validity = if_false.varbinview_validity_mask(); let mask = mask.try_to_mask_fill_null_false(ctx)?; let if_false_view = if_false; diff --git a/vortex-array/src/arrays/varbinview/mod.rs b/vortex-array/src/arrays/varbinview/mod.rs index d2d0041424d..be82b952f30 100644 --- a/vortex-array/src/arrays/varbinview/mod.rs +++ b/vortex-array/src/arrays/varbinview/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::VarBinViewArrayExt; pub use array::VarBinViewData; pub use array::VarBinViewDataParts; pub use vtable::VarBinViewArray; diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index 7c2b674aa9e..102d7365be5 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::hash::Hasher; use std::mem::size_of; use std::sync::Arc; @@ -45,44 +46,53 @@ impl VarBinView { pub const ID: ArrayId = ArrayId::new_ref("vortex.varbinview"); } -impl VTable for VarBinView { - type ArrayData = VarBinViewData; - - type OperationsVTable = Self; - type ValidityVTable = Self; - - fn id(&self) -> ArrayId { - Self::ID - } - - fn array_hash( - array: &VarBinViewData, - state: &mut H, - precision: Precision, - ) { - for buffer in array.buffers.iter() { +impl ArrayHash for VarBinViewData { + fn array_hash(&self, state: &mut H, precision: Precision) { + for buffer in self.buffers.iter() { buffer.array_hash(state, precision); } - array.views.array_hash(state, precision); - array.validity().array_hash(state, precision); + self.views.array_hash(state, precision); } +} - fn array_eq(array: &VarBinViewData, other: &VarBinViewData, precision: Precision) -> bool { - array.buffers.len() == other.buffers.len() - && array +impl ArrayEq for VarBinViewData { + fn array_eq(&self, other: &Self, precision: Precision) -> bool { + self.buffers.len() == other.buffers.len() + && self .buffers .iter() .zip(other.buffers.iter()) .all(|(a, b)| a.array_eq(b, precision)) - && array.views.array_eq(&other.views, precision) - && array.validity().array_eq(&other.validity(), precision) + && self.views.array_eq(&other.views, precision) + } +} + +impl VTable for VarBinView { + type ArrayData = VarBinViewData; + + type OperationsVTable = Self; + type ValidityVTable = Self; + + fn id(&self) -> ArrayId { + Self::ID } fn nbuffers(array: ArrayView<'_, Self>) -> usize { array.data_buffers().len() + 1 } - fn validate(&self, data: &VarBinViewData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &VarBinViewData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + vortex_ensure!( + slots.len() == NUM_SLOTS, + "VarBinViewArray expected {NUM_SLOTS} slots, found {}", + slots.len() + ); vortex_ensure!( data.len() == len, "VarBinViewArray length {} does not match outer length {}", @@ -90,10 +100,8 @@ impl VTable for VarBinView { len ); vortex_ensure!( - data.dtype() == *dtype, - "VarBinViewArray dtype {} does not match outer dtype {}", - data.dtype(), - dtype + matches!(dtype, DType::Binary(_) | DType::Utf8(_)), + "VarBinViewArray dtype must be binary or utf8, got {dtype}" ); Ok(()) } @@ -133,7 +141,7 @@ impl VTable for VarBinView { buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { if !metadata.is_empty() { vortex_bail!( "VarBinViewArray expects empty metadata, got {} bytes", @@ -167,11 +175,16 @@ impl VTable for VarBinView { // If any buffer is on device, skip host validation and use try_new_handle. if buffers.iter().any(|b| b.is_on_device()) { - return VarBinViewData::try_new_handle( + let data = VarBinViewData::try_new_handle( views_handle.clone(), Arc::from(data_handles.to_vec()), dtype.clone(), - validity, + validity.clone(), + )?; + let slots = VarBinViewData::make_slots(&validity, len); + return Ok( + crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, data) + .with_slots(slots), ); } @@ -181,28 +194,20 @@ impl VTable for VarBinView { .collect::>(); let views = Buffer::::from_byte_buffer(views_handle.clone().as_host().clone()); - VarBinViewData::try_new(views, Arc::from(data_buffers), dtype.clone(), validity) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + let data = VarBinViewData::try_new( + views, + Arc::from(data_buffers), + dtype.clone(), + validity.clone(), + )?; + let slots = VarBinViewData::make_slots(&validity, len); + Ok(crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { SLOT_NAMES[idx].to_string() } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "VarBinViewArray expects {} slots, got {}", - NUM_SLOTS, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn reduce_parent( array: ArrayView<'_, Self>, parent: &ArrayRef, diff --git a/vortex-array/src/arrays/varbinview/vtable/validity.rs b/vortex-array/src/arrays/varbinview/vtable/validity.rs index 93e4ff4b09f..f9dfbf64def 100644 --- a/vortex-array/src/arrays/varbinview/vtable/validity.rs +++ b/vortex-array/src/arrays/varbinview/vtable/validity.rs @@ -5,11 +5,12 @@ use vortex_error::VortexResult; use crate::array::ArrayView; use crate::array::ValidityVTable; +use crate::arrays::varbinview::VarBinViewArrayExt; use crate::arrays::varbinview::vtable::VarBinView; use crate::validity::Validity; impl ValidityVTable for VarBinView { fn validity(array: ArrayView<'_, VarBinView>) -> VortexResult { - Ok(array.data().validity()) + Ok(array.varbinview_validity()) } } diff --git a/vortex-array/src/arrays/variant/mod.rs b/vortex-array/src/arrays/variant/mod.rs index 02a89fae953..e50aafd869f 100644 --- a/vortex-array/src/arrays/variant/mod.rs +++ b/vortex-array/src/arrays/variant/mod.rs @@ -10,6 +10,7 @@ pub use self::vtable::VariantArray; use crate::ArrayRef; use crate::array::Array; use crate::array::ArrayParts; +use crate::array::TypedArrayRef; use crate::dtype::DType; pub(super) const NUM_SLOTS: usize = 1; @@ -24,40 +25,16 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; /// always `DType::Variant(child.dtype().nullability())`. The child's validity /// determines which rows are null. #[derive(Clone, Debug)] -pub struct VariantData { - pub(super) slots: Vec>, -} - -impl VariantData { - /// Creates a new VariantArray. Nullability comes from the child's dtype. - pub fn new(child: ArrayRef) -> Self { - Self { - slots: vec![Some(child)], - } - } - - /// Returns the length of this array. - pub fn len(&self) -> usize { - self.child().len() - } - - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - DType::Variant(self.child().dtype().nullability()) - } - - /// Returns `true` if this array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 - } +pub struct VariantData; - /// Returns a reference to the underlying child array. - pub fn child(&self) -> &ArrayRef { - self.slots[0] +pub trait VariantArrayExt: TypedArrayRef { + fn child(&self) -> &ArrayRef { + self.as_ref().slots()[0] .as_ref() - .vortex_expect("VariantArray child slot") + .vortex_expect("validated variant child slot") } } +impl> VariantArrayExt for T {} impl Array { /// Creates a new `VariantArray`. @@ -66,12 +43,9 @@ impl Array { let len = child.len(); let stats = child.statistics().to_owned(); unsafe { - Array::from_parts_unchecked(ArrayParts::new( - Variant, - dtype, - len, - VariantData::new(child), - )) + Array::from_parts_unchecked( + ArrayParts::new(Variant, dtype, len, VariantData).with_slots(vec![Some(child)]), + ) } .with_stats_set(stats) } diff --git a/vortex-array/src/arrays/variant/vtable/mod.rs b/vortex-array/src/arrays/variant/vtable/mod.rs index 49981a6f955..a38629399de 100644 --- a/vortex-array/src/arrays/variant/vtable/mod.rs +++ b/vortex-array/src/arrays/variant/vtable/mod.rs @@ -6,6 +6,7 @@ mod validity; use std::hash::Hasher; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; @@ -20,7 +21,6 @@ use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; -use crate::arrays::variant::NUM_SLOTS; use crate::arrays::variant::SLOT_NAMES; use crate::arrays::variant::VariantData; use crate::buffer::BufferHandle; @@ -37,6 +37,16 @@ impl Variant { pub const ID: ArrayId = ArrayId::new_ref("vortex.variant"); } +impl ArrayHash for VariantData { + fn array_hash(&self, _state: &mut H, _precision: Precision) {} +} + +impl ArrayEq for VariantData { + fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { + true + } +} + impl VTable for Variant { type ArrayData = VariantData; @@ -48,34 +58,39 @@ impl VTable for Variant { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + _data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + vortex_ensure!( + slots[0].is_some(), + "VariantArray child slot must be present" + ); + let child = slots[0] + .as_ref() + .vortex_expect("validated child slot presence"); vortex_ensure!( matches!(dtype, DType::Variant(_)), "Expected Variant DType, got {dtype}" ); vortex_ensure!( - data.child().dtype() == dtype, + child.dtype() == dtype, "VariantArray child dtype {} does not match outer dtype {}", - data.child().dtype(), + child.dtype(), dtype ); vortex_ensure!( - data.len() == len, + child.len() == len, "VariantArray length {} does not match outer length {}", - data.len(), + child.len(), len ); Ok(()) } - fn array_hash(array: &VariantData, state: &mut H, precision: Precision) { - array.child().array_hash(state, precision); - } - - fn array_eq(array: &VariantData, other: &VariantData, precision: Precision) -> bool { - array.child().array_eq(other.child(), precision) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -101,7 +116,7 @@ impl VTable for Variant { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &vortex_session::VortexSession, - ) -> VortexResult { + ) -> VortexResult> { vortex_ensure!( metadata.is_empty(), "VariantArray expects empty metadata, got {} bytes", @@ -115,11 +130,10 @@ impl VTable for Variant { ); // The child carries the nullability for the whole VariantArray. let child = children.get(0, dtype, len)?; - Ok(VariantData::new(child)) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + Ok( + crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, VariantData) + .with_slots(vec![Some(child)]), + ) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { @@ -129,44 +143,10 @@ impl VTable for Variant { } } - fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == NUM_SLOTS, - "VariantArray expects exactly {} slot, got {}", - NUM_SLOTS, - slots.len() - ); - vortex_ensure!( - slots[0].is_some(), - "VariantArray child slot must be present" - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done(array)) } } #[cfg(test)] -mod tests { - use super::*; - use crate::IntoArray; - use crate::arrays::ConstantArray; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::scalar::Scalar; - - #[test] - fn with_slots_rejects_missing_child() { - let child = - ConstantArray::new(Scalar::null(DType::Variant(Nullability::Nullable)), 3).into_array(); - let array = VariantArray::new(child); - let mut data = array.into_data(); - - let err = ::with_slots(&mut data, vec![None]).unwrap_err(); - - assert!(err.to_string().contains("child slot must be present")); - } -} +mod tests {} diff --git a/vortex-array/src/arrays/variant/vtable/operations.rs b/vortex-array/src/arrays/variant/vtable/operations.rs index 0e0495b7a91..fb6fc0eb499 100644 --- a/vortex-array/src/arrays/variant/vtable/operations.rs +++ b/vortex-array/src/arrays/variant/vtable/operations.rs @@ -7,6 +7,7 @@ use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::Variant; +use crate::arrays::variant::VariantArrayExt; use crate::scalar::Scalar; impl OperationsVTable for Variant { diff --git a/vortex-array/src/arrays/variant/vtable/validity.rs b/vortex-array/src/arrays/variant/vtable/validity.rs index 93fc2b786c6..42237c81977 100644 --- a/vortex-array/src/arrays/variant/vtable/validity.rs +++ b/vortex-array/src/arrays/variant/vtable/validity.rs @@ -6,6 +6,7 @@ use vortex_error::VortexResult; use crate::array::ArrayView; use crate::array::ValidityVTable; use crate::arrays::Variant; +use crate::arrays::variant::VariantArrayExt; use crate::validity::Validity; impl ValidityVTable for Variant { diff --git a/vortex-array/src/arrow/convert.rs b/vortex-array/src/arrow/convert.rs index e4fe0c7f9bd..3d60d4bc106 100644 --- a/vortex-array/src/arrow/convert.rs +++ b/vortex-array/src/arrow/convert.rs @@ -691,6 +691,9 @@ mod tests { use crate::arrays::Struct; use crate::arrays::VarBin; use crate::arrays::VarBinView; + use crate::arrays::fixed_size_list::FixedSizeListArrayExt; + use crate::arrays::list::ListArrayExt; + use crate::arrays::listview::ListViewArrayExt; use crate::arrow::FromArrowArray as _; use crate::arrow::convert::TemporalArray; use crate::dtype::DType; diff --git a/vortex-array/src/arrow/executor/bool.rs b/vortex-array/src/arrow/executor/bool.rs index e47f85bf0bf..28862a150d0 100644 --- a/vortex-array/src/arrow/executor/bool.rs +++ b/vortex-array/src/arrow/executor/bool.rs @@ -10,6 +10,7 @@ use vortex_error::VortexResult; use crate::ArrayRef; use crate::ExecutionCtx; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; use crate::arrow::null_buffer::to_null_buffer; /// Convert a canonical BoolArray directly to Arrow. diff --git a/vortex-array/src/arrow/executor/byte.rs b/vortex-array/src/arrow/executor/byte.rs index 3e097f03d63..bace6665bca 100644 --- a/vortex-array/src/arrow/executor/byte.rs +++ b/vortex-array/src/arrow/executor/byte.rs @@ -17,6 +17,7 @@ use crate::ExecutionCtx; use crate::array::ArrayView; use crate::arrays::VarBin; use crate::arrays::VarBinViewArray; +use crate::arrays::varbin::VarBinArrayExt; use crate::arrow::byte_view::execute_varbinview_to_arrow; use crate::arrow::executor::validity::to_arrow_null_buffer; use crate::builtins::ArrayBuiltins; diff --git a/vortex-array/src/arrow/executor/dictionary.rs b/vortex-array/src/arrow/executor/dictionary.rs index 23cc71e65dd..70f1ea8045f 100644 --- a/vortex-array/src/arrow/executor/dictionary.rs +++ b/vortex-array/src/arrow/executor/dictionary.rs @@ -21,7 +21,7 @@ use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Dict; use crate::arrays::DictArray; -use crate::arrays::dict::DictDataParts; +use crate::arrays::dict::DictArrayExt; use crate::arrow::ArrowArrayExecutor; pub(super) fn to_arrow_dictionary( @@ -30,11 +30,11 @@ pub(super) fn to_arrow_dictionary( values_type: &DataType, ctx: &mut ExecutionCtx, ) -> VortexResult { - let array = match array.try_into::() { + let array = match array.try_downcast::() { Ok(dict) => return dict_to_dict(dict, codes_type, values_type, ctx), Err(array) => array, }; - let array = match array.try_into::() { + let array = match array.try_downcast::() { Ok(constant) => return constant_to_dict(constant, codes_type, values_type, ctx), Err(array) => array, }; @@ -78,9 +78,11 @@ fn dict_to_dict( values_type: &DataType, ctx: &mut ExecutionCtx, ) -> VortexResult { - let DictDataParts { codes, values, .. } = array.into_data().into_parts(); - let codes = codes.execute_arrow(Some(codes_type), ctx)?; - let values = values.execute_arrow(Some(values_type), ctx)?; + let codes = array.codes().clone().execute_arrow(Some(codes_type), ctx)?; + let values = array + .values() + .clone() + .execute_arrow(Some(values_type), ctx)?; make_dict_array(codes_type, codes, values) } diff --git a/vortex-array/src/arrow/executor/fixed_size_list.rs b/vortex-array/src/arrow/executor/fixed_size_list.rs index ef0d5b53c7c..47dde9fb264 100644 --- a/vortex-array/src/arrow/executor/fixed_size_list.rs +++ b/vortex-array/src/arrow/executor/fixed_size_list.rs @@ -11,6 +11,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::arrays::FixedSizeList; use crate::arrays::FixedSizeListArray; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::validity::to_arrow_null_buffer; diff --git a/vortex-array/src/arrow/executor/list.rs b/vortex-array/src/arrow/executor/list.rs index e8742bcf5ab..302c4f7f892 100644 --- a/vortex-array/src/arrow/executor/list.rs +++ b/vortex-array/src/arrow/executor/list.rs @@ -21,6 +21,9 @@ use crate::arrays::List; use crate::arrays::ListArray; use crate::arrays::ListView; use crate::arrays::ListViewArray; +use crate::arrays::chunked::ChunkedArrayExt; +use crate::arrays::list::ListArrayExt; +use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::ListViewDataParts; use crate::arrays::listview::ListViewRebuildMode; use crate::arrow::ArrowArrayExecutor; @@ -53,7 +56,7 @@ pub(super) fn to_arrow_list( } // If the Vortex array is a ListViewArray, rebuild to ZCTL if needed and convert. - let array = match array.try_into::() { + let array = match array.try_downcast::() { Ok(array) => { let zctl = if array.is_zero_copy_to_list() { array @@ -139,7 +142,7 @@ fn list_view_zctl( sizes, validity, .. - } = array.into_data().into_parts(); + } = array.into_data_parts(); // For ZCTL, we know that we only care about the final size. assert!(!sizes.is_empty()); diff --git a/vortex-array/src/arrow/executor/list_view.rs b/vortex-array/src/arrow/executor/list_view.rs index 614ea6e7c93..c3de7f303a3 100644 --- a/vortex-array/src/arrow/executor/list_view.rs +++ b/vortex-array/src/arrow/executor/list_view.rs @@ -28,7 +28,7 @@ pub(super) fn to_arrow_list_view( ctx: &mut ExecutionCtx, ) -> VortexResult { // Check for Vortex ListViewArray and convert directly. - let array = match array.try_into::() { + let array = match array.try_downcast::() { Ok(array) => return list_view_to_list_view::(array, elements_field, ctx), Err(array) => array, }; @@ -49,7 +49,7 @@ fn list_view_to_list_view( sizes, validity, .. - } = array.into_data().into_parts(); + } = array.into_data_parts(); let elements = elements.execute_arrow(Some(elements_field.data_type()), ctx)?; vortex_ensure!( diff --git a/vortex-array/src/arrow/executor/mod.rs b/vortex-array/src/arrow/executor/mod.rs index a89fa03a530..890e7f8a46a 100644 --- a/vortex-array/src/arrow/executor/mod.rs +++ b/vortex-array/src/arrow/executor/mod.rs @@ -32,6 +32,8 @@ use vortex_error::vortex_ensure; use crate::ArrayRef; use crate::arrays::List; use crate::arrays::VarBin; +use crate::arrays::list::ListArrayExt; +use crate::arrays::varbin::VarBinArrayExt; use crate::arrow::executor::bool::to_arrow_bool; use crate::arrow::executor::byte::to_arrow_byte_array; use crate::arrow::executor::byte_view::to_arrow_byte_view; diff --git a/vortex-array/src/arrow/executor/run_end.rs b/vortex-array/src/arrow/executor/run_end.rs index a5de374fe10..ca434c2308d 100644 --- a/vortex-array/src/arrow/executor/run_end.rs +++ b/vortex-array/src/arrow/executor/run_end.rs @@ -46,7 +46,7 @@ pub(super) fn to_arrow_run_end( values_type: &Field, ctx: &mut ExecutionCtx, ) -> VortexResult { - let array = match array.try_into::() { + let array = match array.try_downcast::() { Ok(constant) => { return constant_to_run_end(constant, ends_type, values_type, ctx); } diff --git a/vortex-array/src/arrow/executor/struct_.rs b/vortex-array/src/arrow/executor/struct_.rs index 7edd1b05e35..a85005c0a20 100644 --- a/vortex-array/src/arrow/executor/struct_.rs +++ b/vortex-array/src/arrow/executor/struct_.rs @@ -19,6 +19,7 @@ use crate::arrays::Chunked; use crate::arrays::ScalarFnVTable; use crate::arrays::Struct; use crate::arrays::StructArray; +use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::struct_::StructDataParts; use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::validity::to_arrow_null_buffer; @@ -37,7 +38,7 @@ pub(super) fn to_arrow_struct( let len = array.len(); // If the array is chunked, then we invert the chunk-of-struct to struct-of-chunk. - let array = match array.try_into::() { + let array = match array.try_downcast::() { Ok(array) => { // NOTE(ngates): this currently uses the old into_canonical code path, but we should // just call directly into the swizzle-chunks function. @@ -47,16 +48,14 @@ pub(super) fn to_arrow_struct( }; // Attempt to short-circuit if the array is already a Struct: - let array = match array.try_into::() { + let array = match array.try_downcast::() { Ok(array) => { - let parts = array.into_parts(); - let len = parts.len; let StructDataParts { validity, fields, struct_fields, .. - } = parts.data.into_parts(); + } = array.into_data_parts(); let validity = to_arrow_null_buffer(validity, len, ctx)?; return create_from_fields( target_fields.ok_or_else(|| struct_fields.names().clone()), @@ -98,14 +97,12 @@ pub(super) fn to_arrow_struct( }; let struct_array = array.execute::(ctx)?; - let parts = struct_array.into_parts(); - let len = parts.len; let StructDataParts { validity, fields, struct_fields, .. - } = parts.data.into_parts(); + } = struct_array.into_data_parts(); let validity = to_arrow_null_buffer(validity, len, ctx)?; create_from_fields( diff --git a/vortex-array/src/arrow/executor/temporal.rs b/vortex-array/src/arrow/executor/temporal.rs index 3210d54963a..29bcc15763e 100644 --- a/vortex-array/src/arrow/executor/temporal.rs +++ b/vortex-array/src/arrow/executor/temporal.rs @@ -29,6 +29,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::arrays::ExtensionArray; use crate::arrays::PrimitiveArray as VortexPrimitiveArray; +use crate::arrays::extension::ExtensionArrayExt; use crate::arrow::null_buffer::to_null_buffer; use crate::dtype::NativePType; use crate::extension::datetime::AnyTemporal; diff --git a/vortex-array/src/builders/bool.rs b/vortex-array/src/builders/bool.rs index fa4073de98c..4c5eb5d9a61 100644 --- a/vortex-array/src/builders/bool.rs +++ b/vortex-array/src/builders/bool.rs @@ -13,6 +13,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; @@ -150,6 +151,7 @@ mod tests { use crate::LEGACY_SESSION; use crate::VortexSessionExecute; use crate::arrays::ChunkedArray; + use crate::arrays::bool::BoolArrayExt; use crate::assert_arrays_eq; use crate::builders::ArrayBuilder; use crate::builders::BoolBuilder; diff --git a/vortex-array/src/builders/dict/bytes.rs b/vortex-array/src/builders/dict/bytes.rs index 49b24700421..d3b39c14ca0 100644 --- a/vortex-array/src/builders/dict/bytes.rs +++ b/vortex-array/src/builders/dict/bytes.rs @@ -207,6 +207,7 @@ mod test { use crate::ToCanonical; use crate::accessor::ArrayAccessor; use crate::arrays::VarBinArray; + use crate::arrays::dict::DictArrayExt; use crate::builders::dict::dict_encode; #[test] diff --git a/vortex-array/src/builders/dict/mod.rs b/vortex-array/src/builders/dict/mod.rs index eb1807fe855..aa24000ac20 100644 --- a/vortex-array/src/builders/dict/mod.rs +++ b/vortex-array/src/builders/dict/mod.rs @@ -14,6 +14,7 @@ use crate::arrays::DictArray; use crate::arrays::Primitive; use crate::arrays::VarBin; use crate::arrays::VarBinView; +use crate::arrays::primitive::PrimitiveArrayExt; use crate::dtype::PType; use crate::match_each_native_ptype; diff --git a/vortex-array/src/builders/dict/primitive.rs b/vortex-array/src/builders/dict/primitive.rs index d16f909d95c..ac21ea9acd8 100644 --- a/vortex-array/src/builders/dict/primitive.rs +++ b/vortex-array/src/builders/dict/primitive.rs @@ -157,6 +157,7 @@ mod test { use vortex_buffer::buffer; use crate::IntoArray as _; + use crate::arrays::dict::DictArrayExt; use crate::assert_arrays_eq; use crate::builders::dict::dict_encode; use crate::builders::dict::primitive::PrimitiveArray; diff --git a/vortex-array/src/builders/extension.rs b/vortex-array/src/builders/extension.rs index 4875cd39182..ae540c1a6c2 100644 --- a/vortex-array/src/builders/extension.rs +++ b/vortex-array/src/builders/extension.rs @@ -10,6 +10,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; use crate::arrays::ExtensionArray; +use crate::arrays::extension::ExtensionArrayExt; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::builder_with_capacity; diff --git a/vortex-array/src/builders/fixed_size_list.rs b/vortex-array/src/builders/fixed_size_list.rs index ad8a5e91214..b951c0fc74b 100644 --- a/vortex-array/src/builders/fixed_size_list.rs +++ b/vortex-array/src/builders/fixed_size_list.rs @@ -14,6 +14,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; use crate::arrays::FixedSizeListArray; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; @@ -278,6 +279,7 @@ mod tests { use crate::IntoArray as _; use crate::ToCanonical; use crate::arrays::PrimitiveArray; + use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::builders::ArrayBuilder; use crate::builders::fixed_size_list::FixedSizeListArray; use crate::dtype::DType; diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs index b29bbedce58..11253f92a3a 100644 --- a/vortex-array/src/builders/list.rs +++ b/vortex-array/src/builders/list.rs @@ -15,6 +15,7 @@ use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; use crate::arrays::ListArray; +use crate::arrays::listview::ListViewArrayExt; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; @@ -318,6 +319,8 @@ mod tests { use crate::ToCanonical; use crate::arrays::ChunkedArray; use crate::arrays::PrimitiveArray; + use crate::arrays::list::ListArrayExt; + use crate::arrays::listview::ListViewArrayExt; use crate::assert_arrays_eq; use crate::builders::ArrayBuilder; use crate::builders::list::ListArray; diff --git a/vortex-array/src/builders/listview.rs b/vortex-array/src/builders/listview.rs index d88cf5558c0..714276d0660 100644 --- a/vortex-array/src/builders/listview.rs +++ b/vortex-array/src/builders/listview.rs @@ -24,6 +24,7 @@ use crate::ToCanonical; use crate::array::IntoArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; +use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::ListViewRebuildMode; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; @@ -433,6 +434,7 @@ mod tests { use super::ListViewBuilder; use crate::IntoArray; use crate::arrays::ListArray; + use crate::arrays::listview::ListViewArrayExt; use crate::assert_arrays_eq; use crate::builders::ArrayBuilder; use crate::builders::listview::PrimitiveArray; diff --git a/vortex-array/src/builders/struct_.rs b/vortex-array/src/builders/struct_.rs index 9986bc9612b..87f7ecb5e88 100644 --- a/vortex-array/src/builders/struct_.rs +++ b/vortex-array/src/builders/struct_.rs @@ -14,6 +14,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; use crate::arrays::StructArray; +use crate::arrays::struct_::StructArrayExt; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; diff --git a/vortex-array/src/builtins.rs b/vortex-array/src/builtins.rs index ba3b8542eb7..b4a3eb23984 100644 --- a/vortex-array/src/builtins.rs +++ b/vortex-array/src/builtins.rs @@ -14,7 +14,7 @@ use vortex_error::VortexResult; use crate::ArrayRef; use crate::IntoArray; use crate::arrays::ConstantArray; -use crate::arrays::scalar_fn::ScalarFnArrayExt; +use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::dtype::DType; use crate::dtype::FieldName; use crate::expr::Expression; diff --git a/vortex-array/src/canonical.rs b/vortex-array/src/canonical.rs index d48b5d532a2..cdf569a075c 100644 --- a/vortex-array/src/canonical.rs +++ b/vortex-array/src/canonical.rs @@ -17,6 +17,7 @@ use crate::Executable; use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; +use crate::array::child_to_validity; use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::arrays::Decimal; @@ -39,11 +40,14 @@ use crate::arrays::Variant; use crate::arrays::VariantArray; use crate::arrays::bool::BoolDataParts; use crate::arrays::decimal::DecimalDataParts; +use crate::arrays::extension::ExtensionArrayExt; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::arrays::listview::ListViewDataParts; use crate::arrays::listview::ListViewRebuildMode; use crate::arrays::primitive::PrimitiveDataParts; use crate::arrays::struct_::StructDataParts; use crate::arrays::varbinview::VarBinViewDataParts; +use crate::arrays::variant::VariantArrayExt; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::Nullability; @@ -544,13 +548,9 @@ impl Executable for CanonicalValidity { match array.execute::(ctx)? { n @ Canonical::Null(_) => Ok(CanonicalValidity(n)), Canonical::Bool(b) => { - let b = b.into_parts(); - let BoolDataParts { - bits, - offset, - len, - validity, - } = b.data.into_parts(); + let validity = child_to_validity(&b.slots()[0], b.dtype().nullability()); + let len = b.len(); + let BoolDataParts { bits, offset, len } = b.into_data().into_parts(len); Ok(CanonicalValidity(Canonical::Bool( BoolArray::try_new_from_handle(bits, offset, len, validity.execute(ctx)?)?, ))) @@ -560,7 +560,7 @@ impl Executable for CanonicalValidity { ptype, buffer, validity, - } = p.into_data().into_parts(); + } = p.into_data_parts(); Ok(CanonicalValidity(Canonical::Primitive(unsafe { PrimitiveArray::new_unchecked_from_handle(buffer, ptype, validity.execute(ctx)?) }))) @@ -571,7 +571,7 @@ impl Executable for CanonicalValidity { values, values_type, validity, - } = d.into_data().into_parts(); + } = d.into_data_parts(); Ok(CanonicalValidity(Canonical::Decimal(unsafe { DecimalArray::new_unchecked_handle( values, @@ -587,7 +587,7 @@ impl Executable for CanonicalValidity { buffers, views, validity, - } = vbv.into_data().into_parts(); + } = vbv.into_data_parts(); Ok(CanonicalValidity(Canonical::VarBinView(unsafe { VarBinViewArray::new_handle_unchecked( views, @@ -605,7 +605,7 @@ impl Executable for CanonicalValidity { sizes, validity, .. - } = l.into_data().into_parts(); + } = l.into_data_parts(); Ok(CanonicalValidity(Canonical::List(unsafe { ListViewArray::new_unchecked(elements, offsets, sizes, validity.execute(ctx)?) .with_zero_copy_to_list(zctl) @@ -614,19 +614,20 @@ impl Executable for CanonicalValidity { Canonical::FixedSizeList(fsl) => { let list_size = fsl.list_size(); let len = fsl.len(); - let (elements, validity, _) = fsl.into_data().into_parts(); + let parts = fsl.into_data_parts(); + let elements = parts.elements; + let validity = parts.validity; Ok(CanonicalValidity(Canonical::FixedSizeList( FixedSizeListArray::new(elements, list_size, validity.execute(ctx)?, len), ))) } Canonical::Struct(st) => { - let parts = st.into_parts(); - let len = parts.len; + let len = st.len(); let StructDataParts { struct_fields, fields, validity, - } = parts.data.into_parts(); + } = st.into_data_parts(); Ok(CanonicalValidity(Canonical::Struct(unsafe { StructArray::new_unchecked(fields, struct_fields, len, validity.execute(ctx)?) }))) @@ -666,13 +667,9 @@ impl Executable for RecursiveCanonical { match array.execute::(ctx)? { n @ Canonical::Null(_) => Ok(RecursiveCanonical(n)), Canonical::Bool(b) => { - let b = b.into_parts(); - let BoolDataParts { - bits, - offset, - len, - validity, - } = b.data.into_parts(); + let validity = child_to_validity(&b.slots()[0], b.dtype().nullability()); + let len = b.len(); + let BoolDataParts { bits, offset, len } = b.into_data().into_parts(len); Ok(RecursiveCanonical(Canonical::Bool( BoolArray::try_new_from_handle(bits, offset, len, validity.execute(ctx)?)?, ))) @@ -682,7 +679,7 @@ impl Executable for RecursiveCanonical { ptype, buffer, validity, - } = p.into_data().into_parts(); + } = p.into_data_parts(); Ok(RecursiveCanonical(Canonical::Primitive(unsafe { PrimitiveArray::new_unchecked_from_handle(buffer, ptype, validity.execute(ctx)?) }))) @@ -693,7 +690,7 @@ impl Executable for RecursiveCanonical { values, values_type, validity, - } = d.into_data().into_parts(); + } = d.into_data_parts(); Ok(RecursiveCanonical(Canonical::Decimal(unsafe { DecimalArray::new_unchecked_handle( values, @@ -709,7 +706,7 @@ impl Executable for RecursiveCanonical { buffers, views, validity, - } = vbv.into_data().into_parts(); + } = vbv.into_data_parts(); Ok(RecursiveCanonical(Canonical::VarBinView(unsafe { VarBinViewArray::new_handle_unchecked( views, @@ -727,7 +724,7 @@ impl Executable for RecursiveCanonical { sizes, validity, .. - } = l.into_data().into_parts(); + } = l.into_data_parts(); Ok(RecursiveCanonical(Canonical::List(unsafe { ListViewArray::new_unchecked( elements.execute::(ctx)?.0.into_array(), @@ -741,7 +738,9 @@ impl Executable for RecursiveCanonical { Canonical::FixedSizeList(fsl) => { let list_size = fsl.list_size(); let len = fsl.len(); - let (elements, validity, _) = fsl.into_data().into_parts(); + let parts = fsl.into_data_parts(); + let elements = parts.elements; + let validity = parts.validity; Ok(RecursiveCanonical(Canonical::FixedSizeList( FixedSizeListArray::new( elements.execute::(ctx)?.0.into_array(), @@ -752,13 +751,12 @@ impl Executable for RecursiveCanonical { ))) } Canonical::Struct(st) => { - let parts = st.into_parts(); - let len = parts.len; + let len = st.len(); let StructDataParts { struct_fields, fields, validity, - } = parts.data.into_parts(); + } = st.into_data_parts(); let executed_fields = fields .iter() .map(|f| Ok(f.clone().execute::(ctx)?.0.into_array())) @@ -821,7 +819,7 @@ impl Executable for Buffer { /// This will panic if the array's dtype is not primitive. impl Executable for PrimitiveArray { fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - match array.try_into::() { + match array.try_downcast::() { Ok(primitive) => Ok(primitive), Err(array) => Ok(Canonical::execute(array, ctx)?.into_primitive()), } @@ -833,7 +831,7 @@ impl Executable for PrimitiveArray { /// This will panic if the array's dtype is not bool. impl Executable for BoolArray { fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - match array.try_into::() { + match array.try_downcast::() { Ok(bool_array) => Ok(bool_array), Err(array) => Ok(Canonical::execute(array, ctx)?.into_bool()), } @@ -859,7 +857,7 @@ impl Executable for BitBuffer { /// This will panic if the array's dtype is not null. impl Executable for NullArray { fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - match array.try_into::() { + match array.try_downcast::() { Ok(null_array) => Ok(null_array), Err(array) => Ok(Canonical::execute(array, ctx)?.into_null()), } @@ -871,7 +869,7 @@ impl Executable for NullArray { /// This will panic if the array's dtype is not utf8 or binary. impl Executable for VarBinViewArray { fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - match array.try_into::() { + match array.try_downcast::() { Ok(varbinview) => Ok(varbinview), Err(array) => Ok(Canonical::execute(array, ctx)?.into_varbinview()), } @@ -883,7 +881,7 @@ impl Executable for VarBinViewArray { /// This will panic if the array's dtype is not an extension type. impl Executable for ExtensionArray { fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - match array.try_into::() { + match array.try_downcast::() { Ok(ext_array) => Ok(ext_array), Err(array) => Ok(Canonical::execute(array, ctx)?.into_extension()), } @@ -895,7 +893,7 @@ impl Executable for ExtensionArray { /// This will panic if the array's dtype is not decimal. impl Executable for DecimalArray { fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - match array.try_into::() { + match array.try_downcast::() { Ok(decimal) => Ok(decimal), Err(array) => Ok(Canonical::execute(array, ctx)?.into_decimal()), } @@ -907,7 +905,7 @@ impl Executable for DecimalArray { /// This will panic if the array's dtype is not list. impl Executable for ListViewArray { fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - match array.try_into::() { + match array.try_downcast::() { Ok(list) => Ok(list), Err(array) => Ok(Canonical::execute(array, ctx)?.into_listview()), } @@ -919,7 +917,7 @@ impl Executable for ListViewArray { /// This will panic if the array's dtype is not fixed size list. impl Executable for FixedSizeListArray { fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - match array.try_into::() { + match array.try_downcast::() { Ok(fsl) => Ok(fsl), Err(array) => Ok(Canonical::execute(array, ctx)?.into_fixed_size_list()), } @@ -931,7 +929,7 @@ impl Executable for FixedSizeListArray { /// This will panic if the array's dtype is not struct. impl Executable for StructArray { fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - match array.try_into::() { + match array.try_downcast::() { Ok(struct_array) => Ok(struct_array), Err(array) => Ok(Canonical::execute(array, ctx)?.into_struct()), } diff --git a/vortex-array/src/compute/conformance/mask.rs b/vortex-array/src/compute/conformance/mask.rs index ba673a7a8e1..debdff9a806 100644 --- a/vortex-array/src/compute/conformance/mask.rs +++ b/vortex-array/src/compute/conformance/mask.rs @@ -7,6 +7,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; use crate::builtins::ArrayBuiltins; /// Test mask compute function with various array sizes and patterns. diff --git a/vortex-array/src/display/mod.rs b/vortex-array/src/display/mod.rs index 42d9ace73d1..88da4f64fc3 100644 --- a/vortex-array/src/display/mod.rs +++ b/vortex-array/src/display/mod.rs @@ -607,7 +607,9 @@ impl ArrayRef { builder.push_record(null_row); } else { let mut row = Vec::new(); - for field_array in struct_.iter_unmasked_fields() { + for field_array in + crate::arrays::struct_::StructArrayExt::iter_unmasked_fields(&struct_) + { let value = field_array .scalar_at(row_idx) .map_or_else(|e| format!(""), |s| s.to_string()); diff --git a/vortex-array/src/expr/exprs.rs b/vortex-array/src/expr/exprs.rs index 31f37a35dd6..cbf34cdb3c2 100644 --- a/vortex-array/src/expr/exprs.rs +++ b/vortex-array/src/expr/exprs.rs @@ -174,6 +174,7 @@ pub fn nested_case_when( /// /// ``` /// # use vortex_array::arrays::{BoolArray, PrimitiveArray}; +/// # use vortex_array::arrays::bool::BoolArrayExt; /// # use vortex_array::{IntoArray, ToCanonical}; /// # use vortex_array::validity::Validity; /// # use vortex_buffer::buffer; @@ -198,6 +199,7 @@ pub fn eq(lhs: Expression, rhs: Expression) -> Expression { /// /// ``` /// # use vortex_array::arrays::{BoolArray, PrimitiveArray}; +/// # use vortex_array::arrays::bool::BoolArrayExt; /// # use vortex_array::{ IntoArray, ToCanonical}; /// # use vortex_array::validity::Validity; /// # use vortex_buffer::buffer; @@ -222,6 +224,7 @@ pub fn not_eq(lhs: Expression, rhs: Expression) -> Expression { /// /// ``` /// # use vortex_array::arrays::{BoolArray, PrimitiveArray }; +/// # use vortex_array::arrays::bool::BoolArrayExt; /// # use vortex_array::{IntoArray, ToCanonical}; /// # use vortex_array::validity::Validity; /// # use vortex_buffer::buffer; @@ -246,6 +249,7 @@ pub fn gt_eq(lhs: Expression, rhs: Expression) -> Expression { /// /// ``` /// # use vortex_array::arrays::{BoolArray, PrimitiveArray }; +/// # use vortex_array::arrays::bool::BoolArrayExt; /// # use vortex_array::{IntoArray, ToCanonical}; /// # use vortex_array::validity::Validity; /// # use vortex_buffer::buffer; @@ -270,6 +274,7 @@ pub fn gt(lhs: Expression, rhs: Expression) -> Expression { /// /// ``` /// # use vortex_array::arrays::{BoolArray, PrimitiveArray }; +/// # use vortex_array::arrays::bool::BoolArrayExt; /// # use vortex_array::{IntoArray, ToCanonical}; /// # use vortex_array::validity::Validity; /// # use vortex_buffer::buffer; @@ -294,6 +299,7 @@ pub fn lt_eq(lhs: Expression, rhs: Expression) -> Expression { /// /// ``` /// # use vortex_array::arrays::{BoolArray, PrimitiveArray }; +/// # use vortex_array::arrays::bool::BoolArrayExt; /// # use vortex_array::{IntoArray, ToCanonical}; /// # use vortex_array::validity::Validity; /// # use vortex_buffer::buffer; @@ -318,6 +324,7 @@ pub fn lt(lhs: Expression, rhs: Expression) -> Expression { /// /// ``` /// # use vortex_array::arrays::BoolArray; +/// # use vortex_array::arrays::bool::BoolArrayExt; /// # use vortex_array::{IntoArray, ToCanonical}; /// # use vortex_array::expr::{root, lit, or}; /// let xs = BoolArray::from_iter(vec![true, false, true]); @@ -353,6 +360,7 @@ where /// /// ``` /// # use vortex_array::arrays::BoolArray; +/// # use vortex_array::arrays::bool::BoolArrayExt; /// # use vortex_array::{IntoArray, ToCanonical}; /// # use vortex_array::expr::{and, root, lit}; /// let xs = BoolArray::from_iter(vec![true, false, true]).into_array(); diff --git a/vortex-array/src/iter.rs b/vortex-array/src/iter.rs index bfe375c627a..b4cd01d875d 100644 --- a/vortex-array/src/iter.rs +++ b/vortex-array/src/iter.rs @@ -12,6 +12,7 @@ use crate::ArrayRef; use crate::IntoArray; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; +use crate::arrays::chunked::ChunkedArrayExt; use crate::dtype::DType; use crate::stream::ArrayStream; use crate::stream::ArrayStreamAdapter; diff --git a/vortex-array/src/optimizer/mod.rs b/vortex-array/src/optimizer/mod.rs index b7ab6664374..a7051980efd 100644 --- a/vortex-array/src/optimizer/mod.rs +++ b/vortex-array/src/optimizer/mod.rs @@ -94,11 +94,11 @@ fn try_optimize_recursive(array: &ArrayRef) -> VortexResult> { for slot in current_array.slots() { match slot { Some(child) => { - if let Some(new_child) = try_optimize_recursive(&child)? { + if let Some(new_child) = try_optimize_recursive(child)? { new_slots.push(Some(new_child)); any_slot_optimized = true; } else { - new_slots.push(Some(child)); + new_slots.push(Some(child.clone())); } } None => new_slots.push(None), diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index 8ae49e21324..b84736c436f 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -27,6 +27,7 @@ use crate::IntoArray; use crate::ToCanonical; use crate::arrays::BoolArray; use crate::arrays::PrimitiveArray; +use crate::arrays::bool::BoolArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::IntegerPType; diff --git a/vortex-array/src/scalar_fn/fns/between/kernel.rs b/vortex-array/src/scalar_fn/fns/between/kernel.rs index d09ac4bd251..3bf543940c4 100644 --- a/vortex-array/src/scalar_fn/fns/between/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/between/kernel.rs @@ -13,6 +13,7 @@ use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::ScalarFnVTable; use crate::arrays::scalar_fn::ExactScalarFn; +use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; use crate::kernel::ExecuteParentKernel; use crate::optimizer::rules::ArrayParentReduceRule; diff --git a/vortex-array/src/scalar_fn/fns/binary/compare.rs b/vortex-array/src/scalar_fn/fns/binary/compare.rs index 874eaef87f1..0108aac6509 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare.rs +++ b/vortex-array/src/scalar_fn/fns/binary/compare.rs @@ -21,6 +21,7 @@ use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::ScalarFnVTable; use crate::arrays::scalar_fn::ExactScalarFn; +use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; use crate::arrow::Datum; use crate::arrow::IntoArrowArray; diff --git a/vortex-array/src/scalar_fn/fns/case_when.rs b/vortex-array/src/scalar_fn/fns/case_when.rs index a0248fcb2fd..f30a17ee580 100644 --- a/vortex-array/src/scalar_fn/fns/case_when.rs +++ b/vortex-array/src/scalar_fn/fns/case_when.rs @@ -28,6 +28,7 @@ use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::BoolArray; use crate::arrays::ConstantArray; +use crate::arrays::bool::BoolArrayExt; use crate::builders::ArrayBuilder; use crate::builders::builder_with_capacity; use crate::builtins::ArrayBuiltins; diff --git a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs b/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs index c8148cef8d6..dd2a6c9b16b 100644 --- a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs @@ -14,6 +14,7 @@ use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::ScalarFnVTable; use crate::arrays::scalar_fn::ExactScalarFn; +use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; use crate::builtins::ArrayBuiltins; use crate::kernel::ExecuteParentKernel; diff --git a/vortex-array/src/scalar_fn/fns/get_item.rs b/vortex-array/src/scalar_fn/fns/get_item.rs index c86962212b8..353ebc0925b 100644 --- a/vortex-array/src/scalar_fn/fns/get_item.rs +++ b/vortex-array/src/scalar_fn/fns/get_item.rs @@ -12,6 +12,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::ExecutionCtx; use crate::arrays::StructArray; +use crate::arrays::struct_::StructArrayExt; use crate::builtins::ArrayBuiltins; use crate::builtins::ExprBuiltins; use crate::dtype::DType; diff --git a/vortex-array/src/scalar_fn/fns/like/kernel.rs b/vortex-array/src/scalar_fn/fns/like/kernel.rs index acc9de692f5..0489504cc93 100644 --- a/vortex-array/src/scalar_fn/fns/like/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/like/kernel.rs @@ -10,6 +10,7 @@ use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::ScalarFnVTable; use crate::arrays::scalar_fn::ExactScalarFn; +use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; use crate::kernel::ExecuteParentKernel; use crate::optimizer::rules::ArrayParentReduceRule; diff --git a/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs b/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs index 40ab34237ef..7bc643606a7 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs @@ -10,6 +10,7 @@ use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::ScalarFnVTable; use crate::arrays::scalar_fn::ExactScalarFn; +use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; use crate::kernel::ExecuteParentKernel; use crate::optimizer::rules::ArrayParentReduceRule; diff --git a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs index 6bc07e93e71..b4d6c3e6bc4 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs +++ b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs @@ -25,7 +25,9 @@ use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; -use crate::arrays::scalar_fn::ScalarFnArrayExt; +use crate::arrays::bool::BoolArrayExt; +use crate::arrays::listview::ListViewArrayExt; +use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::IntegerPType; diff --git a/vortex-array/src/scalar_fn/fns/mask/mod.rs b/vortex-array/src/scalar_fn/fns/mask/mod.rs index abe80cc6fac..eaf83307b35 100644 --- a/vortex-array/src/scalar_fn/fns/mask/mod.rs +++ b/vortex-array/src/scalar_fn/fns/mask/mod.rs @@ -17,6 +17,7 @@ use crate::IntoArray; use crate::arrays::BoolArray; use crate::arrays::Constant; use crate::arrays::ConstantArray; +use crate::arrays::bool::BoolArrayExt; use crate::arrays::masked::mask_validity_canonical; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; diff --git a/vortex-array/src/scalar_fn/fns/merge.rs b/vortex-array/src/scalar_fn/fns/merge.rs index 4e613179425..fa5e77bcdc3 100644 --- a/vortex-array/src/scalar_fn/fns/merge.rs +++ b/vortex-array/src/scalar_fn/fns/merge.rs @@ -17,6 +17,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray as _; use crate::arrays::StructArray; +use crate::arrays::struct_::StructArrayExt; use crate::dtype::DType; use crate::dtype::FieldNames; use crate::dtype::Nullability; @@ -292,6 +293,7 @@ mod tests { use crate::IntoArray; use crate::ToCanonical; use crate::arrays::PrimitiveArray; + use crate::arrays::struct_::StructArrayExt; use crate::assert_arrays_eq; use crate::dtype::DType; use crate::dtype::Nullability::NonNullable; diff --git a/vortex-array/src/scalar_fn/fns/not/mod.rs b/vortex-array/src/scalar_fn/fns/not/mod.rs index d9c7f96e012..c70c8794fe6 100644 --- a/vortex-array/src/scalar_fn/fns/not/mod.rs +++ b/vortex-array/src/scalar_fn/fns/not/mod.rs @@ -16,6 +16,7 @@ use crate::IntoArray; use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::arrays::ConstantArray; +use crate::arrays::bool::BoolArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::expr::Expression; @@ -122,6 +123,7 @@ impl ScalarFnVTable for Not { mod tests { use crate::IntoArray; use crate::ToCanonical; + use crate::arrays::bool::BoolArrayExt; use crate::dtype::DType; use crate::dtype::Nullability; use crate::expr::col; diff --git a/vortex-array/src/scalar_fn/fns/pack.rs b/vortex-array/src/scalar_fn/fns/pack.rs index d9fca3822bd..cd276cfc561 100644 --- a/vortex-array/src/scalar_fn/fns/pack.rs +++ b/vortex-array/src/scalar_fn/fns/pack.rs @@ -169,6 +169,7 @@ mod tests { use crate::IntoArray; use crate::ToCanonical; use crate::arrays::PrimitiveArray; + use crate::arrays::struct_::StructArrayExt; use crate::assert_arrays_eq; use crate::dtype::Nullability; use crate::expr::col; diff --git a/vortex-array/src/scalar_fn/fns/zip/kernel.rs b/vortex-array/src/scalar_fn/fns/zip/kernel.rs index e66de4e4883..5cfefee70d7 100644 --- a/vortex-array/src/scalar_fn/fns/zip/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/zip/kernel.rs @@ -10,6 +10,7 @@ use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::ScalarFnVTable; use crate::arrays::scalar_fn::ExactScalarFn; +use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; use crate::kernel::ExecuteParentKernel; use crate::optimizer::rules::ArrayParentReduceRule; diff --git a/vortex-array/src/scalar_fn/fns/zip/mod.rs b/vortex-array/src/scalar_fn/fns/zip/mod.rs index 898726af895..e166148965a 100644 --- a/vortex-array/src/scalar_fn/fns/zip/mod.rs +++ b/vortex-array/src/scalar_fn/fns/zip/mod.rs @@ -17,6 +17,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; use crate::builders::ArrayBuilder; use crate::builders::builder_with_capacity; use crate::builtins::ArrayBuiltins; diff --git a/vortex-array/src/test_harness.rs b/vortex-array/src/test_harness.rs index cee718f3920..099b51bd289 100644 --- a/vortex-array/src/test_harness.rs +++ b/vortex-array/src/test_harness.rs @@ -9,6 +9,7 @@ use itertools::Itertools; use vortex_error::VortexResult; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; /// Check that a named metadata matches its previous versioning. /// diff --git a/vortex-array/src/validity.rs b/vortex-array/src/validity.rs index 08f0593381f..a5f6903e712 100644 --- a/vortex-array/src/validity.rs +++ b/vortex-array/src/validity.rs @@ -23,7 +23,8 @@ use crate::IntoArray; use crate::ToCanonical; use crate::arrays::BoolArray; use crate::arrays::ConstantArray; -use crate::arrays::scalar_fn::ScalarFnArrayExt; +use crate::arrays::bool::BoolArrayExt; +use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::Nullability; diff --git a/vortex-array/src/variants.rs b/vortex-array/src/variants.rs index 1e6ef0bb4df..f60eea316a9 100644 --- a/vortex-array/src/variants.rs +++ b/vortex-array/src/variants.rs @@ -16,6 +16,7 @@ use crate::LEGACY_SESSION; use crate::VortexSessionExecute; use crate::aggregate_fn::fns::sum::sum; use crate::arrays::BoolArray; +use crate::arrays::bool::BoolArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::FieldNames; diff --git a/vortex-bench/src/datasets/struct_list_of_ints.rs b/vortex-bench/src/datasets/struct_list_of_ints.rs index 021ad215e52..4999c05adac 100644 --- a/vortex-bench/src/datasets/struct_list_of_ints.rs +++ b/vortex-bench/src/datasets/struct_list_of_ints.rs @@ -17,6 +17,7 @@ use vortex::array::arrays::ChunkedArray; use vortex::array::arrays::ListArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; +use vortex::array::arrays::chunked::ChunkedArrayExt; use vortex::array::arrays::listview::recursive_list_from_list_view; use vortex::array::validity::Validity; use vortex::dtype::FieldNames; diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index 3ed6a883c44..0443fef42b7 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -4,12 +4,16 @@ //! Float compression schemes. use vortex_alp::ALP; +use vortex_alp::ALPArrayExt; +use vortex_alp::ALPRDArrayExt; +use vortex_alp::ALPRDArrayOwnedExt; use vortex_alp::RDEncoder; use vortex_alp::alp_encode; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::ToCanonical; +use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::dtype::PType; use vortex_compressor::scheme::ChildSelection; use vortex_compressor::scheme::DescendantExclusion; @@ -148,14 +152,8 @@ impl Scheme for ALPRDScheme { let alp_rd = encoder.encode(stats.source()); let dtype = alp_rd.dtype().clone(); let right_bit_width = alp_rd.right_bit_width(); - let mut alp_rd_data = alp_rd.into_data(); - - let patches = alp_rd_data - .left_parts_patches() - .map(compress_patches) - .transpose()?; - alp_rd_data.replace_left_parts_patches(patches); - let parts = alp_rd_data.into_parts(); + let mut parts = ALPRDArrayOwnedExt::into_data_parts(alp_rd); + parts.left_parts_patches = parts.left_parts_patches.map(compress_patches).transpose()?; Ok(vortex_alp::ALPRD::try_new( dtype, diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index f0211d5b7da..861a3ffa6b0 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -8,6 +8,7 @@ use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::ToCanonical; use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::scalar::Scalar; use vortex_compressor::builtins::FloatDictScheme; use vortex_compressor::builtins::StringDictScheme; @@ -18,7 +19,9 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; +use vortex_fastlanes::BitPackedArrayExt; use vortex_fastlanes::FoR; +use vortex_fastlanes::FoRArrayExt; use vortex_fastlanes::bitpack_compress::bit_width_histogram; use vortex_fastlanes::bitpack_compress::bitpack_encode; use vortex_fastlanes::bitpack_compress::find_best_bit_width; @@ -27,6 +30,7 @@ use vortex_runend::compress::runend_encode; use vortex_sequence::sequence_encode; use vortex_sparse::Sparse; use vortex_zigzag::ZigZag; +use vortex_zigzag::ZigZagArrayExt; use vortex_zigzag::zigzag_encode; use crate::ArrayAndStats; @@ -337,13 +341,9 @@ impl Scheme for BitPackingScheme { let packed = bitpack_encode(stats.source(), bw, Some(&histogram))?; let packed_stats = packed.statistics().to_owned(); let ptype = packed.dtype().as_ptype(); - let len = packed.len(); - let nullability = packed.dtype().nullability(); - let mut packed_data = packed.into_data(); - - let patches = packed_data.patches(len).map(compress_patches).transpose()?; - packed_data.replace_patches(patches); - let parts = packed_data.into_parts(len, nullability); + let patches = packed.patches().map(compress_patches).transpose()?; + let mut parts = vortex_fastlanes::BitPacked::into_parts(packed); + parts.patches = patches; Ok(vortex_fastlanes::BitPacked::try_new( parts.packed, @@ -853,7 +853,7 @@ mod tests { .into_array(); let btr = BtrBlocksCompressor::default(); - drop(btr.compress(&prim)?); + btr.compress(&prim)?; Ok(()) } diff --git a/vortex-btrblocks/src/schemes/patches.rs b/vortex-btrblocks/src/schemes/patches.rs index 0083918ff98..8a82058ddb7 100644 --- a/vortex-btrblocks/src/schemes/patches.rs +++ b/vortex-btrblocks/src/schemes/patches.rs @@ -5,6 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::ToCanonical; use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::patches::Patches; use vortex_error::VortexError; use vortex_error::VortexResult; diff --git a/vortex-btrblocks/src/schemes/rle.rs b/vortex-btrblocks/src/schemes/rle.rs index 42fd4295bb7..0ffd3f9f582 100644 --- a/vortex-btrblocks/src/schemes/rle.rs +++ b/vortex-btrblocks/src/schemes/rle.rs @@ -9,6 +9,7 @@ use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_compressor::builtins::FloatDictScheme; use vortex_compressor::builtins::StringDictScheme; use vortex_compressor::builtins::is_float_primitive; @@ -24,6 +25,7 @@ use vortex_error::VortexResult; #[cfg(feature = "unstable_encodings")] use vortex_fastlanes::Delta; use vortex_fastlanes::RLE; +use vortex_fastlanes::RLEArrayExt; use crate::ArrayAndStats; use crate::CascadingCompressor; diff --git a/vortex-btrblocks/src/schemes/string.rs b/vortex-btrblocks/src/schemes/string.rs index b4e15b6c76b..6d9aa29a26a 100644 --- a/vortex-btrblocks/src/schemes/string.rs +++ b/vortex-btrblocks/src/schemes/string.rs @@ -8,10 +8,13 @@ use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::ToCanonical; use vortex_array::arrays::VarBinArray; +use vortex_array::arrays::primitive::PrimitiveArrayExt; +use vortex_array::arrays::varbin::VarBinArrayExt; use vortex_compressor::scheme::ChildSelection; use vortex_compressor::scheme::DescendantExclusion; use vortex_error::VortexResult; use vortex_fsst::FSST; +use vortex_fsst::FSSTArrayExt; use vortex_fsst::fsst_compress; use vortex_fsst::fsst_train_compressor; use vortex_sparse::Sparse; diff --git a/vortex-btrblocks/src/schemes/temporal.rs b/vortex-btrblocks/src/schemes/temporal.rs index a2d1bdfd4e8..5ac1ce775ec 100644 --- a/vortex-btrblocks/src/schemes/temporal.rs +++ b/vortex-btrblocks/src/schemes/temporal.rs @@ -10,6 +10,7 @@ use vortex_array::ToCanonical; use vortex_array::aggregate_fn::fns::is_constant::is_constant; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::TemporalArray; +use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::dtype::extension::Matcher; use vortex_array::extension::datetime::AnyTemporal; use vortex_array::extension::datetime::TemporalMetadata; diff --git a/vortex-compressor/src/builtins/dict/float.rs b/vortex-compressor/src/builtins/dict/float.rs index 7b745bb891a..3525b687792 100644 --- a/vortex-compressor/src/builtins/dict/float.rs +++ b/vortex-compressor/src/builtins/dict/float.rs @@ -119,6 +119,7 @@ mod tests { use vortex_array::ToCanonical; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::dict::DictArrayExt; use vortex_array::assert_arrays_eq; use vortex_array::validity::Validity; use vortex_buffer::buffer; diff --git a/vortex-compressor/src/builtins/dict/integer.rs b/vortex-compressor/src/builtins/dict/integer.rs index 1e651a81861..c86579258be 100644 --- a/vortex-compressor/src/builtins/dict/integer.rs +++ b/vortex-compressor/src/builtins/dict/integer.rs @@ -134,6 +134,7 @@ mod tests { use vortex_array::ToCanonical; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::dict::DictArrayExt; use vortex_array::assert_arrays_eq; use vortex_array::validity::Validity; use vortex_buffer::buffer; diff --git a/vortex-compressor/src/builtins/dict/mod.rs b/vortex-compressor/src/builtins/dict/mod.rs index b7ff63d6b38..84d7c552d1d 100644 --- a/vortex-compressor/src/builtins/dict/mod.rs +++ b/vortex-compressor/src/builtins/dict/mod.rs @@ -11,6 +11,8 @@ use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::ToCanonical; use vortex_array::arrays::DictArray; +use vortex_array::arrays::dict::DictArrayExt; +use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::builders::dict::dict_encode; use vortex_error::VortexExpect; use vortex_error::VortexResult; diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index a7594feac4a..bbdd060e59b 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -21,7 +21,13 @@ use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::ListArray; use vortex_array::arrays::ListViewArray; use vortex_array::arrays::StructArray; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::list::ListArrayExt; +use vortex_array::arrays::listview::ListViewArrayExt; use vortex_array::arrays::listview::list_from_list_view; +use vortex_array::arrays::primitive::PrimitiveArrayExt; +use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; @@ -418,7 +424,7 @@ impl CascadingCompressor { )?; Ok( - ListArray::try_new(compressed_elems, compressed_offsets, list_array.validity())? + ListArray::try_new(compressed_elems, compressed_offsets, list_array.validity()?)? .into_array(), ) } diff --git a/vortex-compressor/src/stats/bool.rs b/vortex-compressor/src/stats/bool.rs index 0f85d8f52b2..380c71c1a41 100644 --- a/vortex-compressor/src/stats/bool.rs +++ b/vortex-compressor/src/stats/bool.rs @@ -4,6 +4,7 @@ //! Bool compression statistics. use vortex_array::arrays::BoolArray; +use vortex_array::arrays::bool::BoolArrayExt; use vortex_error::VortexResult; use vortex_mask::AllOr; diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index 6d4bb42d619..1f5688b08b3 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -26,11 +26,13 @@ use vortex::array::validity::Validity::NonNullable; use vortex::buffer::Buffer; use vortex::dtype::PType; use vortex::encodings::alp::ALP; +use vortex::encodings::alp::ALPArrayExt; use vortex::encodings::alp::ALPFloat; use vortex::encodings::alp::Exponents; use vortex::encodings::alp::alp_encode; use vortex::encodings::fastlanes::BitPackedData; use vortex::encodings::fastlanes::FoR; +use vortex::encodings::fastlanes::FoRArrayExt; use vortex::encodings::fastlanes::FoRData; use vortex::encodings::runend::RunEnd; use vortex::error::VortexExpect; diff --git a/vortex-cuda/gpu-scan-cli/src/main.rs b/vortex-cuda/gpu-scan-cli/src/main.rs index fac73e69263..4c343742000 100644 --- a/vortex-cuda/gpu-scan-cli/src/main.rs +++ b/vortex-cuda/gpu-scan-cli/src/main.rs @@ -20,6 +20,7 @@ use tracing_subscriber::util::SubscriberInitExt; use vortex::VortexSessionDefault; use vortex::array::ToCanonical; use vortex::array::arrays::Dict; +use vortex::array::arrays::struct_::StructArrayExt; use vortex::buffer::ByteBufferMut; use vortex::compressor::BtrBlocksCompressorBuilder; use vortex::error::VortexResult; diff --git a/vortex-cuda/src/arrow/canonical.rs b/vortex-cuda/src/arrow/canonical.rs index 4e38efe939e..660c2872259 100644 --- a/vortex-cuda/src/arrow/canonical.rs +++ b/vortex-cuda/src/arrow/canonical.rs @@ -9,6 +9,7 @@ use vortex::array::ToCanonical; use vortex::array::arrays::StructArray; use vortex::array::arrays::bool::BoolDataParts; use vortex::array::arrays::decimal::DecimalDataParts; +use vortex::array::arrays::extension::ExtensionArrayExt; use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::arrays::struct_::StructDataParts; use vortex::array::buffer::BufferHandle; @@ -68,7 +69,7 @@ fn export_canonical( let len = primitive.len(); let PrimitiveDataParts { buffer, validity, .. - } = primitive.into_data().into_parts(); + } = primitive.into_data_parts(); check_validity_empty(&validity)?; @@ -95,7 +96,7 @@ fn export_canonical( values_type, validity, .. - } = decimal.into_data().into_parts(); + } = decimal.into_data_parts(); // verify that there is no null buffer check_validity_empty(&validity)?; @@ -120,7 +121,7 @@ fn export_canonical( let PrimitiveDataParts { buffer, validity, .. - } = values.into_data().into_parts(); + } = values.into_data_parts(); check_validity_empty(&validity)?; @@ -128,13 +129,11 @@ fn export_canonical( export_fixed_size(buffer, len, 0, ctx) } Canonical::Bool(bool_array) => { + let len = bool_array.len(); + let validity = bool_array.validity()?; let BoolDataParts { - bits, - offset, - len, - validity, - .. - } = bool_array.into_data().into_parts(); + bits, offset, len, .. + } = bool_array.into_data().into_parts(len); check_validity_empty(&validity)?; @@ -184,7 +183,7 @@ async fn export_struct( let len = array.len(); let StructDataParts { validity, fields, .. - } = array.into_data().into_parts(); + } = array.into_data_parts(); check_validity_empty(&validity)?; @@ -265,7 +264,6 @@ unsafe extern "C" fn release_array(array: *mut ArrowArray) { for child in children { release_array(child); } - drop(private_data); } // update the release function to NULL to avoid any possibility of double-frees. diff --git a/vortex-cuda/src/arrow/varbinview.rs b/vortex-cuda/src/arrow/varbinview.rs index 72ea96437dd..c000d44908c 100644 --- a/vortex-cuda/src/arrow/varbinview.rs +++ b/vortex-cuda/src/arrow/varbinview.rs @@ -38,7 +38,7 @@ pub(crate) async fn copy_varbinview_to_varbin( buffers, validity, .. - } = array.into_data().into_parts(); + } = array.into_data_parts(); // TODO(aduffy): handle nulls check_validity_empty(&validity)?; diff --git a/vortex-cuda/src/canonical.rs b/vortex-cuda/src/canonical.rs index da87deb9f89..cde3841e196 100644 --- a/vortex-cuda/src/canonical.rs +++ b/vortex-cuda/src/canonical.rs @@ -15,6 +15,7 @@ use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinViewArray; use vortex::array::arrays::bool::BoolDataParts; use vortex::array::arrays::decimal::DecimalDataParts; +use vortex::array::arrays::extension::ExtensionArrayExt; use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::arrays::struct_::StructDataParts; use vortex::array::arrays::varbinview::BinaryView; @@ -45,7 +46,7 @@ impl CanonicalCudaExt for Canonical { struct_fields, validity, .. - } = struct_array.into_data().into_parts(); + } = struct_array.into_data_parts(); let mut host_fields = vec![]; for field in fields.iter() { @@ -63,13 +64,11 @@ impl CanonicalCudaExt for Canonical { Canonical::Bool(bool) => { // NOTE: update to copy to host when adding buffer handle. // Also update other method to copy validity to host. + let len = bool.len(); + let validity = bool.validity()?; let BoolDataParts { - bits, - validity, - offset, - len, - .. - } = bool.into_data().into_parts(); + bits, offset, len, .. + } = bool.into_data().into_parts(len); let bits = BitBuffer::new_with_offset(bits.try_into_host()?.await?, offset, len); Ok(Canonical::Bool(BoolArray::new(bits, validity))) @@ -80,7 +79,7 @@ impl CanonicalCudaExt for Canonical { buffer, validity, .. - } = prim.into_data().into_parts(); + } = prim.into_data_parts(); Ok(Canonical::Primitive(PrimitiveArray::from_byte_buffer( buffer.try_into_host()?.await?, ptype, @@ -94,7 +93,7 @@ impl CanonicalCudaExt for Canonical { values_type, validity, .. - } = decimal.into_data().into_parts(); + } = decimal.into_data_parts(); Ok(Canonical::Decimal(unsafe { DecimalArray::new_unchecked_handle( BufferHandle::new_host(values.try_into_host()?.await?), @@ -110,7 +109,7 @@ impl CanonicalCudaExt for Canonical { buffers, validity, dtype, - } = varbinview.into_data().into_parts(); + } = varbinview.into_data_parts(); // Copy all device views to host let host_views = views.try_into_host()?.await?; diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index cc886f9e81a..4cf2560a766 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -435,12 +435,14 @@ mod tests { use vortex::buffer::Buffer; use vortex::dtype::PType; use vortex::encodings::alp::ALP; + use vortex::encodings::alp::ALPArrayExt; use vortex::encodings::alp::ALPFloat; use vortex::encodings::alp::Exponents; use vortex::encodings::alp::alp_encode; use vortex::encodings::fastlanes::BitPacked; use vortex::encodings::fastlanes::BitPackedArray; use vortex::encodings::fastlanes::FoR; + use vortex::encodings::fastlanes::FoRArrayExt; use vortex::encodings::runend::RunEnd; use vortex::encodings::zigzag::ZigZag; use vortex::error::VortexExpect; diff --git a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs index 19e19119eb0..d9baa6029cd 100644 --- a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs +++ b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs @@ -12,15 +12,22 @@ use vortex::array::ArrayRef; use vortex::array::arrays::Dict; use vortex::array::arrays::Primitive; use vortex::array::arrays::Slice; +use vortex::array::arrays::dict::DictArrayExt; +use vortex::array::arrays::slice::SliceArrayExt; use vortex::array::buffer::BufferHandle; use vortex::dtype::PType; use vortex::encodings::alp::ALP; +use vortex::encodings::alp::ALPArrayExt; use vortex::encodings::alp::ALPFloat; use vortex::encodings::fastlanes::BitPacked; +use vortex::encodings::fastlanes::BitPackedArrayExt; use vortex::encodings::fastlanes::FoR; +use vortex::encodings::fastlanes::FoRArrayExt; use vortex::encodings::runend::RunEnd; +use vortex::encodings::runend::RunEndArrayExt; use vortex::encodings::sequence::Sequence; use vortex::encodings::zigzag::ZigZag; +use vortex::encodings::zigzag::ZigZagArrayExt; use vortex::error::VortexResult; use vortex::error::vortex_bail; use vortex::error::vortex_err; @@ -51,7 +58,7 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool { return arr.patches().is_none() && arr.dtype().as_ptype() == PType::F32; } if id == BitPacked::ID { - return array.as_::().patches(array.len()).is_none(); + return array.as_::().patches().is_none(); } if id == Dict::ID { let arr = array.as_::(); @@ -412,7 +419,7 @@ impl FusedPlan { fn walk_bitpacked(&mut self, array: ArrayRef) -> VortexResult { let bp = array.as_::(); - if bp.patches(array.len()).is_some() { + if bp.patches().is_some() { vortex_bail!("Dynamic dispatch does not support BitPackedArray with patches"); } diff --git a/vortex-cuda/src/executor.rs b/vortex-cuda/src/executor.rs index 4a14be98968..1d6d1db76c3 100644 --- a/vortex-cuda/src/executor.rs +++ b/vortex-cuda/src/executor.rs @@ -127,9 +127,6 @@ impl CudaExecutionCtx { let events = launch_cuda_kernel_impl(&mut launcher, self.strategy.event_flags(), len)?; self.strategy.on_complete(&events, len)?; - - drop(events); - Ok(()) } @@ -151,9 +148,6 @@ impl CudaExecutionCtx { let events = launch_cuda_kernel_with_config(&mut launcher, cfg, self.strategy.event_flags())?; self.strategy.on_complete(&events, len)?; - - drop(events); - Ok(()) } @@ -365,7 +359,7 @@ impl CudaArrayExt for ArrayRef { struct_fields, validity, .. - } = self.try_into::().unwrap().into_data().into_parts(); + } = self.try_downcast::().unwrap().into_data_parts(); let mut cuda_fields = Vec::with_capacity(fields.len()); for field in fields.iter() { diff --git a/vortex-cuda/src/hybrid_dispatch/mod.rs b/vortex-cuda/src/hybrid_dispatch/mod.rs index e81309db88e..61708c8248a 100644 --- a/vortex-cuda/src/hybrid_dispatch/mod.rs +++ b/vortex-cuda/src/hybrid_dispatch/mod.rs @@ -87,7 +87,7 @@ pub async fn try_gpu_dispatch( // TODO(0ax1): execute subtrees concurrently using separate CUDA streams. for subtree in &pending_subtrees { let canonical = subtree.clone().execute_cuda(ctx).await?; - subtree_buffers.push(canonical.into_primitive().into_data().into_parts().buffer); + subtree_buffers.push(canonical.into_primitive().into_data_parts().buffer); } let num_subtrees = subtree_buffers.len(); diff --git a/vortex-cuda/src/kernel/arrays/constant.rs b/vortex-cuda/src/kernel/arrays/constant.rs index dea6b47d95c..2164378b9e9 100644 --- a/vortex-cuda/src/kernel/arrays/constant.rs +++ b/vortex-cuda/src/kernel/arrays/constant.rs @@ -40,7 +40,7 @@ pub(crate) struct ConstantNumericExecutor; impl ConstantNumericExecutor { fn try_specialize(array: ArrayRef) -> Option { - array.try_into::().ok() + array.try_downcast::().ok() } } diff --git a/vortex-cuda/src/kernel/arrays/dict.rs b/vortex-cuda/src/kernel/arrays/dict.rs index c13d8196294..27d4208abe7 100644 --- a/vortex-cuda/src/kernel/arrays/dict.rs +++ b/vortex-cuda/src/kernel/arrays/dict.rs @@ -16,7 +16,7 @@ use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; use vortex::array::arrays::decimal::DecimalDataParts; -use vortex::array::arrays::dict::DictDataParts; +use vortex::array::arrays::dict::DictArrayExt; use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::arrays::varbinview::VarBinViewDataParts; use vortex::array::buffer::BufferHandle; @@ -49,7 +49,7 @@ impl CudaExecute for DictExecutor { ctx: &mut CudaExecutionCtx, ) -> VortexResult { let dict_array = array - .try_into::() + .try_downcast::() .ok() .vortex_expect("Array is not a Dict array"); @@ -65,7 +65,8 @@ impl CudaExecute for DictExecutor { #[expect(clippy::cognitive_complexity)] async fn execute_dict_prim(dict: DictArray, ctx: &mut CudaExecutionCtx) -> VortexResult { - let DictDataParts { values, codes, .. } = dict.into_data().into_parts(); + let values = dict.values().clone(); + let codes = dict.codes().clone(); // Execute both children to get them as primitives on the device let values_canonical = values.execute_cuda(ctx).await?; @@ -98,12 +99,12 @@ async fn execute_dict_prim_typed VortexResult { let dtype = dict.dtype().clone(); - let DictDataParts { values, codes, .. } = dict.into_data().into_parts(); + let values = dict.values().clone(); + let codes = dict.codes().clone(); // Execute codes to get them as primitives on the device let codes_prim = codes.execute_cuda(ctx).await?.into_primitive(); @@ -179,13 +181,13 @@ async fn execute_dict_decimal_typed< values: values_buffer, validity: values_validity, .. - } = values.into_data().into_parts(); + } = values.into_data_parts(); let output_validity = values_validity.take(&codes.clone().into_array())?; let PrimitiveDataParts { buffer: codes_buffer, .. - } = codes.into_data().into_parts(); + } = codes.into_data_parts(); // Copy buffers to device if needed let values_device = ctx.ensure_on_device(values_buffer).await?; @@ -231,7 +233,8 @@ async fn execute_dict_varbinview( ctx: &mut CudaExecutionCtx, ) -> VortexResult { let dtype = dict.dtype().clone(); - let DictDataParts { values, codes, .. } = dict.into_data().into_parts(); + let values = dict.values().clone(); + let codes = dict.codes().clone(); let codes_prim = codes.execute_cuda(ctx).await?.into_primitive(); let codes_ptype = codes_prim.ptype(); @@ -243,13 +246,13 @@ async fn execute_dict_varbinview( buffers: values_data_buffers, validity: values_validity, .. - } = values_vbv.into_data().into_parts(); + } = values_vbv.into_data_parts(); let output_validity = values_validity.take(&codes_prim.clone().into_array())?; let PrimitiveDataParts { buffer: codes_buffer, .. - } = codes_prim.into_data().into_parts(); + } = codes_prim.into_data_parts(); // Move buffers to device if needed. let values_device = ctx.ensure_on_device(values_views_handle).await?; diff --git a/vortex-cuda/src/kernel/arrays/shared.rs b/vortex-cuda/src/kernel/arrays/shared.rs index 0be3a7b82fd..085b30f9057 100644 --- a/vortex-cuda/src/kernel/arrays/shared.rs +++ b/vortex-cuda/src/kernel/arrays/shared.rs @@ -6,6 +6,7 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::Shared; +use vortex::array::arrays::shared::SharedArrayExt; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -26,7 +27,7 @@ impl CudaExecute for SharedExecutor { ctx: &mut CudaExecutionCtx, ) -> VortexResult { let shared = array - .try_into::() + .try_downcast::() .ok() .vortex_expect("Array is not a Shared array"); diff --git a/vortex-cuda/src/kernel/encodings/alp.rs b/vortex-cuda/src/kernel/encodings/alp.rs index 1635927a42e..859de6c59e3 100644 --- a/vortex-cuda/src/kernel/encodings/alp.rs +++ b/vortex-cuda/src/kernel/encodings/alp.rs @@ -17,6 +17,7 @@ use vortex::array::match_each_unsigned_integer_ptype; use vortex::dtype::NativePType; use vortex::encodings::alp::ALP; use vortex::encodings::alp::ALPArray; +use vortex::encodings::alp::ALPArrayExt; use vortex::encodings::alp::ALPFloat; use vortex::encodings::alp::match_each_alp_float_ptype; use vortex::error::VortexResult; @@ -43,7 +44,7 @@ impl CudaExecute for ALPExecutor { ctx: &mut CudaExecutionCtx, ) -> VortexResult { let array = array - .try_into::() + .try_downcast::() .map_err(|_| vortex_err!("Expected ALPArray"))?; match_each_alp_float_ptype!(array.dtype().as_ptype(), |A| { @@ -70,7 +71,7 @@ where let primitive = canonical.into_primitive(); let PrimitiveDataParts { buffer, validity, .. - } = primitive.into_data().into_parts(); + } = primitive.into_data_parts(); let device_input = ctx.ensure_on_device(buffer).await?; diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index 3bdff6b439d..357d444fae4 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -37,7 +37,7 @@ pub(crate) struct BitPackedExecutor; impl BitPackedExecutor { fn try_specialize(array: ArrayRef) -> Option { - array.try_into::().ok() + array.try_downcast::().ok() } } @@ -175,6 +175,7 @@ mod tests { use vortex::array::dtype::NativePType; use vortex::array::validity::Validity::NonNullable; use vortex::buffer::Buffer; + use vortex::encodings::fastlanes::BitPackedArrayExt; use vortex::error::VortexExpect; use vortex::session::VortexSession; @@ -199,7 +200,7 @@ mod tests { // Last two items should be patched let bp_with_patches = BitPacked::encode(&array.into_array(), bw)?; - assert!(bp_with_patches.patches(bp_with_patches.len()).is_some()); + assert!(bp_with_patches.patches().is_some()); let cpu_result = bp_with_patches.to_canonical()?.into_array(); @@ -230,7 +231,7 @@ mod tests { // Last two items should be patched let bp_with_patches = BitPacked::encode(&array.into_array(), 9)?; - assert!(bp_with_patches.patches(bp_with_patches.len()).is_some()); + assert!(bp_with_patches.patches().is_some()); let cpu_result = bp_with_patches.to_canonical()?.into_array(); diff --git a/vortex-cuda/src/kernel/encodings/date_time_parts.rs b/vortex-cuda/src/kernel/encodings/date_time_parts.rs index a0c4a446567..421a759e374 100644 --- a/vortex-cuda/src/kernel/encodings/date_time_parts.rs +++ b/vortex-cuda/src/kernel/encodings/date_time_parts.rs @@ -22,6 +22,7 @@ use vortex::dtype::NativePType; use vortex::dtype::Nullability; use vortex::dtype::PType; use vortex::encodings::datetime_parts::DateTimeParts; +use vortex::encodings::datetime_parts::DateTimePartsArrayExt; use vortex::error::VortexResult; use vortex::error::vortex_bail; use vortex::error::vortex_err; @@ -51,7 +52,7 @@ impl CudaExecute for DateTimePartsExecutor { ) -> VortexResult { let output_len = array.len(); let array = array - .try_into::() + .try_downcast::() .map_err(|_| vortex_err!("Expected DateTimePartsArray"))?; // Extract the temporal metadata from the dtype @@ -150,15 +151,15 @@ where let PrimitiveDataParts { buffer: days_buffer, .. - } = days.into_data().into_parts(); + } = days.into_data_parts(); let PrimitiveDataParts { buffer: seconds_buffer, .. - } = seconds.into_data().into_parts(); + } = seconds.into_data_parts(); let PrimitiveDataParts { buffer: subseconds_buffer, .. - } = subseconds.into_data().into_parts(); + } = subseconds.into_data_parts(); // Move buffers to device if not already there let days_device = ctx.ensure_on_device(days_buffer).await?; diff --git a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs index b5aa921a9b6..3f4bdb92146 100644 --- a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs +++ b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs @@ -10,7 +10,7 @@ use vortex::array::Canonical; use vortex::array::arrays::DecimalArray; use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::encodings::decimal_byte_parts::DecimalByteParts; -use vortex::encodings::decimal_byte_parts::DecimalBytePartsDataParts; +use vortex::encodings::decimal_byte_parts::DecimalBytePartsArrayExt; use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_bail; @@ -31,7 +31,7 @@ impl CudaExecute for DecimalBytePartsExecutor { array: ArrayRef, ctx: &mut CudaExecutionCtx, ) -> VortexResult { - let Ok(array) = array.try_into::() else { + let Ok(array) = array.try_downcast::() else { vortex_bail!("cannot downcast to DecimalBytePartsArray") }; @@ -39,7 +39,7 @@ impl CudaExecute for DecimalBytePartsExecutor { .dtype() .as_decimal_opt() .vortex_expect("DecimalBytePartsArray dtype must be decimal"); - let DecimalBytePartsDataParts { msp, .. } = array.into_data().into_parts(); + let msp = array.msp().clone(); let PrimitiveDataParts { buffer, ptype, @@ -49,8 +49,7 @@ impl CudaExecute for DecimalBytePartsExecutor { .execute_cuda(ctx) .await? .into_primitive() - .into_data() - .into_parts(); + .into_data_parts(); // SAFETY: The primitive array's buffer is already validated with correct type. // The decimal dtype matches the array's dtype, and validity is preserved. diff --git a/vortex-cuda/src/kernel/encodings/for_.rs b/vortex-cuda/src/kernel/encodings/for_.rs index 7fe9d89d64c..490b797b9b8 100644 --- a/vortex-cuda/src/kernel/encodings/for_.rs +++ b/vortex-cuda/src/kernel/encodings/for_.rs @@ -13,12 +13,14 @@ use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::Slice; use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::slice::SliceArrayExt; use vortex::array::match_each_integer_ptype; use vortex::array::match_each_native_simd_ptype; use vortex::dtype::NativePType; use vortex::encodings::fastlanes::BitPacked; use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRArray; +use vortex::encodings::fastlanes::FoRArrayExt; use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_ensure; @@ -36,7 +38,7 @@ pub(crate) struct FoRExecutor; impl FoRExecutor { fn try_specialize(array: ArrayRef) -> Option { - array.try_into::().ok() + array.try_downcast::().ok() } } @@ -98,7 +100,7 @@ where let primitive = canonical.into_primitive(); let PrimitiveDataParts { buffer, validity, .. - } = primitive.into_data().into_parts(); + } = primitive.into_data_parts(); let device_buffer = ctx.ensure_on_device(buffer).await?; diff --git a/vortex-cuda/src/kernel/encodings/runend.rs b/vortex-cuda/src/kernel/encodings/runend.rs index 88b64393a5e..3f3bce2579b 100644 --- a/vortex-cuda/src/kernel/encodings/runend.rs +++ b/vortex-cuda/src/kernel/encodings/runend.rs @@ -21,7 +21,7 @@ use vortex::dtype::NativePType; use vortex::dtype::PType; use vortex::encodings::runend::RunEnd; use vortex::encodings::runend::RunEndArray; -use vortex::encodings::runend::RunEndDataParts; +use vortex::encodings::runend::RunEndArrayExt; use vortex::error::VortexResult; use vortex::error::vortex_bail; use vortex::error::vortex_ensure; @@ -40,7 +40,7 @@ pub(crate) struct RunEndExecutor; impl RunEndExecutor { fn try_specialize(array: ArrayRef) -> Option { - array.try_into::().ok() + array.try_downcast::().ok() } } @@ -61,7 +61,8 @@ impl CudaExecute for RunEndExecutor { let offset = array.offset(); let output_len = array.len(); - let RunEndDataParts { ends, values, .. } = array.into_data().into_parts(); + let ends = array.ends().clone(); + let values = array.values().clone(); let values_ptype = PType::try_from(values.dtype())?; let ends_ptype = PType::try_from(ends.dtype())?; @@ -110,12 +111,12 @@ async fn decode_runend_typed VortexResult { let array = array - .try_into::() + .try_downcast::() .map_err(|_| vortex_err!("SequenceExecutor can only accept SequenceArray"))?; let len = array.len(); diff --git a/vortex-cuda/src/kernel/encodings/zigzag.rs b/vortex-cuda/src/kernel/encodings/zigzag.rs index e7cf89b2181..f13a19bf0e1 100644 --- a/vortex-cuda/src/kernel/encodings/zigzag.rs +++ b/vortex-cuda/src/kernel/encodings/zigzag.rs @@ -16,6 +16,7 @@ use vortex::dtype::NativePType; use vortex::dtype::PType; use vortex::encodings::zigzag::ZigZag; use vortex::encodings::zigzag::ZigZagArray; +use vortex::encodings::zigzag::ZigZagArrayExt; use vortex::error::VortexResult; use vortex::error::vortex_ensure; use vortex::error::vortex_err; @@ -31,7 +32,7 @@ pub(crate) struct ZigZagExecutor; impl ZigZagExecutor { fn try_specialize(array: ArrayRef) -> Option { - array.try_into::().ok() + array.try_downcast::().ok() } } @@ -72,7 +73,7 @@ where let primitive = canonical.into_primitive(); let PrimitiveDataParts { buffer, validity, .. - } = primitive.into_data().into_parts(); + } = primitive.into_data_parts(); let device_buffer = ctx.ensure_on_device(buffer).await?; diff --git a/vortex-cuda/src/kernel/encodings/zstd.rs b/vortex-cuda/src/kernel/encodings/zstd.rs index e5b7da77fec..d1b3b9b4423 100644 --- a/vortex-cuda/src/kernel/encodings/zstd.rs +++ b/vortex-cuda/src/kernel/encodings/zstd.rs @@ -185,7 +185,7 @@ pub(crate) struct ZstdExecutor; impl ZstdExecutor { fn try_specialize(array: ArrayRef) -> Option { - array.try_into::().ok() + array.try_downcast::().ok() } } diff --git a/vortex-cuda/src/kernel/encodings/zstd_buffers.rs b/vortex-cuda/src/kernel/encodings/zstd_buffers.rs index d44b57615dc..a95b9261750 100644 --- a/vortex-cuda/src/kernel/encodings/zstd_buffers.rs +++ b/vortex-cuda/src/kernel/encodings/zstd_buffers.rs @@ -41,7 +41,7 @@ impl CudaExecute for ZstdBuffersExecutor { ctx: &mut CudaExecutionCtx, ) -> VortexResult { let zstd_buffers = array - .try_into::() + .try_downcast::() .map_err(|_| vortex_err!("expected zstd buffers array"))?; decode_zstd_buffers(zstd_buffers, ctx).await } diff --git a/vortex-cuda/src/kernel/filter/decimal.rs b/vortex-cuda/src/kernel/filter/decimal.rs index 5e468dadee3..cdfc4342fa6 100644 --- a/vortex-cuda/src/kernel/filter/decimal.rs +++ b/vortex-cuda/src/kernel/filter/decimal.rs @@ -23,7 +23,7 @@ pub(super) async fn filter_decimal(values, mask, ctx).await?; diff --git a/vortex-cuda/src/kernel/filter/mod.rs b/vortex-cuda/src/kernel/filter/mod.rs index 4cf8efbc203..3c504113b11 100644 --- a/vortex-cuda/src/kernel/filter/mod.rs +++ b/vortex-cuda/src/kernel/filter/mod.rs @@ -19,7 +19,7 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::Filter; -use vortex::array::arrays::filter::FilterDataParts; +use vortex::array::arrays::filter::FilterArrayExt; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_decimal_value_type; use vortex::array::match_each_native_simd_ptype; @@ -50,10 +50,11 @@ impl CudaExecute for FilterExecutor { ctx: &mut CudaExecutionCtx, ) -> VortexResult { let filter_array = array - .try_into::() + .try_downcast::() .map_err(|_| vortex_err!("Expected FilterArray"))?; - let FilterDataParts { child, mask } = filter_array.into_data().into_parts(); + let child = filter_array.child().clone(); + let mask = filter_array.data().filter_mask().clone(); // Early return for trivial cases. match mask { diff --git a/vortex-cuda/src/kernel/filter/primitive.rs b/vortex-cuda/src/kernel/filter/primitive.rs index f45fe3f2994..c74c56e5b06 100644 --- a/vortex-cuda/src/kernel/filter/primitive.rs +++ b/vortex-cuda/src/kernel/filter/primitive.rs @@ -24,7 +24,7 @@ where { let PrimitiveDataParts { buffer, validity, .. - } = array.into_data().into_parts(); + } = array.into_data_parts(); let filtered_validity = validity.filter(&mask)?; let filtered_values = filter_sized::(buffer, mask, ctx).await?; diff --git a/vortex-cuda/src/kernel/filter/varbinview.rs b/vortex-cuda/src/kernel/filter/varbinview.rs index 8d673180d58..84632f5cca8 100644 --- a/vortex-cuda/src/kernel/filter/varbinview.rs +++ b/vortex-cuda/src/kernel/filter/varbinview.rs @@ -20,7 +20,7 @@ pub(super) async fn filter_varbinview( buffers, validity, dtype, - } = array.into_data().into_parts(); + } = array.into_data_parts(); let filtered_validity = validity.filter(&mask)?; diff --git a/vortex-cuda/src/kernel/patches/mod.rs b/vortex-cuda/src/kernel/patches/mod.rs index 3d611ee4759..206ac8bc1cd 100644 --- a/vortex-cuda/src/kernel/patches/mod.rs +++ b/vortex-cuda/src/kernel/patches/mod.rs @@ -70,12 +70,12 @@ pub(crate) async fn execute_patches< let PrimitiveDataParts { buffer: indices_buffer, .. - } = indices.into_data().into_parts(); + } = indices.into_data_parts(); let PrimitiveDataParts { buffer: values_buffer, .. - } = values.into_data().into_parts(); + } = values.into_data_parts(); let d_patch_indices = ctx.ensure_on_device(indices_buffer).await?; let d_patch_values = ctx.ensure_on_device(values_buffer).await?; @@ -170,7 +170,7 @@ mod tests { let PrimitiveDataParts { buffer: cuda_buffer, .. - } = values.into_data().into_parts(); + } = values.into_data_parts(); let handle = ctx.ensure_on_device(cuda_buffer).await.unwrap(); let device_buf = handle diff --git a/vortex-cuda/src/kernel/slice/mod.rs b/vortex-cuda/src/kernel/slice/mod.rs index 5326780b5f9..647faa64fcd 100644 --- a/vortex-cuda/src/kernel/slice/mod.rs +++ b/vortex-cuda/src/kernel/slice/mod.rs @@ -7,7 +7,7 @@ use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::arrays::Slice; -use vortex::array::arrays::slice::SliceDataParts; +use vortex::array::arrays::slice::SliceArrayExt; use vortex::error::VortexResult; use vortex::error::vortex_err; @@ -26,15 +26,15 @@ impl CudaExecute for SliceExecutor { array: ArrayRef, ctx: &mut CudaExecutionCtx, ) -> VortexResult { - let slice_array = array.try_into::().map_err(|array| { + let slice_array = array.try_downcast::().map_err(|array| { vortex_err!( "SliceExecutor requires input of SliceArray, was {}", array.encoding_id() ) })?; - let SliceDataParts { child, range } = slice_array.into_data().into_parts(); - let child = child.execute_cuda(ctx).await?; + let range = slice_array.data().slice_range().clone(); + let child = slice_array.child().clone().execute_cuda(ctx).await?; match child { Canonical::Null(null_array) => null_array.into_array().slice(range)?.to_canonical(), diff --git a/vortex-duckdb/src/convert/vector.rs b/vortex-duckdb/src/convert/vector.rs index 14c3f5eb3d1..029a5ae0638 100644 --- a/vortex-duckdb/src/convert/vector.rs +++ b/vortex-duckdb/src/convert/vector.rs @@ -377,6 +377,9 @@ mod tests { use vortex::array::ToCanonical; use vortex::array::arrays::BoolArray; + use vortex::array::arrays::fixed_size_list::FixedSizeListArrayExt; + use vortex::array::arrays::listview::ListViewArrayExt; + use vortex::array::arrays::struct_::StructArrayExt; use vortex::array::assert_arrays_eq; use vortex::error::VortexExpect; use vortex::mask::Mask; diff --git a/vortex-duckdb/src/datasource.rs b/vortex-duckdb/src/datasource.rs index 12aef769c66..64f9f0c5474 100644 --- a/vortex-duckdb/src/datasource.rs +++ b/vortex-duckdb/src/datasource.rs @@ -24,6 +24,7 @@ use vortex::array::VortexSessionExecute; use vortex::array::arrays::ScalarFnVTable; use vortex::array::arrays::Struct; use vortex::array::arrays::StructArray; +use vortex::array::arrays::scalar_fn::ScalarFnArrayExt; use vortex::array::optimizer::ArrayOptimizer; use vortex::dtype::DType; use vortex::dtype::FieldNames; diff --git a/vortex-duckdb/src/exporter/bool.rs b/vortex-duckdb/src/exporter/bool.rs index 75dfa195bc9..cb488ce5297 100644 --- a/vortex-duckdb/src/exporter/bool.rs +++ b/vortex-duckdb/src/exporter/bool.rs @@ -3,6 +3,7 @@ use vortex::array::ExecutionCtx; use vortex::array::arrays::BoolArray; +use vortex::array::arrays::bool::BoolArrayExt; use vortex::buffer::BitBuffer; use vortex::error::VortexResult; use vortex::mask::Mask; diff --git a/vortex-duckdb/src/exporter/decimal.rs b/vortex-duckdb/src/exporter/decimal.rs index 765e76272ca..2fe226b7f31 100644 --- a/vortex-duckdb/src/exporter/decimal.rs +++ b/vortex-duckdb/src/exporter/decimal.rs @@ -47,7 +47,7 @@ pub(crate) fn new_exporter( decimal_dtype, values_type, values, - } = array.into_data().into_parts(); + } = array.into_data_parts(); let dest_values_type = precision_to_duckdb_storage_size(&decimal_dtype)?; let nullability = validity.nullability(); let validity = validity.to_array(len).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/dict.rs b/vortex-duckdb/src/exporter/dict.rs index b88059f1e98..40a7d77acf7 100644 --- a/vortex-duckdb/src/exporter/dict.rs +++ b/vortex-duckdb/src/exporter/dict.rs @@ -11,6 +11,7 @@ use vortex::array::arrays::Constant; use vortex::array::arrays::ConstantArray; use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; +use vortex::array::arrays::dict::DictArrayExt; use vortex::array::match_each_integer_ptype; use vortex::dtype::IntegerPType; use vortex::error::VortexResult; diff --git a/vortex-duckdb/src/exporter/fixed_size_list.rs b/vortex-duckdb/src/exporter/fixed_size_list.rs index 9e57bf18085..d7fbfa64647 100644 --- a/vortex-duckdb/src/exporter/fixed_size_list.rs +++ b/vortex-duckdb/src/exporter/fixed_size_list.rs @@ -10,6 +10,7 @@ //! [`DType::FixedSizeList`]: vortex_array::dtype::DType::FixedSizeList use vortex::array::ExecutionCtx; use vortex::array::arrays::FixedSizeListArray; +use vortex::array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -38,7 +39,10 @@ pub(crate) fn new_exporter( ) -> VortexResult> { let list_size = array.list_size(); let len = array.len(); - let (elements, validity, dtype) = array.into_data().into_parts(); + let parts = array.into_data_parts(); + let elements = parts.elements; + let validity = parts.validity; + let dtype = parts.dtype; let mask = validity.to_array(len).execute::(ctx)?; let elements_exporter = new_array_exporter_with_flatten(elements, cache, ctx, true)?; diff --git a/vortex-duckdb/src/exporter/list.rs b/vortex-duckdb/src/exporter/list.rs index f2945123a13..7c807a7d6ef 100644 --- a/vortex-duckdb/src/exporter/list.rs +++ b/vortex-duckdb/src/exporter/list.rs @@ -50,7 +50,7 @@ pub(crate) fn new_exporter( offsets, validity, dtype, - } = array.into_data().into_parts(); + } = array.into_data_parts(); let num_elements = elements.len(); let validity = validity.to_array(array_len).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/list_view.rs b/vortex-duckdb/src/exporter/list_view.rs index 226f399c92f..5bdd9fe5ab7 100644 --- a/vortex-duckdb/src/exporter/list_view.rs +++ b/vortex-duckdb/src/exporter/list_view.rs @@ -53,7 +53,7 @@ pub(crate) fn new_exporter( offsets, sizes, validity, - } = array.into_data().into_parts(); + } = array.into_data_parts(); // Cache an `elements` vector up front so that future exports can reference it. let num_elements = elements.len(); let nullability = validity.nullability(); diff --git a/vortex-duckdb/src/exporter/mod.rs b/vortex-duckdb/src/exporter/mod.rs index 6b4069f701c..502038b858c 100644 --- a/vortex-duckdb/src/exporter/mod.rs +++ b/vortex-duckdb/src/exporter/mod.rs @@ -31,6 +31,7 @@ use vortex::array::arrays::Dict; use vortex::array::arrays::List; use vortex::array::arrays::StructArray; use vortex::array::arrays::TemporalArray; +use vortex::array::arrays::struct_::StructArrayExt; use vortex::encodings::runend::RunEnd; use vortex::encodings::sequence::Sequence; use vortex::error::VortexResult; @@ -139,27 +140,27 @@ fn new_array_exporter_with_flatten( ctx: &mut ExecutionCtx, flatten: bool, ) -> VortexResult> { - let array = match array.try_into::() { + let array = match array.try_downcast::() { Ok(array) => return constant::new_exporter(array), Err(array) => array, }; - let array = match array.try_into::() { + let array = match array.try_downcast::() { Ok(array) => return sequence::new_exporter(&array), Err(array) => array, }; - let array = match array.try_into::() { + let array = match array.try_downcast::() { Ok(array) => return run_end::new_exporter(array, cache, ctx), Err(array) => array, }; - let array = match array.try_into::() { + let array = match array.try_downcast::() { Ok(array) => return dict::new_exporter_with_flatten(&array, cache, ctx, flatten), Err(array) => array, }; - let array = match array.try_into::() { + let array = match array.try_downcast::() { Ok(array) => return list::new_exporter(array, cache, ctx), Err(array) => array, }; diff --git a/vortex-duckdb/src/exporter/run_end.rs b/vortex-duckdb/src/exporter/run_end.rs index af21438130d..fa62388cf57 100644 --- a/vortex-duckdb/src/exporter/run_end.rs +++ b/vortex-duckdb/src/exporter/run_end.rs @@ -11,7 +11,7 @@ use vortex::array::search_sorted::SearchSorted; use vortex::array::search_sorted::SearchSortedSide; use vortex::dtype::IntegerPType; use vortex::encodings::runend::RunEndArray; -use vortex::encodings::runend::RunEndDataParts; +use vortex::encodings::runend::RunEndArrayExt; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -38,7 +38,8 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let offset = array.offset(); - let RunEndDataParts { ends, values, .. } = array.into_data().into_parts(); + let ends = array.ends().clone(); + let values = array.values().clone(); let ends = ends.execute::(ctx)?; let values_exporter = new_array_exporter(values.clone(), cache, ctx)?; diff --git a/vortex-duckdb/src/exporter/struct_.rs b/vortex-duckdb/src/exporter/struct_.rs index 19ac59c69d9..f53ec437bf2 100644 --- a/vortex-duckdb/src/exporter/struct_.rs +++ b/vortex-duckdb/src/exporter/struct_.rs @@ -5,6 +5,7 @@ use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::arrays::BoolArray; use vortex::array::arrays::StructArray; +use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::arrays::struct_::StructDataParts; use vortex::array::builtins::ArrayBuiltins; use vortex::error::VortexResult; @@ -32,7 +33,7 @@ pub(crate) fn new_exporter( struct_fields, fields, .. - } = array.into_data().into_parts(); + } = array.into_data_parts(); let validity = validity.to_array(len).execute::(ctx)?; if validity.to_bit_buffer().true_count() == 0 { diff --git a/vortex-duckdb/src/exporter/varbinview.rs b/vortex-duckdb/src/exporter/varbinview.rs index 23142b5c4cb..8067b08a856 100644 --- a/vortex-duckdb/src/exporter/varbinview.rs +++ b/vortex-duckdb/src/exporter/varbinview.rs @@ -37,7 +37,7 @@ pub(crate) fn new_exporter( dtype, views, buffers, - } = array.into_data().into_parts(); + } = array.into_data_parts(); let validity = validity.to_array(len).execute::(ctx)?; if validity.all_false() { let ltype = LogicalType::try_from(dtype)?; diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index 3aee513b102..776a91f3216 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -13,6 +13,7 @@ use vortex::array::IntoArray; use vortex::array::ToCanonical; use vortex::array::arrays::NullArray; use vortex::array::arrays::PrimitiveArray; +use vortex::array::arrays::struct_::StructArrayExt; use vortex::array::validity::Validity; use vortex::buffer::Buffer; use vortex::dtype::DType; @@ -427,6 +428,7 @@ mod tests { use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinViewArray; + use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::validity::Validity; use vortex::buffer::buffer; #[cfg(not(miri))] diff --git a/vortex-ffi/src/expression.rs b/vortex-ffi/src/expression.rs index 7216629599c..1f7dcf67195 100644 --- a/vortex-ffi/src/expression.rs +++ b/vortex-ffi/src/expression.rs @@ -296,6 +296,7 @@ mod tests { use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinViewArray; + use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::validity::Validity; use vortex::buffer::Buffer; use vortex::buffer::buffer; diff --git a/vortex-file/src/tests.rs b/vortex-file/src/tests.rs index b4a31932994..833c2d5c772 100644 --- a/vortex-file/src/tests.rs +++ b/vortex-file/src/tests.rs @@ -24,6 +24,8 @@ use vortex_array::arrays::StructArray; use vortex_array::arrays::TemporalArray; use vortex_array::arrays::VarBinArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::dict::DictArrayExt; +use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::DecimalDType; diff --git a/vortex-file/tests/test_write_table.rs b/vortex-file/tests/test_write_table.rs index 74f5d0951d2..d0522c5a78c 100644 --- a/vortex-file/tests/test_write_table.rs +++ b/vortex-file/tests/test_write_table.rs @@ -15,6 +15,7 @@ use vortex_array::arrays::DictArray; use vortex_array::arrays::ListViewArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; +use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::FieldNames; use vortex_array::field_path; use vortex_array::scalar_fn::session::ScalarFnSession; diff --git a/vortex-jni/src/array.rs b/vortex-jni/src/array.rs index 527371c6357..c024188eeb7 100644 --- a/vortex-jni/src/array.rs +++ b/vortex-jni/src/array.rs @@ -29,6 +29,9 @@ use vortex::array::ArrayView; use vortex::array::ToCanonical; use vortex::array::arrays::VarBin; use vortex::array::arrays::VarBinView; +use vortex::array::arrays::extension::ExtensionArrayExt; +use vortex::array::arrays::struct_::StructArrayExt; +use vortex::array::arrays::varbin::VarBinArrayExt; use vortex::array::arrow::IntoArrowArray; use vortex::dtype::DType; use vortex::dtype::i256; diff --git a/vortex-layout/src/layouts/file_stats.rs b/vortex-layout/src/layouts/file_stats.rs index 8fc2397e959..f6b779b431d 100644 --- a/vortex-layout/src/layouts/file_stats.rs +++ b/vortex-layout/src/layouts/file_stats.rs @@ -11,6 +11,7 @@ use vortex_array::ArrayRef; use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical as _; use vortex_array::VortexSessionExecute; +use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::expr::stats::Stat; diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index 733b575b684..5ac3be5c6da 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -205,6 +205,7 @@ mod tests { use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; + use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::VarBinViewBuilder; use vortex_array::dtype::DType; diff --git a/vortex-layout/src/layouts/repartition.rs b/vortex-layout/src/layouts/repartition.rs index 741aef8b853..74b1bde0e98 100644 --- a/vortex-layout/src/layouts/repartition.rs +++ b/vortex-layout/src/layouts/repartition.rs @@ -475,6 +475,7 @@ mod tests { let _output = buf.pop_front().unwrap(); // Transition SharedState from Source to Cached for ALL slices sharing this Arc. + use vortex_array::arrays::shared::SharedArrayExt; shared_handle.get_or_compute(|source| source.to_canonical())?; // Before the fix this panicked with "attempt to subtract with overflow". diff --git a/vortex-layout/src/layouts/struct_/reader.rs b/vortex-layout/src/layouts/struct_/reader.rs index 0d803cdd4c7..e8ebb3dc1ad 100644 --- a/vortex-layout/src/layouts/struct_/reader.rs +++ b/vortex-layout/src/layouts/struct_/reader.rs @@ -13,6 +13,7 @@ use vortex_array::IntoArray; use vortex_array::MaskFuture; use vortex_array::ToCanonical; use vortex_array::arrays::StructArray; +use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; @@ -398,6 +399,7 @@ mod tests { use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; + use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::assert_arrays_eq; use vortex_array::assert_nth_scalar; use vortex_array::dtype::DType; diff --git a/vortex-layout/src/layouts/table.rs b/vortex-layout/src/layouts/table.rs index 9df9ed66809..ff068bf6e8e 100644 --- a/vortex-layout/src/layouts/table.rs +++ b/vortex-layout/src/layouts/table.rs @@ -17,6 +17,7 @@ use vortex_array::ArrayContext; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::ToCanonical; +use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Field; use vortex_array::dtype::FieldName; diff --git a/vortex-layout/src/layouts/zoned/zone_map.rs b/vortex-layout/src/layouts/zoned/zone_map.rs index bd8e14099d4..39ab7179236 100644 --- a/vortex-layout/src/layouts/zoned/zone_map.rs +++ b/vortex-layout/src/layouts/zoned/zone_map.rs @@ -10,6 +10,7 @@ use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::fns::sum::sum; use vortex_array::arrays::StructArray; +use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; @@ -278,6 +279,8 @@ mod tests { use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; + use vortex_array::arrays::bool::BoolArrayExt; + use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::assert_arrays_eq; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::VarBinViewBuilder; diff --git a/vortex-python/src/arrays/builtins/chunked.rs b/vortex-python/src/arrays/builtins/chunked.rs index 293932c21c2..5ca1aafd67f 100644 --- a/vortex-python/src/arrays/builtins/chunked.rs +++ b/vortex-python/src/arrays/builtins/chunked.rs @@ -5,9 +5,9 @@ use pyo3::PyRef; use pyo3::pyclass; use pyo3::pymethods; use vortex::array::arrays::Chunked; +use vortex::array::arrays::chunked::ChunkedArrayExt; use crate::arrays::PyArrayRef; -use crate::arrays::native::AsArrayRef; use crate::arrays::native::EncodingSubclass; use crate::arrays::native::PyNativeArray; @@ -23,7 +23,9 @@ impl EncodingSubclass for PyChunkedArray { impl PyChunkedArray { pub fn chunks(self_: PyRef<'_, Self>) -> Vec { self_ - .as_array_ref() + .as_super() + .inner() + .as_::() .iter_chunks() .map(|chunk| PyArrayRef::from(chunk.clone())) .collect() diff --git a/vortex-python/src/arrays/builtins/struct_.rs b/vortex-python/src/arrays/builtins/struct_.rs index 0fe52aa0402..b4453d91501 100644 --- a/vortex-python/src/arrays/builtins/struct_.rs +++ b/vortex-python/src/arrays/builtins/struct_.rs @@ -7,9 +7,10 @@ use pyo3::PyResult; use pyo3::pyclass; use pyo3::pymethods; use vortex::array::arrays::Struct; +use vortex::array::arrays::struct_::StructArrayExt; +use vortex::error::VortexExpect; use crate::arrays::PyArrayRef; -use crate::arrays::native::AsArrayRef; use crate::arrays::native::EncodingSubclass; use crate::arrays::native::PyNativeArray; use crate::error::PyVortexResult; @@ -26,14 +27,23 @@ impl EncodingSubclass for PyStructArray { impl PyStructArray { /// Returns the given field of the struct array. pub fn field(self_: PyRef<'_, Self>, name: &str) -> PyVortexResult { - let field = self_.as_array_ref().unmasked_field_by_name(name)?.clone(); + let field = self_ + .as_super() + .inner() + .as_opt::() + .vortex_expect("Failed to downcast array") + .unmasked_field_by_name(name)? + .clone(); Ok(PyArrayRef::from(field)) } /// Get an ordered list of field names for the struct fields. pub fn names(self_: PyRef<'_, Self>) -> PyResult> { Ok(self_ - .as_array_ref() + .as_super() + .inner() + .as_opt::() + .vortex_expect("Failed to downcast array") .struct_fields() .names() .iter() diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index 43d67dee265..7926f7aa3e3 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -26,6 +26,8 @@ use vortex::array::ArrayRef; use vortex::array::IntoArray; use vortex::array::ToCanonical; use vortex::array::arrays::Chunked; +use vortex::array::arrays::bool::BoolArrayExt; +use vortex::array::arrays::chunked::ChunkedArrayExt; use vortex::array::arrow::IntoArrowArray; use vortex::array::builtins::ArrayBuiltins; use vortex::array::match_each_integer_ptype; diff --git a/vortex-python/src/arrays/py/vtable.rs b/vortex-python/src/arrays/py/vtable.rs index dbbc767f253..e98135cff97 100644 --- a/vortex-python/src/arrays/py/vtable.rs +++ b/vortex-python/src/arrays/py/vtable.rs @@ -5,7 +5,10 @@ use std::hash::Hash; use std::sync::Arc; use vortex::array::Array; +use vortex::array::ArrayEq; +use vortex::array::ArrayHash; use vortex::array::ArrayId; +use vortex::array::ArrayParts; use vortex::array::ArrayRef; use vortex::array::ArrayView; use vortex::array::ExecutionCtx; @@ -35,6 +38,18 @@ pub struct PythonVTable { pub id: ArrayId, } +impl ArrayHash for PythonArray { + fn array_hash(&self, state: &mut H, _precision: Precision) { + Arc::as_ptr(&self.object).hash(state); + } +} + +impl ArrayEq for PythonArray { + fn array_eq(&self, other: &Self, _precision: Precision) -> bool { + Arc::ptr_eq(&self.object, &other.object) + } +} + impl VTable for PythonVTable { type ArrayData = PythonArray; @@ -45,21 +60,19 @@ impl VTable for PythonVTable { self.id.clone() } - fn validate(&self, data: &PythonArray, dtype: &DType, len: usize) -> VortexResult<()> { + fn validate( + &self, + data: &PythonArray, + dtype: &DType, + len: usize, + _slots: &[Option], + ) -> VortexResult<()> { vortex_ensure!(data.vtable.id == self.id, "PythonArray vtable id mismatch"); vortex_ensure!(&data.dtype == dtype, "PythonArray dtype mismatch"); vortex_ensure!(data.len == len, "PythonArray len mismatch"); Ok(()) } - fn array_hash(array: &PythonArray, state: &mut H, _precision: Precision) { - Arc::as_ptr(&array.object).hash(state); - } - - fn array_eq(array: &PythonArray, other: &PythonArray, _precision: Precision) -> bool { - Arc::ptr_eq(&array.object, &other.object) - } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { 0 } @@ -96,28 +109,15 @@ impl VTable for PythonVTable { _buffers: &[BufferHandle], _children: &dyn vortex::array::serde::ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { _ = bytes; vortex_bail!("PythonArray deserialization is not supported"); } - fn slots(_array: ArrayView<'_, Self>) -> &[Option] { - &[] - } - fn slot_name(_array: ArrayView<'_, Self>, _idx: usize) -> String { vortex_panic!("PythonArray has no slots") } - fn with_slots(_array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.is_empty(), - "PythonArray has no slots, got {}", - slots.len() - ); - Ok(()) - } - fn execute(_array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { todo!() } diff --git a/vortex-sqllogictest/bin/sqllogictests-runner.rs b/vortex-sqllogictest/bin/sqllogictests-runner.rs index 987bc2bb210..d20b851677f 100644 --- a/vortex-sqllogictest/bin/sqllogictests-runner.rs +++ b/vortex-sqllogictest/bin/sqllogictests-runner.rs @@ -46,103 +46,120 @@ async fn main() -> anyhow::Result<()> { let crate_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); let path = crate_path.join("slt/"); - - let all_errors = futures::stream::iter(list_files(path)?) - .map(|path| { - let mpb = mpb.clone(); - - async move { - let path = path.canonicalize()?; - - let mut errors = vec![]; - let factory = Arc::new(VortexFormatFactory::new()); - let session_state_builder = SessionStateBuilder::new() - .with_default_features() - .with_table_factory( - factory.get_ext().to_uppercase(), - Arc::new(DefaultTableFactory::new()), - ) - .with_file_formats(vec![factory]); - - let session = SessionContext::new_with_state(session_state_builder.build()) - .enable_url_table(); - - let filename = path - .file_name() - .vortex_expect("must be file") - .to_string_lossy(); - let records = parse_file(path.as_path())?; - - if !path.components().any(|comp| comp.as_os_str() == "duckdb") { - let df_pb = mpb.add(ProgressBar::new(records.len() as u64)); - df_pb.set_message(format!("DF {filename}")); - df_pb.set_style(ProgressStyle::default_spinner()); - - let mut df_runner = Runner::new(|| async { - Ok(DataFusion::new( - session.clone(), - path.clone(), - df_pb.clone(), - )) - }); - - df_runner.add_label("datafusion"); - df_runner.with_column_validator(strict_column_validator); - df_runner.with_normalizer(value_normalizer); - df_runner.with_validator(df_value_validator); - - for record in records.iter() { - if let Record::Halt { .. } = record { - break; - } - - if let Err(e) = df_runner.run_async(record.clone()).await { - errors.push(format!("DF Failure: {e}")); - } + let has_tpch_data = crate_path.join("slt/tpch/data/lineitem.vortex").exists(); + + let all_errors = futures::stream::iter( + list_files(path)? + .into_iter() + .filter(|path| { + has_tpch_data || !path.components().any(|comp| comp.as_os_str() == "tpch") + }) + .collect::>(), + ) + .map(|path| { + let mpb = mpb.clone(); + + async move { + let path = path.canonicalize()?; + + let mut errors = vec![]; + let factory = Arc::new(VortexFormatFactory::new()); + let session_state_builder = SessionStateBuilder::new() + .with_default_features() + .with_table_factory( + factory.get_ext().to_uppercase(), + Arc::new(DefaultTableFactory::new()), + ) + .with_file_formats(vec![factory]); + + let session = + SessionContext::new_with_state(session_state_builder.build()).enable_url_table(); + + let filename = path + .file_name() + .vortex_expect("must be file") + .to_string_lossy(); + let records = parse_file(path.as_path())?; + + if !path.components().any(|comp| comp.as_os_str() == "duckdb") { + let df_pb = mpb.add(ProgressBar::new(records.len() as u64)); + df_pb.set_message(format!("DF {filename}")); + df_pb.set_style(ProgressStyle::default_spinner()); + + let mut df_runner = Runner::new(|| async { + Ok(DataFusion::new( + session.clone(), + path.clone(), + df_pb.clone(), + )) + }); + + df_runner.add_label("datafusion"); + df_runner.with_column_validator(strict_column_validator); + df_runner.with_normalizer(value_normalizer); + df_runner.with_validator(df_value_validator); + + for record in records.iter() { + if let Record::Halt { .. } = record { + break; } - df_pb.finish_and_clear(); + if let Err(e) = df_runner.run_async(record.clone()).await { + errors.push(format!("DF Failure: {e}")); + } } - if !path - .components() - .any(|comp| comp.as_os_str() == "datafusion") - { - let duckdb_pb = mpb.add(ProgressBar::new(records.len() as u64)); - duckdb_pb.set_message(format!("DuckDB {filename}")); - - let mut duckdb_runner = Runner::new(|| async { - DuckDB::try_new(duckdb_pb.clone()) - .map_err(|e| DuckDBTestError::Other(e.to_string())) - }); - - duckdb_runner.add_label("duckdb"); - duckdb_runner.with_column_validator(strict_column_validator); - duckdb_runner.with_normalizer(value_normalizer); - - for record in records.iter() { - if let Record::Halt { .. } = record { - break; - } - - if let Err(e) = duckdb_runner.run_async(record.clone()).await { - errors.push(format!("DuckDB Failure: {e}")); - } + df_pb.finish_and_clear(); + } + + if !path + .components() + .any(|comp| comp.as_os_str() == "datafusion") + { + let duckdb_pb = mpb.add(ProgressBar::new(records.len() as u64)); + duckdb_pb.set_message(format!("DuckDB {filename}")); + + let mut duckdb_runner = Runner::new(|| async { + DuckDB::try_new(duckdb_pb.clone()) + .map_err(|e| DuckDBTestError::Other(e.to_string())) + }); + + duckdb_runner.add_label("duckdb"); + duckdb_runner.with_column_validator(strict_column_validator); + duckdb_runner.with_normalizer(value_normalizer); + + for record in records.iter() { + if let Record::Halt { .. } = record { + break; } - duckdb_pb.finish_and_clear(); + if let Err(e) = duckdb_runner.run_async(record.clone()).await { + errors.push(format!("DuckDB Failure: {e}")); + } } - anyhow::Ok(errors) + duckdb_pb.finish_and_clear(); } - }) - .buffer_unordered(args.test_threads) - .try_collect::>() - .await?; - for err in all_errors.into_iter().flatten() { + anyhow::Ok(errors) + } + }) + .buffer_unordered(args.test_threads) + .try_collect::>() + .await?; + + let errors = all_errors.into_iter().flatten().collect::>(); + for err in &errors { eprintln!("Failure: {err}"); } + if !has_tpch_data { + eprintln!("Skipping TPC-H sqllogictests because slt/tpch/data is not present."); + } + + if !errors.is_empty() { + anyhow::bail!("{} sqllogictest failure(s)", errors.len()); + } + Ok(()) } diff --git a/vortex-tensor/public-api.lock b/vortex-tensor/public-api.lock index 349d65144f9..f73e581d515 100644 --- a/vortex-tensor/public-api.lock +++ b/vortex-tensor/public-api.lock @@ -32,15 +32,11 @@ pub type vortex_tensor::encodings::turboquant::TurboQuant::OperationsVTable = vo pub type vortex_tensor::encodings::turboquant::TurboQuant::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild -pub fn vortex_tensor::encodings::turboquant::TurboQuant::array_eq(array: &vortex_tensor::encodings::turboquant::TurboQuantData, other: &vortex_tensor::encodings::turboquant::TurboQuantData, precision: vortex_array::hash::Precision) -> bool - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::array_hash(array: &vortex_tensor::encodings::turboquant::TurboQuantData, state: &mut H, precision: vortex_array::hash::Precision) - pub fn vortex_tensor::encodings::turboquant::TurboQuant::buffer(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle pub fn vortex_tensor::encodings::turboquant::TurboQuant::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_tensor::encodings::turboquant::TurboQuant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_tensor::encodings::turboquant::TurboQuant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_tensor::encodings::turboquant::TurboQuant::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -56,11 +52,7 @@ pub fn vortex_tensor::encodings::turboquant::TurboQuant::serialize(array: vortex pub fn vortex_tensor::encodings::turboquant::TurboQuant::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_tensor::encodings::turboquant::TurboQuant::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::with_slots(array: &mut vortex_tensor::encodings::turboquant::TurboQuantData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +pub fn vortex_tensor::encodings::turboquant::TurboQuant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::array::vtable::operations::OperationsVTable for vortex_tensor::encodings::turboquant::TurboQuant @@ -68,7 +60,7 @@ pub fn vortex_tensor::encodings::turboquant::TurboQuant::scalar_at(array: vortex impl vortex_array::array::vtable::validity::ValidityChild for vortex_tensor::encodings::turboquant::TurboQuant -pub fn vortex_tensor::encodings::turboquant::TurboQuant::validity_child(array: &vortex_tensor::encodings::turboquant::TurboQuantData) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_tensor::encodings::turboquant::TurboQuant::validity_child(array: vortex_array::array::view::ArrayView<'_, vortex_tensor::encodings::turboquant::TurboQuant>) -> vortex_array::array::erased::ArrayRef impl vortex_array::arrays::dict::take::TakeExecute for vortex_tensor::encodings::turboquant::TurboQuant @@ -102,21 +94,13 @@ impl vortex_tensor::encodings::turboquant::TurboQuantData pub fn vortex_tensor::encodings::turboquant::TurboQuantData::bit_width(&self) -> u8 -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::centroids(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::codes(&self) -> &vortex_array::array::erased::ArrayRef - pub fn vortex_tensor::encodings::turboquant::TurboQuantData::dimension(&self) -> u32 -pub unsafe fn vortex_tensor::encodings::turboquant::TurboQuantData::new_unchecked(dtype: &vortex_array::dtype::DType, codes: vortex_array::array::erased::ArrayRef, norms: vortex_array::array::erased::ArrayRef, centroids: vortex_array::array::erased::ArrayRef, rotation_signs: vortex_array::array::erased::ArrayRef) -> Self - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::norms(&self) -> &vortex_array::array::erased::ArrayRef +pub unsafe fn vortex_tensor::encodings::turboquant::TurboQuantData::new_unchecked(dimension: u32, bit_width: u8) -> Self pub fn vortex_tensor::encodings::turboquant::TurboQuantData::padded_dim(&self) -> u32 -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::rotation_signs(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::try_new(dtype: &vortex_array::dtype::DType, codes: vortex_array::array::erased::ArrayRef, norms: vortex_array::array::erased::ArrayRef, centroids: vortex_array::array::erased::ArrayRef, rotation_signs: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_tensor::encodings::turboquant::TurboQuantData::try_new(dimension: u32, bit_width: u8) -> vortex_error::VortexResult pub fn vortex_tensor::encodings::turboquant::TurboQuantData::validate(dtype: &vortex_array::dtype::DType, codes: &vortex_array::array::erased::ArrayRef, norms: &vortex_array::array::erased::ArrayRef, centroids: &vortex_array::array::erased::ArrayRef, rotation_signs: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult<()> @@ -128,6 +112,14 @@ impl core::fmt::Debug for vortex_tensor::encodings::turboquant::TurboQuantData pub fn vortex_tensor::encodings::turboquant::TurboQuantData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::hash::ArrayEq for vortex_tensor::encodings::turboquant::TurboQuantData + +pub fn vortex_tensor::encodings::turboquant::TurboQuantData::array_eq(&self, other: &Self, _precision: vortex_array::hash::Precision) -> bool + +impl vortex_array::hash::ArrayHash for vortex_tensor::encodings::turboquant::TurboQuantData + +pub fn vortex_tensor::encodings::turboquant::TurboQuantData::array_hash(&self, state: &mut H, _precision: vortex_array::hash::Precision) + pub struct vortex_tensor::encodings::turboquant::TurboQuantScheme impl core::clone::Clone for vortex_tensor::encodings::turboquant::TurboQuantScheme @@ -158,6 +150,38 @@ pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::matches(&self, ca pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::scheme_name(&self) -> &'static str +pub trait vortex_tensor::encodings::turboquant::TurboQuantArrayExt: vortex_array::array::typed::TypedArrayRef + +pub fn vortex_tensor::encodings::turboquant::TurboQuantArrayExt::bit_width(&self) -> u8 + +pub fn vortex_tensor::encodings::turboquant::TurboQuantArrayExt::centroids(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn vortex_tensor::encodings::turboquant::TurboQuantArrayExt::codes(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn vortex_tensor::encodings::turboquant::TurboQuantArrayExt::dimension(&self) -> u32 + +pub fn vortex_tensor::encodings::turboquant::TurboQuantArrayExt::norms(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn vortex_tensor::encodings::turboquant::TurboQuantArrayExt::padded_dim(&self) -> u32 + +pub fn vortex_tensor::encodings::turboquant::TurboQuantArrayExt::rotation_signs(&self) -> &vortex_array::array::erased::ArrayRef + +impl> vortex_tensor::encodings::turboquant::TurboQuantArrayExt for T + +pub fn T::bit_width(&self) -> u8 + +pub fn T::centroids(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn T::codes(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn T::dimension(&self) -> u32 + +pub fn T::norms(&self) -> &vortex_array::array::erased::ArrayRef + +pub fn T::padded_dim(&self) -> u32 + +pub fn T::rotation_signs(&self) -> &vortex_array::array::erased::ArrayRef + pub fn vortex_tensor::encodings::turboquant::turboquant_encode(ext: &vortex_array::arrays::extension::vtable::ExtensionArray, config: &vortex_tensor::encodings::turboquant::TurboQuantConfig, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub type vortex_tensor::encodings::turboquant::TurboQuantArray = vortex_array::array::typed::Array diff --git a/vortex-tensor/src/encodings/turboquant/array/data.rs b/vortex-tensor/src/encodings/turboquant/array/data.rs index 5006bb061f3..53f34fc25a7 100644 --- a/vortex-tensor/src/encodings/turboquant/array/data.rs +++ b/vortex-tensor/src/encodings/turboquant/array/data.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use vortex_array::ArrayRef; +use vortex_array::TypedArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; @@ -28,28 +29,6 @@ use crate::utils::tensor_list_size; /// A degenerate TurboQuant array has zero rows and `bit_width == 0`, with all slots empty. #[derive(Clone, Debug)] pub struct TurboQuantData { - /// Child arrays stored as slots. See [`Slot`] for positions: - /// - /// - [`Codes`](Slot::Codes): Non-nullable `FixedSizeListArray` with - /// `list_size == padded_dim`. Each row holds one u8 centroid index per padded coordinate. - /// Null vectors are represented by all-zero codes. The cascade compressor handles packing - /// to the actual `bit_width` on disk. - /// - /// - [`Norms`](Slot::Norms): Per-vector L2 norms, one per row. The dtype matches the element - /// type of the Vector (e.g., f64 norms for f64 vectors) and carries the nullability of the - /// parent dtype. Null vectors have null norms. This child determines the validity of the - /// entire TurboQuant array, enabling O(1) L2 norm readthrough without decompression. - /// - /// - [`Centroids`](Slot::Centroids): `PrimitiveArray` codebook with `2^bit_width` entries - /// that is shared across all rows. We always store these as f32 regardless of the input - /// element type because quantization itself introduces far more error than f32 precision - /// loss, and f16 inputs can be upcast to f32 before quantization. - /// - /// - [`RotationSigns`](Slot::RotationSigns): `BitPackedArray` of `3 * padded_dim` 1-bit sign - /// values for the 3-round SRHT rotation, stored in inverse application order, and shared - /// across all rows. - pub(crate) slots: Vec>, - /// The vector dimension `d`, cached from the `FixedSizeList` storage dtype's list size. /// /// Stored as a convenience field to avoid repeatedly extracting it from `dtype`. @@ -72,17 +51,20 @@ impl TurboQuantData { /// /// Returns an error if the provided components do not satisfy the invariants documented /// in [`new_unchecked`](Self::new_unchecked). - pub fn try_new( - dtype: &DType, - codes: ArrayRef, - norms: ArrayRef, - centroids: ArrayRef, - rotation_signs: ArrayRef, - ) -> VortexResult { - Self::validate(dtype, &codes, &norms, ¢roids, &rotation_signs)?; - - // SAFETY: we validate that the inputs are valid above. - Ok(unsafe { Self::new_unchecked(dtype, codes, norms, centroids, rotation_signs) }) + pub fn try_new(dimension: u32, bit_width: u8) -> VortexResult { + vortex_ensure!( + dimension >= TurboQuant::MIN_DIMENSION, + "TurboQuant requires dimension >= {}, got {dimension}", + TurboQuant::MIN_DIMENSION + ); + vortex_ensure!( + bit_width <= 8, + "bit_width is expected to be between 0 and 8, got {bit_width}" + ); + Ok(Self { + dimension, + bit_width, + }) } /// Build a TurboQuant array without validation. @@ -104,40 +86,8 @@ impl TurboQuantData { /// - For degenerate (empty) arrays: all children must be empty. /// /// Violating these invariants may produce incorrect results during decompression. - pub unsafe fn new_unchecked( - dtype: &DType, - codes: ArrayRef, - norms: ArrayRef, - centroids: ArrayRef, - rotation_signs: ArrayRef, - ) -> Self { - #[cfg(debug_assertions)] - Self::validate(dtype, &codes, &norms, ¢roids, &rotation_signs) - .vortex_expect("[Debug Assertion]: Invalid TurboQuantData parameters"); - - let dimension = dtype - .as_extension_opt() - .and_then(|ext| tensor_list_size(ext).ok()) - .vortex_expect("dtype must be a Vector extension type with FixedSizeList storage"); - - let bit_width = if centroids.is_empty() { - 0 - } else { - // Guaranteed to be 1-8 by validate(). - #[expect(clippy::cast_possible_truncation)] - { - centroids.len().trailing_zeros() as u8 - } - }; - - let mut slots = vec![None; Slot::COUNT]; - slots[Slot::Codes as usize] = Some(codes); - slots[Slot::Norms as usize] = Some(norms); - slots[Slot::Centroids as usize] = Some(centroids); - slots[Slot::RotationSigns as usize] = Some(rotation_signs); - + pub unsafe fn new_unchecked(dimension: u32, bit_width: u8) -> Self { Self { - slots, dimension, bit_width, } @@ -235,6 +185,20 @@ impl TurboQuantData { Ok(()) } + pub(crate) fn make_slots( + codes: ArrayRef, + norms: ArrayRef, + centroids: ArrayRef, + rotation_signs: ArrayRef, + ) -> Vec> { + let mut slots = vec![None; Slot::COUNT]; + slots[Slot::Codes as usize] = Some(codes); + slots[Slot::Norms as usize] = Some(norms); + slots[Slot::Centroids as usize] = Some(centroids); + slots[Slot::RotationSigns as usize] = Some(rotation_signs); + slots + } + /// The vector dimension `d`, as stored in the [`Vector`](crate::vector::Vector) extension /// dtype's `FixedSizeList` storage. pub fn dimension(&self) -> u32 { @@ -253,35 +217,44 @@ impl TurboQuantData { pub fn padded_dim(&self) -> u32 { self.dimension.next_power_of_two() } +} - /// The quantized codes child (`FixedSizeListArray`, one row per vector). - pub fn codes(&self) -> &ArrayRef { - self.slot(Slot::Codes as usize) +pub trait TurboQuantArrayExt: TypedArrayRef { + fn dimension(&self) -> u32 { + std::ops::Deref::deref(self).dimension() } - /// Per-vector L2 norms. The dtype matches the Vector's element type (f16, f32, or f64). - pub fn norms(&self) -> &ArrayRef { - self.slot(Slot::Norms as usize) + fn bit_width(&self) -> u8 { + std::ops::Deref::deref(self).bit_width() } - /// The codebook centroids (`PrimitiveArray`, length `2^bit_width`). - /// - /// Always f32 regardless of input element type: quantization noise dominates f32 - /// precision loss, and f16 inputs are upcast before quantization anyway. - pub fn centroids(&self) -> &ArrayRef { - self.slot(Slot::Centroids as usize) + fn padded_dim(&self) -> u32 { + std::ops::Deref::deref(self).padded_dim() } - /// The SRHT rotation signs (`BitPackedArray`, `3 * padded_dim` 1-bit values). - /// - /// Stored in inverse application order for efficient decode. - pub fn rotation_signs(&self) -> &ArrayRef { - self.slot(Slot::RotationSigns as usize) + fn codes(&self) -> &ArrayRef { + self.as_ref().slots()[Slot::Codes as usize] + .as_ref() + .vortex_expect("TurboQuantArray codes slot") } - fn slot(&self, idx: usize) -> &ArrayRef { - self.slots[idx] + fn norms(&self) -> &ArrayRef { + self.as_ref().slots()[Slot::Norms as usize] .as_ref() - .vortex_expect("required slot is None") + .vortex_expect("TurboQuantArray norms slot") + } + + fn centroids(&self) -> &ArrayRef { + self.as_ref().slots()[Slot::Centroids as usize] + .as_ref() + .vortex_expect("TurboQuantArray centroids slot") + } + + fn rotation_signs(&self) -> &ArrayRef { + self.as_ref().slots()[Slot::RotationSigns as usize] + .as_ref() + .vortex_expect("TurboQuantArray rotation_signs slot") } } + +impl> TurboQuantArrayExt for T {} diff --git a/vortex-tensor/src/encodings/turboquant/compress.rs b/vortex-tensor/src/encodings/turboquant/compress.rs index 34f2f2de8d9..ce617f7b9e8 100644 --- a/vortex-tensor/src/encodings/turboquant/compress.rs +++ b/vortex-tensor/src/encodings/turboquant/compress.rs @@ -10,6 +10,8 @@ use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; diff --git a/vortex-tensor/src/encodings/turboquant/compute/cosine_similarity.rs b/vortex-tensor/src/encodings/turboquant/compute/cosine_similarity.rs index 87aa37562cd..2f6745249bf 100644 --- a/vortex-tensor/src/encodings/turboquant/compute/cosine_similarity.rs +++ b/vortex-tensor/src/encodings/turboquant/compute/cosine_similarity.rs @@ -36,12 +36,14 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::match_each_float_ptype; use vortex_buffer::BufferMut; use vortex_error::VortexResult; use vortex_error::vortex_ensure_eq; use crate::encodings::turboquant::TurboQuant; +use crate::encodings::turboquant::TurboQuantArrayExt; use crate::encodings::turboquant::array::float_from_f32; use crate::utils::tensor_element_ptype; diff --git a/vortex-tensor/src/encodings/turboquant/compute/slice.rs b/vortex-tensor/src/encodings/turboquant/compute/slice.rs index a8daef6466b..c857bbabad0 100644 --- a/vortex-tensor/src/encodings/turboquant/compute/slice.rs +++ b/vortex-tensor/src/encodings/turboquant/compute/slice.rs @@ -10,6 +10,7 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::encodings::turboquant::TurboQuant; +use crate::encodings::turboquant::TurboQuantArrayExt; impl SliceReduce for TurboQuant { fn slice( diff --git a/vortex-tensor/src/encodings/turboquant/compute/take.rs b/vortex-tensor/src/encodings/turboquant/compute/take.rs index 7614f1577a7..25f43acad41 100644 --- a/vortex-tensor/src/encodings/turboquant/compute/take.rs +++ b/vortex-tensor/src/encodings/turboquant/compute/take.rs @@ -9,6 +9,7 @@ use vortex_array::arrays::dict::TakeExecute; use vortex_error::VortexResult; use crate::encodings::turboquant::TurboQuant; +use crate::encodings::turboquant::TurboQuantArrayExt; impl TakeExecute for TurboQuant { fn take( diff --git a/vortex-tensor/src/encodings/turboquant/decompress.rs b/vortex-tensor/src/encodings/turboquant/decompress.rs index 467662184ff..737faeea9f6 100644 --- a/vortex-tensor/src/encodings/turboquant/decompress.rs +++ b/vortex-tensor/src/encodings/turboquant/decompress.rs @@ -12,6 +12,7 @@ use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::match_each_float_ptype; @@ -20,6 +21,7 @@ use vortex_buffer::BufferMut; use vortex_error::VortexResult; use crate::encodings::turboquant::TurboQuant; +use crate::encodings::turboquant::TurboQuantArrayExt; use crate::encodings::turboquant::array::float_from_f32; use crate::encodings::turboquant::array::rotation::RotationMatrix; use crate::utils::tensor_element_ptype; diff --git a/vortex-tensor/src/encodings/turboquant/mod.rs b/vortex-tensor/src/encodings/turboquant/mod.rs index 59554f398d2..72e1c0714b7 100644 --- a/vortex-tensor/src/encodings/turboquant/mod.rs +++ b/vortex-tensor/src/encodings/turboquant/mod.rs @@ -91,6 +91,7 @@ //! ``` mod array; +pub use array::data::TurboQuantArrayExt; pub use array::data::TurboQuantData; pub use array::scheme::TurboQuantScheme; diff --git a/vortex-tensor/src/encodings/turboquant/tests.rs b/vortex-tensor/src/encodings/turboquant/tests.rs index ef77bb9e3ec..a9fb12a4f31 100644 --- a/vortex-tensor/src/encodings/turboquant/tests.rs +++ b/vortex-tensor/src/encodings/turboquant/tests.rs @@ -13,6 +13,8 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::dtype::extension::ExtDType; use vortex_array::extension::EmptyMetadata; use vortex_array::session::ArraySession; @@ -22,6 +24,7 @@ use vortex_error::VortexResult; use vortex_session::VortexSession; use crate::encodings::turboquant::TurboQuant; +use crate::encodings::turboquant::TurboQuantArrayExt; use crate::encodings::turboquant::TurboQuantConfig; use crate::encodings::turboquant::array::rotation::RotationMatrix; use crate::encodings::turboquant::turboquant_encode; diff --git a/vortex-tensor/src/encodings/turboquant/vtable.rs b/vortex-tensor/src/encodings/turboquant/vtable.rs index 39915b320f3..93a6bc1b43b 100644 --- a/vortex-tensor/src/encodings/turboquant/vtable.rs +++ b/vortex-tensor/src/encodings/turboquant/vtable.rs @@ -34,6 +34,7 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::encodings::turboquant::TurboQuantArrayExt; use crate::encodings::turboquant::TurboQuantData; use crate::encodings::turboquant::array::slots::Slot; use crate::encodings::turboquant::compute::rules::PARENT_KERNELS; @@ -86,9 +87,20 @@ impl TurboQuant { centroids: ArrayRef, rotation_signs: ArrayRef, ) -> VortexResult { - let data = TurboQuantData::try_new(&dtype, codes, norms, centroids, rotation_signs)?; - - let parts = ArrayParts::new(TurboQuant, dtype, data.norms().len(), data); + let len = norms.len(); + TurboQuantData::validate(&dtype, &codes, &norms, ¢roids, &rotation_signs)?; + let ext = TurboQuant::validate_dtype(&dtype)?; + let dimension = tensor_list_size(ext)?; + let bit_width = if centroids.is_empty() { + 0 + } else { + u8::try_from(centroids.len().trailing_zeros()) + .map_err(|_| vortex_err!("centroids bit_width does not fit in u8"))? + }; + let data = TurboQuantData::try_new(dimension, bit_width)?; + let parts = ArrayParts::new(TurboQuant, dtype, len, data).with_slots( + TurboQuantData::make_slots(codes, norms, centroids, rotation_signs), + ); Array::try_from_parts(parts) } @@ -96,6 +108,19 @@ impl TurboQuant { vtable!(TurboQuant, TurboQuant, TurboQuantData); +impl ArrayHash for TurboQuantData { + fn array_hash(&self, state: &mut H, _precision: Precision) { + self.dimension.hash(state); + self.bit_width.hash(state); + } +} + +impl ArrayEq for TurboQuantData { + fn array_eq(&self, other: &Self, _precision: Precision) -> bool { + self.dimension == other.dimension && self.bit_width == other.bit_width + } +} + impl VTable for TurboQuant { type ArrayData = TurboQuantData; type OperationsVTable = TurboQuant; @@ -105,61 +130,60 @@ impl VTable for TurboQuant { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - let ext = dtype - .as_extension_opt() - .filter(|e| e.is::()) - .ok_or_else(|| { - vortex_err!("TurboQuant dtype must be a Vector extension type, got {dtype}") - })?; - - let dimension = tensor_list_size(ext)?; - vortex_ensure!( - dimension >= Self::MIN_DIMENSION, - "TurboQuant requires dimension >= {}, got {dimension}", - Self::MIN_DIMENSION + fn validate( + &self, + data: &Self::ArrayData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + vortex_ensure_eq!( + slots.len(), + Slot::COUNT, + "TurboQuantArray expects {} slots, got {}", + Slot::COUNT, + slots.len() ); - vortex_ensure_eq!(data.dimension(), dimension); + let codes = slots[Slot::Codes as usize] + .as_ref() + .ok_or_else(|| vortex_err!("TurboQuantArray codes slot"))?; + let norms = slots[Slot::Norms as usize] + .as_ref() + .ok_or_else(|| vortex_err!("TurboQuantArray norms slot"))?; + let centroids = slots[Slot::Centroids as usize] + .as_ref() + .ok_or_else(|| vortex_err!("TurboQuantArray centroids slot"))?; + let rotation_signs = slots[Slot::RotationSigns as usize] + .as_ref() + .ok_or_else(|| vortex_err!("TurboQuantArray rotation_signs slot"))?; + + TurboQuantData::validate(dtype, codes, norms, centroids, rotation_signs)?; + vortex_ensure_eq!( + norms.len(), + len, + "TurboQuant norms length does not match outer length", + ); + vortex_ensure_eq!( + data.dimension, + tensor_list_size(TurboQuant::validate_dtype(dtype)?)? + ); - // TODO(connor): In the future, we may not need to validate `len` on the array data because - // the child arrays will be located somewhere else. - // bit_width == 0 is only valid for degenerate (empty) arrays. A non-empty array with - // bit_width == 0 would have zero centroids while codes reference centroid indices. - vortex_ensure!( - data.bit_width > 0 || len == 0, - "bit_width == 0 is only valid for empty arrays, got len={len}" + let expected_bit_width = if centroids.is_empty() { + 0 + } else { + u8::try_from(centroids.len().trailing_zeros()) + .map_err(|_| vortex_err!("centroids bit_width does not fit in u8"))? + }; + vortex_ensure_eq!( + data.bit_width, + expected_bit_width, + "TurboQuant bit_width does not match centroids slot", ); Ok(()) } - fn array_hash(array: &TurboQuantData, state: &mut H, precision: Precision) { - array.dimension.hash(state); - array.bit_width.hash(state); - for slot in &array.slots { - slot.is_some().hash(state); - if let Some(child) = slot { - child.array_hash(state, precision); - } - } - } - - fn array_eq(array: &TurboQuantData, other: &TurboQuantData, precision: Precision) -> bool { - array.dimension == other.dimension - && array.bit_width == other.bit_width - && array.slots.len() == other.slots.len() - && array - .slots - .iter() - .zip(other.slots.iter()) - .all(|(a, b)| match (a, b) { - (Some(a), Some(b)) => a.array_eq(b, precision), - (None, None) => true, - _ => false, - }) - } - fn nbuffers(_array: ArrayView) -> usize { 0 } @@ -184,7 +208,7 @@ impl VTable for TurboQuant { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult> { vortex_ensure_eq!( metadata.len(), 1, @@ -239,37 +263,26 @@ impl VTable for TurboQuant { let signs_dtype = DType::Primitive(PType::U8, Nullability::NonNullable); let rotation_signs = children.get(3, &signs_dtype, signs_len)?; - Ok(TurboQuantData { - slots: vec![ - Some(codes_array), - Some(norms_array), - Some(centroids), - Some(rotation_signs), - ], - dimension, - bit_width, - }) - } - - fn slots(array: ArrayView<'_, Self>) -> &[Option] { - &array.data().slots + Ok(ArrayParts::new( + TurboQuant, + dtype.clone(), + len, + TurboQuantData { + dimension, + bit_width, + }, + ) + .with_slots(TurboQuantData::make_slots( + codes_array, + norms_array, + centroids, + rotation_signs, + ))) } fn slot_name(_array: ArrayView, idx: usize) -> String { Slot::from_index(idx).name().to_string() } - - fn with_slots(array: &mut TurboQuantData, slots: Vec>) -> VortexResult<()> { - vortex_ensure!( - slots.len() == Slot::COUNT, - "TurboQuantArray expects {} slots, got {}", - Slot::COUNT, - slots.len() - ); - array.slots = slots; - Ok(()) - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done(execute_decompress(array, ctx)?)) } @@ -293,7 +306,7 @@ impl VTable for TurboQuant { } impl ValidityChild for TurboQuant { - fn validity_child(array: &TurboQuantData) -> &ArrayRef { - array.norms() + fn validity_child(array: ArrayView<'_, TurboQuant>) -> ArrayRef { + array.norms().clone() } } diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index acf867c5cd8..0b7064fb783 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -12,6 +12,7 @@ use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; +use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; @@ -172,8 +173,8 @@ impl ScalarFnVTable for InnerProduct { // Extract the storage array from each extension input. We pass the storage (FSL) rather // than the extension array to avoid canonicalizing the extension wrapper. - let lhs_storage = lhs.data().storage_array(); - let rhs_storage = rhs.data().storage_array(); + let lhs_storage = lhs.storage_array(); + let rhs_storage = rhs.storage_array(); let lhs_flat = extract_flat_elements(lhs_storage, list_size, ctx)?; let rhs_flat = extract_flat_elements(rhs_storage, list_size, ctx)?; diff --git a/vortex-tensor/src/scalar_fns/l2_norm.rs b/vortex-tensor/src/scalar_fns/l2_norm.rs index dfb650fff42..aa695044a05 100644 --- a/vortex-tensor/src/scalar_fns/l2_norm.rs +++ b/vortex-tensor/src/scalar_fns/l2_norm.rs @@ -12,6 +12,7 @@ use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; +use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; @@ -30,6 +31,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use crate::encodings::turboquant::TurboQuant; +use crate::encodings::turboquant::TurboQuantArrayExt; use crate::matcher::AnyTensor; use crate::scalar_fns::ApproxOptions; use crate::utils::extract_flat_elements; @@ -146,7 +148,7 @@ impl ScalarFnVTable for L2Norm { let ext = input.dtype().as_extension(); let list_size = tensor_list_size(ext)? as usize; - let storage = input.data().storage_array(); + let storage = input.storage_array(); let flat = extract_flat_elements(storage, list_size, ctx)?; match_each_float_ptype!(flat.ptype(), |T| { diff --git a/vortex-tensor/src/utils.rs b/vortex-tensor/src/utils.rs index e3f231be83b..b17fa04d110 100644 --- a/vortex-tensor/src/utils.rs +++ b/vortex-tensor/src/utils.rs @@ -8,6 +8,7 @@ use vortex_array::arrays::Constant; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::PType; diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bytebool.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bytebool.rs index aa9b299472a..53cf79463d6 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bytebool.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bytebool.rs @@ -6,6 +6,7 @@ use vortex::array::ArrayRef; use vortex::array::IntoArray; use vortex::array::arrays::BoolArray; use vortex::array::arrays::StructArray; +use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::dtype::FieldNames; use vortex::array::validity::Validity; use vortex::encodings::bytebool::ByteBool; diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/patched.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/patched.rs index 3241de111d2..b9a43dfb1e2 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/patched.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/patched.rs @@ -6,7 +6,6 @@ use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::Patched; -use vortex_array::arrays::PatchedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::patches::Patches; use vortex_array::vtable::ArrayVTable; @@ -38,7 +37,7 @@ impl FlatLayoutFixture for PatchedFixture { None, )?; - Ok(PatchedArray::from_array_and_patches(array, &patches, &mut ctx)?.into_array()) + Ok(Patched::from_array_and_patches(array, &patches, &mut ctx)?.into_array()) } fn expected_encodings(&self) -> Vec { diff --git a/vortex-tui/src/browse/ui/layouts.rs b/vortex-tui/src/browse/ui/layouts.rs index 36a895c7269..a00c363610e 100644 --- a/vortex-tui/src/browse/ui/layouts.rs +++ b/vortex-tui/src/browse/ui/layouts.rs @@ -28,6 +28,7 @@ use ratatui::widgets::Widget; use ratatui::widgets::Wrap; use vortex::array::ArrayRef; use vortex::array::ToCanonical; +use vortex::array::arrays::struct_::StructArrayExt; use vortex::error::VortexExpect; use vortex::layout::layouts::flat::Flat; use vortex::layout::layouts::zoned::Zoned; diff --git a/vortex/benches/common_encoding_tree_throughput.rs b/vortex/benches/common_encoding_tree_throughput.rs index 6f48a19de3a..9098589426d 100644 --- a/vortex/benches/common_encoding_tree_throughput.rs +++ b/vortex/benches/common_encoding_tree_throughput.rs @@ -21,19 +21,24 @@ use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::TemporalArray; use vortex::array::arrays::VarBinArray; use vortex::array::arrays::VarBinViewArray; +use vortex::array::arrays::varbin::VarBinArrayExt; use vortex::array::builtins::ArrayBuiltins; use vortex::dtype::DType; use vortex::dtype::PType; use vortex::encodings::alp::ALP; +use vortex::encodings::alp::ALPArrayExt; use vortex::encodings::alp::alp_encode; use vortex::encodings::datetime_parts::DateTimeParts; use vortex::encodings::datetime_parts::split_temporal; use vortex::encodings::fastlanes::BitPacked; use vortex::encodings::fastlanes::FoR; +use vortex::encodings::fastlanes::FoRArrayExt; use vortex::encodings::fsst::FSST; +use vortex::encodings::fsst::FSSTArrayExt; use vortex::encodings::fsst::fsst_compress; use vortex::encodings::fsst::fsst_train_compressor; use vortex::encodings::runend::RunEnd; +use vortex::encodings::runend::RunEndArrayExt; use vortex::error::VortexExpect; use vortex::extension::datetime::TimeUnit; diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index b5531beb767..5e27475a90f 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -155,8 +155,8 @@ fn bench_delta_compress_u32(bencher: Bencher) { with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| &uint_array) .bench_refs(|a| { - let (bases, deltas) = delta_compress(a, &mut SESSION.create_execution_ctx()).unwrap(); - DeltaData::try_new(bases.into_array(), deltas.into_array(), 0, a.len()).unwrap() + let (_bases, _deltas) = delta_compress(a, &mut SESSION.create_execution_ctx()).unwrap(); + DeltaData::try_new(0).unwrap() }); }